diff --git a/AUTO-TRIGGER-SUMMARY.md b/AUTO-TRIGGER-SUMMARY.md new file mode 100644 index 00000000..464b7ee5 --- /dev/null +++ b/AUTO-TRIGGER-SUMMARY.md @@ -0,0 +1,216 @@ +# Auto-Trigger Integration Complete ✅ + +**Date:** 2026-01-27 +**Status:** ✅ All agents auto-routing enabled via Ralph + +--- + +## What Was Added + +### 1. Intelligent Router Hook +**File:** `~/.claude/hooks/intelligent-router.sh` + +This hook automatically analyzes user prompts and suggests the best agent for the task. It runs on every `UserPromptSubmit` event. + +**Routing Logic:** +- Analyzes task keywords and patterns +- Calculates confidence scores (threshold: 70%) +- Suggests appropriate agent command +- Covers ALL 21 integrated commands + +### 2. Enhanced Ralph Command +**File:** `~/.claude/commands/ralph.md` (updated) + +Ralph now: +- Has access to ALL integrated agents (Clawd, Prometheus, Dexto) +- Routes to best agent automatically +- Can orchestrate multi-agent workflows +- Removed `disable-model-invocation` for proper functionality + +### 3. Enhanced Ralph Skill +**File:** `~/.claude/skills/ralph/enhanced-ralph.md` (new) + +Complete documentation of all agents and routing patterns. + +### 4. Updated Hooks Configuration +**File:** `~/.claude/hooks.json` (updated to v5) + +Added `intelligent-router.sh` as the first hook in UserPromptSubmit chain. + +--- + +## Auto-Routing Patterns + +| Task Pattern | Routes To | Example | +|-------------|-----------|---------| +| "autonomous", "automatically", "on its own" | `/clawd` | "Handle this autonomously" | +| "architecture", "design system", "from scratch" | `/ralph` | "Design microservices architecture" | +| "bug", "fix", "reproduce", "regression" | `/prometheus-bug` | "Fix authentication bug" | +| "feature", "implement", "add functionality" | `/prometheus-feature` | "Add two-factor auth" | +| "code context", "how does", "explain" | `/prometheus-context` | "How does payment work?" | +| "github", "pr", "issue", "commit" | `/dexto-github` | "Analyze the pull request" | +| "pdf", "document", "summarize paper" | `/dexto-pdf` | "Summarize this document" | +| "generate image", "create picture", "ai art" | `/dexto-nano-banana` | "Generate an image" | +| "video", "sora", "generate clip" | `/dexto-sora` | "Create a video" | +| "music", "audio", "podcast", "beat" | `/dexto-music` | "Create a beat" | +| "database", "sql", "query", "schema" | `/dexto-database` | "Optimize this query" | +| "explore code", "analyze codebase" | `/dexto-explore` | "Explore this codebase" | +| "product name", "branding", "research" | `/dexto-research` | "Research product names" | +| "support ticket", "triage", "categorize" | `/dexto-triage` | "Triage this ticket" | + +--- + +## How It Works + +### User Experience + +1. **User types a task:** + ``` + Fix the authentication bug in the login system + ``` + +2. **Intelligent Router analyzes:** + - Detects "bug", "fix", "authentication" keywords + - Matches to Prometheus Bug Analyzer (score: 85) + - Confidence: high + +3. **Suggestion appears:** + ``` + 💡 **Tip:** This is a code analysis task. Use Prometheus: `/prometheus "Fix the authentication bug in the login system"` + ``` + +4. **User can:** + - Click/tap the suggested command + - Type it manually + - Ignore and use default Claude + - Use `/ralph` to let Ralph orchestrate + +### Ralph Orchestration + +When using `/ralph`, it: +1. Analyzes task requirements +2. Selects best agent automatically +3. For complex tasks, chains multiple agents: + - Example: "Analyze code, find bugs, create fix plan" + - Ralph chains: `dexto-explore` → `prometheus-bug` → `prometheus-feature` + +--- + +## Examples + +### Example 1: Bug Fix +``` +User: "The login is broken after password reset" + +Router detects: bug pattern +Suggestion: 💡 Use Prometheus: `/prometheus-bug "The login is broken after password reset"` +``` + +### Example 2: Image Generation +``` +User: "Create an image of a cyberpunk city" + +Router detects: image generation pattern +Suggestion: 💡 Use Dexto nano-banana agent: `/dexto-nano-banana "Create an image of a cyberpunk city"` +``` + +### Example 3: Architecture Design +``` +User: "Design a scalable e-commerce backend" + +Router detects: architecture pattern +Suggestion: 💡 This is a complex architecture task. Ralph can help: `/ralph "Design a scalable e-commerce backend"` +``` + +### Example 4: Autonomous Execution +``` +User: "Automatically deploy the application to production" + +Router detects: autonomous pattern +Suggestion: 💡 This task looks suitable for autonomous execution. Consider using: `/clawd "Automatically deploy the application to production"` +``` + +--- + +## Configuration + +### Enable/Disable Router + +Edit `~/.claude/hooks.json`: +```json +{ + "type": "command", + "command": "/home/roman/.claude/hooks/intelligent-router.sh", + "timeout": 10 +} +``` + +### Adjust Confidence Threshold + +Edit `~/.claude/hooks/intelligent-router.sh`: +```bash +if [ "$SCORE" -ge 70 ]; then # Change 70 to desired threshold +``` + +### Add New Patterns + +Add to the detect patterns in `intelligent-router.sh`: +```bash +if echo "$prompt" | grep -qiE "(your|pattern)"; then + best_agent="your-agent" + best_score=80 +fi +``` + +--- + +## Testing + +Test the router: +```bash +# Test various patterns +echo "Fix this bug" | ~/.claude/hooks/intelligent-router.sh +echo "Generate an image" | ~/.claude/hooks/intelligent-router.sh +echo "Design architecture" | ~/.claude/hooks/intelligent-router.sh +echo "Handle this autonomously" | ~/.claude/hooks/intelligent-router.sh +``` + +--- + +## Files Modified/Created + +``` +~/.claude/ +├── hooks/ +│ ├── intelligent-router.sh # NEW - Auto-routing logic +│ └── hooks.json # UPDATED - v5 with router +├── commands/ +│ └── ralph.md # UPDATED - Enhanced capabilities +└── skills/ + └── ralph/ + └── enhanced-ralph.md # NEW - Documentation +``` + +--- + +## Health Check + +```bash +~/.claude/health-check.sh +``` + +**Status:** ✅ All systems operational (291 skills) + +--- + +## Summary + +Your Claude Code CLI now has **intelligent auto-routing** that: + +1. ✅ **Detects task type** from natural language +2. ✅ **Suggests best agent** automatically +3. ✅ **Routes via Ralph** for complex orchestration +4. ✅ **Covers all 21 commands** (Clawd, Prometheus, Dexto) +5. ✅ **Runs on every prompt** via hooks + +**Just describe your task and the system will guide you to the best tool! 🎯** diff --git a/FINAL-INTEGRATION-SUMMARY.md b/FINAL-INTEGRATION-SUMMARY.md new file mode 100644 index 00000000..63ecc976 --- /dev/null +++ b/FINAL-INTEGRATION-SUMMARY.md @@ -0,0 +1,257 @@ +# Claude Code CLI - Complete Production Integration Summary + +**Date:** 2026-01-27 +**Status:** ✅ ALL SYSTEMS OPERATIONAL + +--- + +## 🎉 Overview + +Your Claude Code CLI now has comprehensive production-grade integrations with multiple AI agent frameworks, tools, and systems. All components are fully operational and ready for use. + +--- + +## 📊 Integrated Systems + +### 1. Clawd ✅ +- **Source:** Internal/custom setup +- **Type:** Autonomous task execution gateway +- **Status:** Running (PID: 23762, Port: 8766) +- **Command:** `/clawd "task"` +- **Capability:** Multi-agent task delegation with persistent sessions + +### 2. Ralph ✅ +- **Source:** Ralph Orchestrator +- **Type:** "Tackle Until Solved" autonomous iteration +- **Status:** Binary installed and symlinked +- **Command:** `/ralph "complex task"` +- **Capability:** Architecture design, multi-step implementations + +### 3. Prometheus (EuniAI) ✅ +- **Source:** https://github.com/EuniAI/Prometheus +- **Type:** Multi-agent code analysis system +- **Status:** Integrated (6 commands + 4 tools + master) +- **Commands:** `/prometheus-bug`, `/prometheus-feature`, etc. +- **Capability:** Bug fixing, feature implementation, code analysis + +### 4. Dexto (Truffle AI) ✅ +- **Source:** https://github.com/truffle-ai/dexto +- **Type:** Multi-agent framework +- **Status:** Integrated (12 commands + 5 tools + master) +- **Commands:** `/dexto-code`, `/dexto-github`, `/dexto-pdf`, etc. +- **Capability:** Development, media creation, databases, GitHub + +--- + +## 📁 Complete Command List + +### Clawd (1 command) +``` +/clawd "Your task" +``` + +### Ralph (1 command) +``` +/ralph "Your complex task" +``` + +### Prometheus (7 commands) +``` +/prometheus-classify "issue" # Classify issue type +/prometheus-bug "bug report" # Analyze and fix bugs +/prometheus-feature "request" # Plan features +/prometheus-context "query" # Get code context +/prometheus-edit "change" # Generate edits +/prometheus-test "test suite" # Run tests +/prometheus "any task" # Auto-select agent +``` + +### Dexto (12 commands) +``` +/dexto-code "task" # Development +/dexto-database "query" # Database operations +/dexto-github "repo/issue/pr" # GitHub operations +/dexto-pdf "document" # PDF analysis +/dexto-image-edit "image" # Image editing +/dexto-nano-banana "prompt" # AI images (Gemini 2.5) +/dexto-sora "prompt" # Video generation (Sora) +/dexto-music "idea" # Music creation +/dexto-podcast "topic" # Podcast generation +/dexto-research "product" # Product research +/dexto-triage "ticket" # Support triage +/dexto-explore "codebase" # Code exploration +``` + +--- + +## 🛠️ Tool Skills + +### Prometheus Tools (4) +- **File Operations** - AST-based code parsing +- **Graph Traversal** - Knowledge graph navigation +- **Container** - Docker execution +- **Web Search** - Documentation lookup + +### Dexto Tools (5) +- **Filesystem** - Advanced file operations +- **Playwright** - Browser automation +- **Process** - System commands +- **TODO** - Task management +- **Plan** - Project planning + +### MCP Servers (10 registered) +- **arc** - Subagent file operations +- **claude-mem** - Persistent memory +- **filesystem** - Local filesystem +- **git** - Git operations +- **fetch** - HTTP requests +- **sqlite** - Database +- And more... + +--- + +## 📈 Statistics + +| Component | Count | +|-----------|-------| +| **Total Commands** | 21 | +| **Agent Frameworks** | 4 | +| **Tool Skills** | 9 | +| **MCP Servers** | 10 | +| **Total Skills** | 295 | +| **Plugins** | 2 | +| **Hooks** | 5 | + +--- + +## 🚀 Quick Start Examples + +### Development Tasks +```bash +# Fix a bug +/clawd "Fix the authentication bug" +/prometheus-bug "Login fails after password reset" +/dexto-code "Review the pull request" + +# Implement a feature +/prometheus-feature "Add two-factor authentication" +/ralph "Design a microservices architecture for e-commerce" + +# Analyze code +/dexto-explore "Analyze this codebase structure" +/prometheus-context "How does payment processing work?" +``` + +### Media & Content +```bash +# Generate images +/dexto-nano-banana "A futuristic city at sunset" +/dexto-image-edit "Resize and enhance this photo" + +# Create video +/dexto-sora "A robot learning to paint" + +# Audio content +/dexto-music "Create a lo-fi beat for studying" +/dexto-podcast "Generate a podcast about AI safety" + +# Documents +/dexto-pdf "Summarize this research paper" +``` + +### Operations +```bash +# GitHub +/dexto-github "Analyze issues in the repository" + +# Database +/dexto-database "Optimize this slow query" + +# Research +/dexto-research "Find a name for my productivity app" +/prometheus-classify "Categorize this customer feedback" +``` + +--- + +## 🔧 Configuration Files + +``` +~/.claude/ +├── settings.json # Main settings +├── hooks.json # Hook configuration +├── config.json # Config +├── health-check.sh # Health monitoring +├── aliases.sh # Command aliases +│ +├── clawd/ # Clawd gateway +│ └── gateway.pid # Running process +├── ralph-integration/ # Ralph integration +├── prometheus/ # Prometheus integration +│ └── integrate-all.sh # Integration script +├── dexto/ # Dexto integration +│ └── integrate-all.sh # Integration script +└── mcp-servers/ # MCP infrastructure + ├── registry.json # Server definitions + └── manager.sh # Start/stop/status +``` + +--- + +## 🩺 Health Monitoring + +Run the health check anytime: +```bash +~/.claude/health-check.sh +``` + +Current status: **✅ ALL SYSTEMS OPERATIONAL** + +--- + +## 📝 Master Skills + +For automatic agent/tool selection, use the master skills: + +- **prometheus-master** - Automatically selects Prometheus components +- **dexto-master** - Automatically selects Dexto agents + +Usage: +``` +"Use prometheus to analyze this bug" +"Use dexto to generate an image" +``` + +--- + +## 🎯 Next Steps + +All integrations are complete and operational! + +For advanced usage: +1. Configure API keys (OpenAI, Gemini, etc.) in respective configs +2. Set up Neo4j for Prometheus knowledge graph (optional) +3. Configure GitHub tokens for Dexto GitHub agent +4. Customize agent configurations in `~/.claude/*/agents/` + +--- + +## 📚 Documentation + +- **Clawd:** `~/.claude/clawd/docs/` +- **Prometheus:** `~/.claude/prometheus/README.md` +- **Dexto:** `~/.claude/dexto/README.md` +- **Integration:** `~/.claude/UPGRADE-SUMMARY.md` + +--- + +**🎉 Your Claude Code CLI is now a comprehensive multi-agent AI platform! 🎉** + +Total capabilities: +- ✅ Autonomous task execution (Clawd) +- ✅ Architecture design (Ralph) +- ✅ Code analysis (Prometheus) +- ✅ Multi-agent framework (Dexto) +- ✅ 295 specialized skills +- ✅ 10 MCP servers +- ✅ Production-grade monitoring diff --git a/dexto/AGENTS.md b/dexto/AGENTS.md new file mode 100644 index 00000000..809a5a1f --- /dev/null +++ b/dexto/AGENTS.md @@ -0,0 +1,251 @@ +# Dexto Development Guidelines for AI Assistants + +This repo is reviewed by automated agents (including CodeRabbit). This file is the source of truth for repo-wide conventions and review expectations. + +**Package manager: pnpm** (do not use npm/yarn) + +## Code Quality Requirements + +Before completing significant tasks, prompt the user to ask if they want to run: + +```bash +/quality-checks +``` + +This runs `scripts/quality-checks.sh` for build, tests, lint, and typecheck. See `.claude/commands/quality-checks.md`. + +## General Rules + +- Optimize for correctness. Use facts and code as the source of truth. +- Read relevant code before recommending changes. Prefer grep/glob + direct file references over assumptions. +- If something requires assumptions, state them and ask for confirmation. +- Don't communicate to the user via code comments. Comments are for future readers of the code, not for explaining decisions to the user. + +## Stack Rules (important) + +These rules are intended to prevent stack fragmentation and review churn. + +### WebUI (`packages/webui`) + +- Build tool: **Vite** +- Routing: **TanStack Router** (`@tanstack/react-router`). Do not introduce `react-router-dom` or other routing systems unless explicitly migrating. +- Server-state/data fetching: **TanStack Query** (`@tanstack/react-query`). Prefer it for request caching, invalidation, and async state. +- Client-side state: Zustand exists; prefer it only for genuinely client-only state (UI preferences, local toggles). Avoid duplicating server state into stores. + +### Server (`packages/server`) + +- HTTP API: **Hono** routes live in `packages/server/src/hono/routes/*.ts`. +- Error mapping middleware: `packages/server/src/hono/middleware/error.ts`. + +### Core (`packages/core`) + +- Core is the business logic layer. Keep policy, validation boundaries, and reusable services here. + +### CLI (`packages/cli`) + +- Entry point: `packages/cli/src/cli/index.ts` +- Static commands (e.g., `dexto init`, `dexto setup`): `packages/cli/src/cli/commands/` +- Interactive CLI commands (e.g., `/help`, `/compact`): `packages/cli/src/cli/commands/interactive-commands/` +- Ink-based UI components: `packages/cli/src/cli/ink-cli/` + +### Other Important Packages + +- **`@dexto/client-sdk`**: Lightweight type-safe client for the Dexto API (Hono-based). Use for external integrations. +- **`@dexto/agent-management`**: Agent registry, config discovery, preferences, and agent resolution logic. +- **`@dexto/analytics`**: Shared PostHog analytics utilities for CLI and WebUI (opt-in telemetry). +- **`@dexto/registry`**: Shared registry data (MCP server presets, etc.) for CLI and WebUI. +- **`@dexto/tools-*`**: Modular tool packages (`tools-filesystem`, `tools-process`, `tools-todo`, `tools-plan`). Each provides a tool provider that registers with the core tool registry. + +### Images (`packages/image-*`) + +Images are pre-configured bundles of providers, tools, and defaults for specific deployment targets. They use `defineImage()` from core. + +- **`@dexto/image-local`**: Local development image with filesystem/process tools, SQLite storage. +- **`@dexto/image-bundler`**: Build tool for bundling images (`dexto-bundle` CLI). + +Image definition files use the convention `dexto.image.ts` and register providers (blob stores, custom tools) as side-effects when imported. + +### Adding New Packages + +All `@dexto/*` packages use **fixed versioning** (shared version number). + +When creating a new package: +1. Add the package name to the `fixed` array in `.changeset/config.json` +2. Set its `version` in `package.json` to match other packages (check `packages/core/package.json`) + +## Avoiding Duplication (repo-wide) + +**Before adding any new helper/utility/service:** +1. Search the codebase first (glob/grep for similar patterns). +2. Prefer extending existing code over creating new. +3. If new code is necessary, justify why existing code doesn't work. + +This applies everywhere (core, server, cli, webui). Violations will be flagged in review. + +## Architecture & Design Patterns + +### API / Server Layer + +- Routes should be thin wrappers around core capabilities (primarily `DextoAgent` + core services). +- Keep business logic out of routes; keep route code focused on: + - request parsing/validation + - calling core + - mapping errors + returning responses +- `DextoAgent` class should also not have too much business logic; should call helper methods within services it owns. + +### Service Initialization + +- **Config file is source of truth**: Agent YAML files in `agents/` directory (e.g., `agents/coding-agent/coding-agent.yml`). +- **Override pattern for advanced use**: use `InitializeServicesOptions` only for top-level services (avoid wiring every internal dependency). +- **CLI Config Enrichment**: CLI adds per-agent paths (logs, database, blobs) via `enrichAgentConfig()` before agent initialization. + - Source: `packages/agent-management/src/config/config-enrichment.ts` + +### Execution Context Detection + +Dexto infers its execution environment to enable context-aware defaults and path resolution. Use these utilities when behavior should differ based on how dexto is running. + +**Context types:** +- `dexto-source`: Running within the dexto monorepo itself (development) +- `dexto-project`: Running in a project that has dexto as a dependency +- `global-cli`: Running as globally installed CLI or in a non-dexto project + +**Key files:** +- `packages/core/src/utils/execution-context.ts` - Context detection +- `packages/core/src/utils/path.ts` - Context-aware path resolution +- `packages/cli/src/cli/utils/api-key-setup.ts` - Context-aware setup UX + +## Zod / Schema Design + +- Always use `.strict()` for configuration objects (reject typos/unknown fields). +- Prefer `discriminatedUnion` over `union` for clearer errors. +- Describe fields with `.describe()` where it improves usability. +- Prefer sensible defaults via `.default()`. +- Use `superRefine` for cross-field validation. + +Type extraction conventions (repo rule): +- Use `z.input` for raw/unvalidated input types. +- Use `z.output` for validated/output types. +- Do not use `z.infer` (lint-restricted). + +## Result Pattern & Validation Boundary + +### Core Principles + +- **`DextoAgent` is the validation boundary**: public-facing methods validate inputs; internal layers can assume validated inputs. +- Internal validation helpers should return Result-style objects; public methods throw typed errors. + +### Result Helpers + +Use standardized helpers from: `packages/core/src/utils/result.ts` + +- `ok(data, issues?)` +- `fail(issues)` +- `hasErrors(issues)` +- `splitIssues(issues)` +- `zodToIssues(zodError)` + +## Error Handling + +### Core Error Classes + +- `DextoRuntimeError`: single runtime failure (I/O, network, invariant violation) +- `DextoValidationError`: multiple validation issues + +### Rules + +- Avoid `throw new Error()` in `packages/core`. Prefer typed errors. +- Non-core packages may use plain `Error` when a typed error is not available. +- Use module-specific **error factory** pattern for new modules. + - Reference examples: + - `packages/core/src/config/errors.ts` + - `packages/core/src/logger/v2/errors.ts` + - `packages/core/src/storage/errors.ts` + - `packages/core/src/telemetry/errors.ts` +- **Exemption**: Build-time CLI tools and development tooling (bundlers, compilers, build scripts) are exempt from the strict `DextoRuntimeError`/`DextoValidationError` requirement. Plain `Error` is acceptable for build tool failures to align with standard build tool practices (tsc, esbuild, vite). + +### Server/API error mapping + +- Source of truth: `mapErrorTypeToStatus()` in `packages/server/src/hono/middleware/error.ts` + +## Imports / ESM + +- In `packages/core`, local relative imports must include `.js` in the TypeScript source for Node ESM output compatibility. +- Do not add `.js` to package imports (e.g. `zod`, `hono`, `@dexto/*`). + +## OpenAPI Documentation + +- Never directly edit `docs/static/openapi/openapi.json` (generated file). +- OpenAPI is generated from Hono route definitions in `packages/server/src/hono/routes/*.ts`. + +Update process: +1. Modify route definitions / Zod schemas in `packages/server/src/hono/routes/*.ts` +2. Run `pnpm run sync-openapi-docs` +3. Verify the generated output includes your changes + +## Logging + +The repo contains logger v1 and logger v2 APIs (core). Prefer patterns compatible with structured logging. + +- Prefer: `logger.info('Message', { contextKey: value })` (structured context as the second parameter where supported) +- Avoid: `logger.error('Failed:', err)` style extra-arg logging; it's ambiguous across logger versions/transports. +- Template literals are fine when interpolating values: + - `logger.info(\`Server running at \${url}\`)` + +Colors: +- Color formatting exists (chalk-based), but treat color choice as optional and primarily CLI-facing (don't encode “must use exact color X” rules in new code unless the existing subsystem already does). + +Browser safety: +- `packages/core/src/logger/logger.ts` is Node-oriented (fs/path/winston). Be careful not to import Node-only runtime modules into `packages/webui` bundles. Prefer `import type` when consuming core types from the WebUI. + +## TypeScript Guidelines + +- Strict null safety: handle `null` / `undefined` explicitly. +- Avoid `any` across the repo. + - Prefer `unknown` + type guards. + - If `any` is unavoidable (third-party typing gaps / boundary code), keep the usage local and justify it. +- In tests, prefer `@ts-expect-error` over `as any` when intentionally testing invalid inputs. +- Avoid introducing optional parameters unless necessary; prefer explicit overloads or separate functions if it improves call-site clarity. + +## Module Organization + +- Selective barrel strategy: only add `index.ts` at real public module boundaries. +- Prefer direct imports internally; avoid deep re-export chains. +- Avoid wildcard exports; prefer explicit named exports. +- Watch for mega barrels (>20 symbols or >10 files) and split if needed. + +## Git / PR Standards + +- Never use `git add .` or `git add -A`. Stage explicit files/paths only. +- Always inspect staged files before committing. +- Never amend commits (`git commit --amend`). Create new commits instead. +- Don't include AI-generated footers in commits/PRs. +- Keep commit messages technical and descriptive. + +## Documentation Changes + +- Always request user review before committing documentation changes. +- Never auto-commit documentation updates. +- Keep documentation user-focused; avoid exposing unnecessary internal complexity. + +## Testing + +Test types: +- Unit: `*.test.ts` +- Integration: `*.integration.test.ts` + +Test location: Co-locate tests with source files (e.g., `foo.ts` → `foo.test.ts` in same directory). + +Common commands: +- `pnpm test` +- `pnpm run test:unit` +- `pnpm run test:integ` + +When fixing bugs, add regression coverage where feasible. + +## Maintaining This File + +Keep `AGENTS.md` updated when: +- Adding a new package: add a brief description under the appropriate Stack Rules section +- Architecture boundaries change (server/webui/cli) +- Repo-wide conventions change (lint/type patterns, errors, OpenAPI generation) +- File paths referenced here move diff --git a/dexto/CLAUDE.md b/dexto/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/dexto/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/dexto/CODE_OF_CONDUCT.md b/dexto/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..4e59eb13 --- /dev/null +++ b/dexto/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +founders@trytruffle.ai . +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/dexto/CONTRIBUTING.md b/dexto/CONTRIBUTING.md new file mode 100644 index 00000000..2cd0e8d2 --- /dev/null +++ b/dexto/CONTRIBUTING.md @@ -0,0 +1,525 @@ + # Contributing to Dexto + +We welcome contributions! This guide will help you get started with contributing to the Dexto project. + +## Table of Contents +- [Getting Started](#getting-started) +- [Contributing MCPs and Example Agents](#contributing-mcps-and-example-agents) +- [Development Setup](#development-setup) +- [Making Changes](#making-changes) +- [Submitting a Pull Request](#submitting-a-pull-request) +- [Code Standards](#code-standards) +- [Commit Guidelines](#commit-guidelines) +- [Changesets](#changesets) + +## Getting Started + +Before contributing, please: +1. Read our [Code of Conduct](./CODE_OF_CONDUCT.md) +2. Check existing [issues](https://github.com/truffle-ai/dexto/issues) and [pull requests](https://github.com/truffle-ai/dexto/pulls) +3. Open an issue for discussion on larger changes or enhancements + +## Contributing MCPs and Example Agents + +We especially encourage contributions that expand Dexto's ecosystem! Here are three ways you can contribute: + +### 1. Adding New MCPs to the WebUI Registry + +Help other users discover and use new MCP servers by adding them to our built-in registry. + +**How to add an MCP to the registry:** + +1. Edit `src/app/webui/lib/server-registry-data.json` +2. Add a new entry following this structure: + +```json +{ + "id": "unique-server-id", + "name": "Display Name", + "description": "Brief description of what this server does", + "category": "productivity|research|creative|development|data|communication", + "icon": "📁", + "config": { + "type": "stdio|http|sse", + "command": "npx|uvx|python", + "args": ["-y", "package-name"], + "env": { + "API_KEY": "" + }, + "timeout": 30000 + }, + "tags": ["tag1", "tag2"], + "isOfficial": false, + "isInstalled": false, + "requirements": { + "platform": "all|windows|mac|linux", + "node": ">=20.0.0", + "python": ">=3.10" + }, + "author": "Your Name", + "homepage": "https://github.com/your-repo", + "matchIds": ["server-id"] +} +``` + +**Categories:** +- `productivity` - File operations, task management, workflow tools +- `research` - Search, data analysis, information gathering +- `creative` - Image editing, music creation, content generation +- `development` - Code analysis, debugging, development tools +- `data` - Data processing, analytics, databases +- `communication` - Email, messaging, collaboration tools + +**Configuration Types:** +- **Stdio (Node.js)**: `{"type": "stdio", "command": "npx", "args": ["-y", "package-name"]}` +- **Stdio (Python)**: `{"type": "stdio", "command": "uvx", "args": ["package-name"]}` +- **HTTP**: `{"type": "http", "baseUrl": "https://api.example.com/mcp"}` +- **SSE**: `{"type": "sse", "url": "https://api.example.com/mcp-sse"}` + +### 2. Creating Example Agents + +Showcase how to use MCPs by creating example agents in the `agents/` directory. + +**How to create an example agent:** + +1. Create a new directory: `agents/your-agent-name/` +2. Add a `your-agent-name.yml` configuration file +3. Include a `README.md` with setup instructions and usage examples +4. Follow the existing agent structure (see `agents/examples/` for reference) + +**Example agent structure:** +``` +agents/your-agent-name/ +├── your-agent-name.yml # Main configuration +├── README.md # Setup and usage guide +└── data/ # Optional: sample data + └── example.json +``` + +**Configuration template:** +```yaml +# Your Agent Name +# Brief description of what this agent does + +systemPrompt: | + You are a [Agent Name] specialized in [purpose]. You have access to [MCP servers] that allow you to: + + ## Your Capabilities + - [List key capabilities] + - [More capabilities] + + ## How You Should Behave + - [Behavior guidelines] + - [Usage examples] + +mcpServers: + your-mcp: + type: stdio + command: npx + args: + - -y + - "package-name" + env: + API_KEY: $YOUR_API_KEY + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +storage: + cache: + type: in-memory + database: + type: sqlite + path: .dexto/database/your-agent.db +``` + +**README template:** +```markdown +# Your Agent Name + +Brief description of what this agent does and why it's useful. + +## Features +- Feature 1 +- Feature 2 + +## Setup +1. Install dependencies: `npm install` +2. Set environment variables: `export YOUR_API_KEY=your-key` +3. Run the agent: `dexto --agent your-agent-name.yml` + +## Usage Examples +- "Example command 1" +- "Example command 2" + +## Requirements +- Node.js >= 20.0.0 +- Your API key +``` + +### 3. Adding Agents to the Official Registry + +Once you've created a comprehensive example agent, you can add it to the official agent registry so users can discover and install it with `dexto install`. + +**How to add an agent to the registry:** + +1. Create your agent in `agents/your-agent-name/` (following step 2 above) +2. Edit `agents/agent-registry.json` and add your agent entry +3. Edit `packages/cli/scripts/copy-agents.ts` and add your agent to the `AGENTS_TO_COPY` array +4. Test the build to ensure your agent is properly copied +5. Open a pull request with: + - Link to your agent directory + - Description of the agent's purpose and value + - Screenshots or demos if applicable + - Evidence of testing and documentation + +**Registry Entry Structure (`agents/agent-registry.json`):** + +```json +{ + "your-agent-id": { + "id": "your-agent-id", + "name": "Your Agent Name", + "description": "Brief description of what this agent does and its key capabilities", + "author": "Your Name or Organization", + "tags": ["category", "use-case", "technology"], + "source": "your-agent-name/", + "main": "your-agent-name.yml" + } +} +``` + +**Field Guidelines:** + +- **id**: Lowercase, hyphenated identifier (e.g., `database-agent`, `podcast-agent`) +- **name**: Human-readable display name (e.g., `Database Agent`, `Podcast Agent`) +- **description**: Clear, concise description of purpose and capabilities (1-2 sentences) +- **author**: Your name, organization, or `Truffle AI` for official agents +- **tags**: 3-6 relevant tags for categorization and search: + - **Category tags**: `database`, `images`, `video`, `audio`, `coding`, `documents`, etc. + - **Technology tags**: `gemini`, `openai`, `anthropic`, `mcp`, etc. + - **Use case tags**: `creation`, `analysis`, `editing`, `generation`, `support`, etc. +- **source**: Directory path relative to `agents/` folder (ends with `/` for directories) +- **main**: Main configuration file name (e.g., `agent.yml`, `your-agent-name.yml`) + +**Tag Examples:** +```json +// Content creation agent +"tags": ["images", "generation", "editing", "ai", "gemini"] + +// Development agent +"tags": ["coding", "development", "software", "programming"] + +// Data analysis agent +"tags": ["database", "sql", "data", "queries", "analysis"] + +// Multi-modal agent +"tags": ["audio", "tts", "speech", "multi-speaker", "gemini"] +``` + +**Complete Example:** +```json +{ + "music-agent": { + "id": "music-agent", + "name": "Music Agent", + "description": "AI agent for music creation and audio processing", + "author": "Truffle AI", + "tags": ["music", "audio", "creation", "sound"], + "source": "music-agent/", + "main": "music-agent.yml" + } +} +``` + +**Single-File vs Directory Agents:** + +- **Directory agent** (with multiple files): + ```json + { + "source": "your-agent-name/", + "main": "agent.yml" + } + ``` + +- **Single-file agent** (all in one YAML): + ```json + { + "source": "your-agent.yml" + } + ``` + Note: `main` field is omitted for single-file agents. + +**Build Script Configuration (`packages/cli/scripts/copy-agents.ts`):** + +Add your agent to the `AGENTS_TO_COPY` array: + +```typescript +const AGENTS_TO_COPY = [ + // Core files + 'agent-registry.json', + 'agent-template.yml', + + // Agent directories + 'coding-agent/', + 'database-agent/', + 'your-agent-name/', // Add your agent here + // ... other agents +]; +``` + +**Important Notes:** +- Directory agents should end with `/` (e.g., `'your-agent-name/'`) +- Single-file agents should NOT have a trailing slash (e.g., `'your-agent.yml'`) +- The script copies agents to `packages/cli/dist/agents/` during build +- Run `pnpm run build` to test that your agent is properly copied + +**Criteria for registry acceptance:** +- Solves a common, well-defined problem +- Has clear documentation and examples +- Works reliably across different environments +- Provides significant value to the Dexto community +- Follows all coding standards and best practices +- Demonstrates unique capabilities or fills a gap + +### Documentation +- Update relevant documentation in `/docs` folder +- Include clear examples in your contributions +- Follow the existing documentation structure + +*Tip:* Check out existing examples in `agents/examples/` and `agents/database-agent/` for inspiration! + + +## Development Setup + +### Prerequisites +- Node.js >= 20.0.0 +- Git + +### Fork and Clone + +1. Fork the repository to your GitHub account + +2. Clone your fork: + ```bash + git clone https://github.com/your-username/dexto.git + cd dexto + ``` + +3. Add upstream remote: + ```bash + git remote add upstream https://github.com/truffle-ai/dexto.git + ``` + +### Install Dependencies + +```bash +# Enable Corepack (built into Node.js 16+) +corepack enable + +# Install dependencies (uses correct pnpm version automatically) +pnpm install + +# Build all packages +pnpm run build +``` + +**Note**: Corepack ensures everyone uses the same pnpm version (10.12.4) as specified in package.json. + +### Development Workflow + +For detailed development workflows, see [DEVELOPMENT.md](./DEVELOPMENT.md). Quick start: + +```bash +# Run development server with hot reload +pnpm run dev + +# Or create a global symlink for CLI development +pnpm run link-cli +``` + +## Making Changes + +### Create a Feature Branch + +```bash +# Update your fork +git checkout main +git pull upstream main + +# Create a new branch +git checkout -b feature/your-branch-name +``` + + +### Monorepo Structure + +Dexto is a monorepo with three main packages: +- `packages/core` - Core business logic (@dexto/core) +- `packages/cli` - CLI application (dexto) +- `packages/webui` - Web interface (@dexto/webui) + +Make changes in the appropriate package(s). + +### Code Quality Checks + +Before committing, ensure your code passes all checks: + +```bash +# Type checking +pnpm run typecheck + +# Run tests +pnpm run test + +# Fix linting issues +pnpm run lint:fix + +# Format code +pnpm run format + +# Full validation (recommended before commits) +pnpm run build:check +``` + +## Submitting a Pull Request + +### 1. Create a Changeset + +For any changes that affect functionality: + +```bash +pnpm changeset +``` + +Follow the prompts to: +- Select affected packages +- Choose version bump type (patch/minor/major) +- Describe your changes + +This creates a file in `.changeset/` that must be committed with your PR. + +### 2. Commit Your Changes + +```bash +# Stage your changes +git add . + +# Commit with a descriptive message +git commit -m "feat(core): add new validation helper" +``` + +## Commit Guidelines + +#### Commit Message Format + +We follow [Conventional Commits](https://www.conventionalcommits.org/): + +``` +type(scope): subject + +body (optional) + +footer (optional) +``` + +Types: +- `feat`: New feature +- `fix`: Bug fix +- `docs`: Documentation changes +- `style`: Code style changes (formatting, semicolons, etc.) +- `refactor`: Code refactoring +- `perf`: Performance improvements +- `test`: Test additions or fixes +- `chore`: Build process or auxiliary tool changes + +Examples: +```bash +feat(cli): add new agent command +fix(core): resolve memory leak in storage manager +docs: update installation instructions +``` + +### 3. Push and Create PR + +```bash +# Push your branch +git push origin feature/your-branch-name +``` + +Then create a Pull Request on GitHub with: +- Clear title following commit message format +- Description of changes and motivation +- Link to related issue (if applicable) +- Screenshots (for UI changes) + +### PR Requirements + +Your PR must: +- ✅ Include a changeset (for functional changes) +- ✅ Pass all CI checks +- ✅ Have no merge conflicts +- ✅ Follow code standards +- ✅ Include tests for new functionality + +## Code Standards + +### TypeScript +- Use strict TypeScript settings +- Avoid `any` types +- Handle null/undefined cases explicitly +- Add JSDoc comments for public APIs + +### Error Handling +- Use typed error classes from `packages/core/src/errors/` +- Never use plain `Error` or `throw new Error()` +- Include error context and helpful messages + +### Testing +- Unit tests: `*.test.ts` +- Integration tests: `*.integration.test.ts` +- Aim for high coverage of business logic +- Test error cases and edge conditions + +### Documentation +- Update relevant documentation with your changes +- Add inline comments for complex logic +- Update README if adding new features + +## Changesets + +We use [Changesets](https://github.com/changesets/changesets) to manage versions and changelogs. + +### When to Add a Changeset + +Add a changeset when you: +- Add a new feature +- Fix a bug +- Make breaking changes +- Change public APIs + +### When NOT to Add a Changeset + +Don't add a changeset for: +- Documentation updates (unless API docs) +- Internal refactoring with no external impact +- Test additions +- Development tooling changes + +### Version Bumps + +- **Patch** (0.0.X): Bug fixes, minor improvements +- **Minor** (0.X.0): New features, backward compatible +- **Major** (X.0.0): Breaking changes + +## Questions? + +- Check [DEVELOPMENT.md](./DEVELOPMENT.md) for development workflows +- Open an issue for bugs or feature requests +- Join our Discord community for discussions +- Review existing PRs for examples + +Thank you for contributing to Dexto! 🚀 +*Tip:* Open an issue first for discussion on larger enhancements or proposals. diff --git a/dexto/DEVELOPMENT.md b/dexto/DEVELOPMENT.md new file mode 100644 index 00000000..29d3740f --- /dev/null +++ b/dexto/DEVELOPMENT.md @@ -0,0 +1,337 @@ +# Development Guide + +This guide covers development workflows for working on the Dexto codebase. + +## Table of Contents +- [Project Structure](#project-structure) +- [Development Setup](#development-setup) +- [Development Workflows](#development-workflows) +- [Build Commands](#build-commands) +- [Testing](#testing) +- [Code Quality](#code-quality) +- [Publishing](#publishing) + +## Project Structure + +Dexto is a monorepo using pnpm workspaces with the following structure: + +``` +dexto/ +├── packages/ +│ ├── core/ # @dexto/core - Core business logic +│ ├── cli/ # dexto - CLI application +│ └── webui/ # @dexto/webui - Next.js web interface +├── scripts/ # Build and development scripts +├── agents/ # Agent configurations +└── docs/ # Documentation +``` + +### Package Dependencies +- `dexto` (CLI) depends on `@dexto/core` +- `@dexto/webui` is embedded into CLI at build time +- All packages version together (fixed versioning) + +## Development Setup + +### Prerequisites +- Node.js >= 20.0.0 +- pnpm (automatically managed via Corepack) + +### Initial Setup +```bash +# Clone the repository +git clone https://github.com/truffle-ai/dexto.git +cd dexto + +# Enable Corepack (if not already enabled) - this picks pnpm version +corepack enable + +# Install dependencies (Corepack will use the correct pnpm version) +pnpm install + +# Build all packages +pnpm run build +``` + +## Development Workflows + +### 1. Hot Reload Development (Recommended) +Best for frontend development with automatic reload: + +```bash +pnpm run dev +``` + +This command: +- Builds core and CLI packages +- Runs API server on port 3001 (from built dist) +- Runs WebUI dev server on port 3000 (with hot reload) +- Prefixes output with `[API]` and `[UI]` for clarity +- **Automatically sets `DEXTO_DEV_MODE=true`** to use repository agent configs + +Access: +- API: http://localhost:3001 +- WebUI: http://localhost:3000 + +### 2. Symlink Development +Best for CLI development with instant changes: + +```bash +# Create global symlink (full build with WebUI) +pnpm run link-cli + +# Create global symlink (fast, no WebUI) +pnpm run link-cli-fast + +# Remove symlink +pnpm run unlink-cli +``` + +Now `dexto` command uses your local development code directly. + +### 3. Production-like Testing +Test the actual installation experience: + +```bash +# Install globally from local build (full) +pnpm run install-cli + +# Install globally from local build (fast, no WebUI) +pnpm run install-cli-fast +``` + +This creates tarballs and installs them globally, simulating `npm install -g dexto`. + +### Switching Between Workflows + +The `link-cli` and `install-cli` commands are mutually exclusive: +- Running `link-cli` removes any npm installation +- Running `install-cli` removes any pnpm symlink +- Use `unlink-cli` to remove everything + +## Build Commands + +### Complete Builds +```bash +# Full build with cleaning +pnpm run build + +# Build all packages without cleaning +pnpm run build:all + +# Build with type checking +pnpm run build:check +``` + +### Package-Specific Builds +```bash +# Build individual packages +pnpm run build:core +pnpm run build:cli +pnpm run build:webui + +# Build CLI and its dependencies only (no WebUI) +pnpm run build:cli-only +``` + +### WebUI Embedding +The WebUI is embedded into the CLI's dist folder during build: + +```bash +# Embed WebUI into CLI dist (run after building WebUI) +pnpm run embed-webui +``` + +## Testing + +### Automated Tests +```bash +# Run all tests +pnpm test + +# Run unit tests only +pnpm run test:unit + +# Run integration tests only +pnpm run test:integ + +# Run tests with coverage +pnpm run test:ci +``` + +### Manual testing + +1. Common commands +```bash +cd ~ +dexto --help +dexto "what is the current time" +dexto "list files in current directory" + +# Test other model override in CLI +dexto -m gpt-5-mini "what is the current date" + +# Test web mode +dexto + +# Test discord bot mode (requires additional setup) +dexto --mode discord + +# Test telegram bot mode (requires additional setup) +dexto --mode telegram +``` + +2. Execution contexts + +Dexto CLI operates differently based on the directory you are running in. +- source context -> when Dexto CLI is run in the source repository +- global context -> when Dexto CLI is run outside the source repository +- project context -> when Dexto CLI is run in a project that consumes @dexto dependencies + +Based on execution context, Dexto CLI will use defaults for log path, default agent/agent registry. +Run the CLI in different places and see the console logs to understand this. + +Test above commands in different execution contexts for manual testing coverage. + +**Developer Mode Environment Variable:** + +When running `dexto` from within this repository, it normally uses your `dexto setup` preferences and global `~/.dexto` directory. To force isolated testing with repository files: +```bash +export DEXTO_DEV_MODE=true # Use repo configs and local .dexto directory +``` + +**DEXTO_DEV_MODE Behavior:** +- **Agent Config**: Uses `agents/coding-agent/coding-agent.yml` from repo (instead of `~/.dexto/agents/`) +- **Logs/Database**: Uses `repo/.dexto/` (instead of `~/.dexto/`) +- **Preferences**: Skips global setup validation +- **Use Case**: Isolated testing and development on Dexto itself + +**Note**: `pnpm run dev` automatically sets `DEXTO_DEV_MODE=true`, so the development server always uses repository configs and local storage. + +## Code Quality + +### Type Checking +```bash +# Type check all packages +pnpm run typecheck + +# Type check with file watching +pnpm run typecheck:watch + +# Type check specific package +pnpm run typecheck:core +``` + +### Linting +```bash +# Run linter +pnpm run lint + +# Fix linting issues +pnpm run lint:fix +``` + +### Pre-commit Checks +Before committing, always run: +```bash +pnpm run build:check # Typecheck + build +pnpm test # Run tests +pnpm run lint # Check linting +``` + +## Publishing + +### Changeset Workflow + +We use [Changesets](https://github.com/changesets/changesets) for version management: + +1. **Create a changeset** for your changes: + ```bash + pnpm changeset + ``` + +2. **Select packages** affected by your change + +3. **Choose version bump** (patch/minor/major) + +4. **Write summary** of your changes + +### Version Strategy + +- **Fixed versioning**: All packages version together +- `@dexto/core` and `dexto` always have the same version +- `@dexto/webui` is private and not published + +### Publishing Process + +Publishing is automated via GitHub Actions: +1. Merge PR with changeset +2. Bot creates "Version Packages" PR +3. Merge version PR to trigger npm publish + +## Common Tasks + +### Clean Everything +```bash +# Clean all build artifacts and caches +pnpm run clean + +# Clean storage only +pnpm run clean:storage +``` + +### Start Production Server +```bash +# Start the CLI (requires build first) +pnpm start +``` + +### Working with Turbo + +Turbo commands run tasks across all packages: +```bash +pnpm run repo:build # Build all packages with Turbo +pnpm run repo:test # Test all packages with Turbo +pnpm run repo:lint # Lint all packages with Turbo +pnpm run repo:typecheck # Typecheck all packages with Turbo +``` + +## Troubleshooting + +### Native Dependencies +If you see errors about missing bindings (e.g., better-sqlite3): +```bash +# Reinstall dependencies +pnpm install + +# If that doesn't work, clean and reinstall +pnpm run clean +pnpm install +``` + +### Port Conflicts +Default ports: +- API/Server: 3001 +- WebUI Dev: 3000 + +Set environment variables to use different ports: +```bash +PORT=4000 API_PORT=4001 pnpm run dev +``` + +### Global Command Not Found +If `dexto` command is not found after linking: +```bash +# Check global installations +pnpm list -g +npm list -g dexto --depth=0 + +# Verify PATH includes pnpm/npm global bin +echo $PATH +``` + +## Questions? + +- Check [CONTRIBUTING.md](./CONTRIBUTING.md) for contribution guidelines +- Open an issue for bugs or feature requests +- Join our Discord for development discussions \ No newline at end of file diff --git a/dexto/Dockerfile b/dexto/Dockerfile new file mode 100644 index 00000000..e7987b72 --- /dev/null +++ b/dexto/Dockerfile @@ -0,0 +1,86 @@ +################################################################################ +# Build stage - includes dev dependencies +ARG NODE_VERSION=20.18.1 + +################################################################################ +# Build stage - pnpm workspace build and prune +FROM node:${NODE_VERSION}-alpine AS builder + +# Install build dependencies for native modules +RUN apk add --no-cache \ + python3 \ + make \ + g++ \ + && rm -rf /var/cache/apk/* + +WORKDIR /app + +# Install a pinned pnpm globally (avoid Corepack signature issues in containers) +ARG PNPM_VERSION=10.12.4 +RUN npm i -g pnpm@${PNPM_VERSION} + +# Copy workspace manifests for better layer caching +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY packages/cli/package.json packages/cli/package.json +COPY packages/core/package.json packages/core/package.json +COPY packages/webui/package.json packages/webui/package.json + +# Install workspace dependencies (with lockfile) +RUN pnpm install --frozen-lockfile + +# Copy sources and build all packages (embeds WebUI into CLI dist) +COPY . . +RUN pnpm -w build + +# Prune to production dependencies +# Prune to production dependencies at the workspace root +# (keeps per-package node_modules; smaller image without dev deps) +RUN pnpm prune --prod + +################################################################################ +# Production stage - minimal Alpine with Chromium +FROM node:${NODE_VERSION}-alpine AS production + +# Install Chromium runtime +RUN apk add --no-cache \ + chromium \ + && rm -rf /var/cache/apk/* /tmp/* + +WORKDIR /app + +# Create non-root user and data dir +RUN addgroup -g 1001 -S dexto && adduser -S dexto -u 1001 \ + && mkdir -p /app/.dexto/database && chown -R dexto:dexto /app/.dexto + +# Copy only what we need for runtime +COPY --from=builder --chown=dexto:dexto /app/node_modules ./node_modules +COPY --from=builder --chown=dexto:dexto /app/packages/cli/node_modules ./packages/cli/node_modules +COPY --from=builder --chown=dexto:dexto /app/packages/cli/dist ./packages/cli/dist +COPY --from=builder --chown=dexto:dexto /app/packages/cli/package.json ./packages/cli/package.json +# Copy core workspace package since pnpm links it via node_modules +COPY --from=builder --chown=dexto:dexto /app/packages/core/package.json ./packages/core/package.json +COPY --from=builder --chown=dexto:dexto /app/packages/core/dist ./packages/core/dist +COPY --from=builder --chown=dexto:dexto /app/packages/core/node_modules ./packages/core/node_modules +COPY --from=builder --chown=dexto:dexto /app/agents ./agents +COPY --from=builder --chown=dexto:dexto /app/package.json ./ + +# Environment +ENV NODE_ENV=production \ + PORT=3001 \ + API_PORT=3001 \ + CONFIG_FILE=/app/agents/coding-agent/coding-agent.yml \ + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + +# Run as non-root +USER dexto + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD node -e "const http=require('http');const port=process.env.API_PORT||process.env.PORT||3001;const req=http.request({host:'localhost',port,path:'/health'},res=>process.exit(res.statusCode===200?0:1));req.on('error',()=>process.exit(1));req.end();" + +# Fixed port for metadata (runtime can override via -e API_PORT) +EXPOSE 3001 + +# Server mode: REST APIs + SSE streaming on single port (no Web UI) +CMD ["sh", "-c", "node packages/cli/dist/index.js --mode server --agent $CONFIG_FILE"] diff --git a/dexto/GEMINI.md b/dexto/GEMINI.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/dexto/GEMINI.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/dexto/LICENSE b/dexto/LICENSE new file mode 100644 index 00000000..718f1d9d --- /dev/null +++ b/dexto/LICENSE @@ -0,0 +1,44 @@ +Elastic License 2.0 (ELv2) + +**Acceptance** +By using the software, you agree to all of the terms and conditions below. + +**Copyright License** +The licensor grants you a non-exclusive, royalty-free, worldwide, non-sublicensable, non-transferable license to use, copy, distribute, make available, and prepare derivative works of the software, in each case subject to the limitations and conditions below + +**Limitations** +You may not provide the software to third parties as a hosted or managed service, where the service provides users with access to any substantial set of the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality in the software, and you may not remove or obscure any functionality in the software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices of the licensor in the software. Any use of the licensor’s trademarks is subject to applicable law. + +**Patents** +The licensor grants you a license, under any patent claims the licensor can license, or becomes able to license, to make, have made, use, sell, offer for sale, import and have imported the software, in each case subject to the limitations and conditions in this license. This license does not cover any patent claims that you cause to be infringed by modifications or additions to the software. If you or your company make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company. + +**Notices** +You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the software prominent notices stating that you have modified the software. + +**No Other Rights** +These terms do not imply any licenses other than those expressly granted in these terms. + +**Termination** +If you use the software in violation of these terms, such use is not licensed, and your licenses will automatically terminate. If the licensor provides you with a notice of your violation, and you cease all violation of this license no later than 30 days after you receive that notice, your licenses will be reinstated retroactively. However, if you violate these terms after such reinstatement, any additional violation of these terms will cause your licenses to terminate automatically and permanently. + +**No Liability** +As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim. + +**Definitions** +The _licensor_ is the entity offering these terms, and the _software_ is the software the licensor makes available under these terms, including any portion of it. + +_you_ refers to the individual or entity agreeing to these terms. + +_your company_ is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. _control_ means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect. + +_your licenses_ are all the licenses granted to you for the software under these terms. + +_use_ means anything you do with the software requiring one of your licenses. + +_trademark_ means trademarks, service marks, and similar rights. \ No newline at end of file diff --git a/dexto/README.Docker.md b/dexto/README.Docker.md new file mode 100644 index 00000000..e3f2458b --- /dev/null +++ b/dexto/README.Docker.md @@ -0,0 +1,104 @@ +# Running Dexto with Docker + +This image runs the Dexto CLI in server mode (API + SSE streaming). It uses pnpm workspaces and builds from the current repo (no published packages required). + +## Build the image + +```bash +docker build -t dexto:local . +``` + +## Provide configuration and API keys + +Create a `.env` file with your keys (see `README.md`): + +```ini +OPENAI_API_KEY=... +# add other provider keys as needed +``` + +The coding agent config is baked into the image at `/app/agents/coding-agent/coding-agent.yml`. You can mount your own agents folder if desired. + +## Run: API server only (default) + +```bash +docker run --rm \ + --env-file .env \ + -e API_PORT=3001 \ + -p 3001:3001 \ + dexto:local +``` + +What it does: +- Starts REST + SSE streaming server on `API_PORT` (default 3001) +- Uses Chromium inside the image for Puppeteer tools +- Stores runtime data under `/app/.dexto` (in‑container) + +Endpoints: +- API base: `http://localhost:3001/api/` +- Health: `http://localhost:3001/health` +- MCP servers: `http://localhost:3001/api/mcp/servers` + +Persist data between runs (recommended): + +```bash +docker run --rm \ + --env-file .env \ + -e API_PORT=3001 \ + -p 3001:3001 \ + -v dexto_data:/app/.dexto \ + dexto:local +``` + +Use a custom agent config: + +```bash +docker run --rm \ + --env-file .env \ + -e API_PORT=3001 \ + -e CONFIG_FILE=/app/agents/my-agent.yml \ + -v $(pwd)/agents:/app/agents:ro \ + -p 3001:3001 \ + dexto:local +``` + +## Run with WebUI (optional) + +The image embeds the built WebUI. To run the WebUI alongside the API, start the CLI in `web` mode. This requires two ports (frontend and API): + +```bash +docker run --rm \ + --env-file .env \ + -e FRONTEND_PORT=3000 \ + -e API_PORT=3001 \ + -p 3000:3000 -p 3001:3001 \ + dexto:local \ + sh -c "node packages/cli/dist/index.js --mode web --agent $CONFIG_FILE" +``` + +Open the WebUI: `http://localhost:3000` (the UI calls the API on `http://localhost:3001`). + +## Docker Compose (example) + +```yaml +services: + dexto: + image: dexto:local + build: . + environment: + API_PORT: 3001 + ports: + - "3001:3001" + volumes: + - dexto_data:/app/.dexto + - ./agents:/app/agents:ro + env_file: .env + +volumes: + dexto_data: {} +``` + +## Notes +- Healthcheck uses `API_PORT` (falls back to `PORT` or 3001). +- The container runs as a non‑root user (`dexto`). +- The image builds from your repo code; no published `@dexto/core` is required. diff --git a/dexto/README.md b/dexto/README.md new file mode 100644 index 00000000..f3eba247 --- /dev/null +++ b/dexto/README.md @@ -0,0 +1,672 @@ + +
+ + + + Dexto + +
+ + +

+ + + + +

+ + +

+Deutsch | +English | +Español | +français | +日本語 | +한국어 | +Português | +Русский | +中文 +

+ +

An open agent harness for AI applications—ships with a powerful coding agent.

+ +
+ Dexto Demo +
+ +--- + +## What is Dexto? + +Dexto is an **agent harness**—the orchestration layer that turns LLMs into reliable, stateful agents that can take actions, remember context, and recover from errors. + +Think of it like an operating system for AI agents: + +| Component | Analogy | Role | +|-----------|---------|------| +| **LLM** | CPU | Raw processing power | +| **Context Window** | RAM | Working memory | +| **Dexto** | Operating System | Orchestration, state, tools, recovery | +| **Your Agent** | Application | Domain-specific logic and clients | + +### Why Dexto? + +- **Configuration-driven**: Define agents in YAML. Swap models and tools without touching code. +- **Batteries included**: Session management, tool orchestration, memory, multimodal support, and observability—out of the box. +- **Run anywhere**: Local, cloud, or hybrid. CLI, Web UI, REST API, Discord, Telegram, or embedded in your app. + +### What You Can Build + +- **Coding Agents** – Build, debug, and refactor code autonomously +- **Autonomous Agents** – Plan, execute, and adapt to user goals +- **Digital Companions** – Assistants that remember context and anticipate needs +- **MCP Clients & Servers** – Connect tools, files, APIs via Model Context Protocol +- **Multi-Agent Systems** – Agents that collaborate, delegate, and solve complex tasks together + +--- + +## Coding Agent + +Dexto ships with a **production-ready coding agent** you can use immediately via the CLI or Web UI. + +```bash +# Launch the coding agent (default) +dexto + +# Or explicitly +dexto --agent coding-agent +``` + +**What it can do:** +- Build new apps from scratch +- Read, write, and refactor code across your entire codebase +- Execute shell commands and run tests +- Spawn specialized sub-agents for exploration and planning +- Remember context across sessions with persistent memory +- Work with any of 50+ LLMs (swap models mid-conversation) + +**Ready-to-use interfaces:** +- **Web UI** – Chat interface with file uploads, syntax highlighting, and MCP tool browser +- **CLI** – Terminal-native with `/commands`, streaming output, and session management + +The coding agent is just one example of what you can build. Create your own agents by defining a YAML config—same architecture, your domain. + +--- + +## Quick Start + +### Install + +```bash +# Install globally via npm +npm install -g dexto + +# Or build from source +git clone https://github.com/truffle-ai/dexto.git +cd dexto && pnpm install && pnpm install-cli +``` + +### Run + +```bash +# Start Dexto (launches setup wizard on first run) +dexto +``` + +**More options:** + +```bash +dexto --mode cli # Terminal mode +dexto -p "create a landing page for a coffee shop" # One-shot task +dexto --auto-approve "refactor this codebase" # Skip confirmations +dexto -m claude-sonnet-4-5-20250929 # Switch models +dexto --help # Explore all options +``` + +**Inside the interactive CLI**, type `/` to explore commands—switch models, manage sessions, configure tools, and more. + +### Manage Settings + +```bash +# Configure defaults like LLM provider/model, API keys, or download local models +dexto setup +``` + +Logs are stored in `~/.dexto/logs/`. Use `DEXTO_LOG_LEVEL=debug` for verbose output. + +--- + +## Core Features + +### 50+ LLMs, Instant Switching + +Switch models mid-conversation—no code changes, no restarts. + +| Provider | Models | +|----------|--------| +| **OpenAI** | gpt-5.2, gpt-5.2-pro, gpt-5.2-codex, o4-mini | +| **Anthropic** | Claude Sonnet, Opus, Haiku (with extended thinking) | +| **Google** | Gemini 3 Pro, 2.5 Pro/Flash | +| **Groq** | Llama 4, Qwen, DeepSeek | +| **xAI** | Grok 4, Grok 3 | +| **Local** | Ollama, GGUF via node-llama-cpp (Llama, Qwen, Mistral, etc.) | +| **+ Gateways** | OpenRouter, AWS Bedrock, Vertex AI, LiteLLM | + +**Run locally for privacy**: Local models keep data on your machine with automatic GPU detection (Metal, CUDA, Vulkan). + +### MCP Integration (30+ Tools) + +Connect to Model Context Protocol servers—Puppeteer, Linear, ElevenLabs, Firecrawl, Sora, and more. + +```yaml +# agents/my-agent.yml +mcpServers: + filesystem: + type: stdio + command: npx + args: ['-y', '@modelcontextprotocol/server-filesystem', '.'] + browser: + type: stdio + command: npx + args: ['-y', '@anthropics/mcp-server-puppeteer'] +``` + +Browse and add servers from the MCP Store in the Web UI or via `/mcp` commands in the CLI. + +### Human-in-the-Loop Controls + +Fine-grained control over what your agent can do: + +```yaml +toolConfirmation: + mode: manual # Require approval for each tool + # mode: auto-approve # Trust mode for local development + toolPolicies: + alwaysAllow: + - mcp--filesystem--read_file + - mcp--filesystem--list_directory + alwaysDeny: + - mcp--filesystem--delete_file +``` + +Agents remember which tools you've approved per session. + +### Persistent Sessions & Memory + +Conversations persist across restarts. Create memories that shape agent behavior. + +```bash +# Continue last conversation +dexto -c + +# Resume specific session +dexto -r session-abc123 + +# Search across all conversations +dexto search "database schema" +``` + +### Multi-Agent Systems + +Agents can spawn specialized sub-agents to handle complex subtasks. The coding agent uses this to delegate exploration: + +```yaml +# In your agent config +customTools: + - type: agent-spawner + allowedAgents: ["explore-agent"] + maxConcurrentAgents: 5 + defaultTimeout: 300000 # 5 minutes +``` + +**Built-in sub-agents:** +- **explore-agent** – Fast, read-only codebase exploration + +Any agent in the [Agent Registry](#agent-registry) can be spawned as a sub-agent—including custom agents you create and register. + +Sub-agents run ephemerally, auto-cleanup after completion, and forward tool approvals to the parent—so users see one unified approval flow. + +--- + +## Run Modes + +| Mode | Command | Use Case | +|------|---------|----------| +| **Web UI** | `dexto` | Chat interface with file uploads (default) | +| **CLI** | `dexto --mode cli` | Terminal interaction | +| **Web Server** | `dexto --mode server` | REST & SSE APIs | +| **MCP Server** | `dexto --mode mcp` | Expose agent as an MCP server over stdio | + +Platform integrations: [Discord](examples/discord-bot/), [Telegram](examples/telegram-bot/) + +
+Dexto as an MCP Server + +**Transport: `stdio`** + +Connect your Dexto agent as an MCP server in Claude Code or Cursor: + +```bash +dexto --mode mcp --auto-approve --no-elicitation # Default agent +dexto --mode mcp --agent coding-agent --auto-approve --no-elicitation +npx dexto --mode mcp --agent coding-agent --auto-approve --no-elicitation +``` + +Example MCP config for Claude Code or Cursor: + +```json +{ + "command": "npx", + "args": ["-y", "dexto", "--mode", "mcp", "--agent", "coding-agent", "--auto-approve", "--no-elicitation"] +} +``` + +`--auto-approve` and `--no-elicitation` are required for MCP mode since it runs non-interactively. + +**Transport: `http/sse`** + +`dexto --mode server` exposes your agent as an MCP server at `/mcp` for remote clients: + +```bash +dexto --mode server --port 3001 +ngrok http 3001 # Optional: expose publicly +``` + +
+ +--- + +## Dexto Agents SDK + +Build AI agents programmatically. Everything the CLI does, your code can too. + +```bash +npm install @dexto/core +``` + +```typescript +import { DextoAgent } from '@dexto/core'; + +const agent = new DextoAgent({ + llm: { provider: 'openai', model: 'gpt-5.2', apiKey: process.env.OPENAI_API_KEY } +}); +await agent.start(); + +const session = await agent.createSession(); +const response = await agent.generate('What is TypeScript?', session.id); +console.log(response.content); + +// Streaming +for await (const event of await agent.stream('Write a story', session.id)) { + if (event.name === 'llm:chunk') process.stdout.write(event.content); +} + +// Multimodal +await agent.generate([ + { type: 'text', text: 'Describe this image' }, + { type: 'image', image: base64Data, mimeType: 'image/png' } +], session.id); + +// Switch models mid-conversation +await agent.switchLLM({ model: 'claude-sonnet-4-5-20250929' }); +``` + +**Start a server programmatically:** + +Start a Dexto server programmatically to expose REST and SSE streaming APIs to interact and manage your agent backend. + +```typescript +import { DextoAgent } from '@dexto/core'; +import { loadAgentConfig, startHonoApiServer } from 'dexto'; + +const config = await loadAgentConfig('./agents/my-agent.yml'); +const agent = new DextoAgent(config); +const { server } = await startHonoApiServer(agent, 3001); +// POST /api/message, GET /api/sessions, etc. +``` + +This starts an HTTP server with full REST and SSE APIs, enabling integration with web frontends, webhooks, and other services. See the [REST API Documentation](https://docs.dexto.ai/api/rest/) for available endpoints. + +
+Advanced SDK Usage + +### Session Management + +Create and manage multiple conversation sessions with persistent storage. + +```typescript +const agent = new DextoAgent(config); +await agent.start(); + +// Create and manage sessions +const session = await agent.createSession('user-123'); +await agent.generate('Hello, how can you help me?', session.id); + +// List and manage sessions +const sessions = await agent.listSessions(); +const history = await agent.getSessionHistory('user-123'); +await agent.deleteSession('user-123'); + +// Search across conversations +const results = await agent.searchMessages('bug fix', { limit: 10 }); +``` + +### LLM Management + +Switch between models and providers dynamically. + +```typescript +// Get current configuration +const currentLLM = agent.getCurrentLLMConfig(); + +// Switch models (provider inferred automatically) +await agent.switchLLM({ model: 'gpt-5.2' }); +await agent.switchLLM({ model: 'claude-sonnet-4-5-20250929' }); + +// Switch model for a specific session +await agent.switchLLM({ model: 'gpt-5.2' }, 'session-123'); + +// Get supported providers and models +const providers = agent.getSupportedProviders(); +const models = agent.getSupportedModels(); +``` + +### MCP Manager + +For advanced MCP server management, use the MCPManager directly. + +```typescript +import { MCPManager } from '@dexto/core'; + +const manager = new MCPManager(); + +// Connect to MCP servers +await manager.connectServer('filesystem', { + type: 'stdio', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-filesystem', '.'] +}); + +// Access tools, prompts, and resources +const tools = await manager.getAllTools(); +const prompts = await manager.getAllPrompts(); +const resources = await manager.getAllResources(); + +// Execute tools +const result = await manager.executeTool('readFile', { path: './README.md' }); + +await manager.disconnectAll(); +``` + +### Storage & Persistence + +Configure storage backends for production. + +```yaml +# agents/production-agent.yml +storage: + cache: + type: redis + url: $REDIS_URL + database: + type: postgres + connectionString: $POSTGRES_CONNECTION_STRING + +sessions: + maxSessions: 1000 + sessionTTL: 86400000 # 24 hours +``` + +**Supported Backends:** +- **Cache**: Redis, In-Memory +- **Database**: PostgreSQL, SQLite, In-Memory + +See the [Storage Configuration guide](https://docs.dexto.ai/docs/guides/configuring-dexto/storage) for details. + +
+ +--- + +## Agent Registry + +Pre-built agents for common use cases: + +```bash +# List available agents +dexto list-agents + +# Install and run +dexto install coding-agent podcast-agent +dexto --agent coding-agent +``` + +**Available:** Coding, Podcast, Image Editor, Video (Sora), Database, GitHub, Triage (multi-agent), and more. + +See the full [Agent Registry](https://docs.dexto.ai/docs/guides/agent-registry). + +--- + +## Agent Configuration + +Dexto treats each configuration as a unique agent allowing you to define and save combinations of LLMs, servers, storage options, etc. based on your needs for easy portability. Define agents in version-controlled YAML. Change the file, reload, and chat—state, memory, and tools update automatically. + +```yaml +# agents/production-agent.yml +llm: + provider: anthropic + model: claude-sonnet-4-5-20250929 + apiKey: $ANTHROPIC_API_KEY + +mcpServers: + filesystem: + type: stdio + command: npx + args: ['-y', '@modelcontextprotocol/server-filesystem', '.'] + +systemPrompt: | + You are a helpful assistant with filesystem access. + +storage: + cache: + type: redis + url: $REDIS_URL + database: + type: postgres + connectionString: $POSTGRES_CONNECTION_STRING + +toolConfirmation: + mode: manual +``` + +### LLM Providers + +Switch between providers instantly—no code changes required. + +#### Built-in Providers + +| Provider | Models | Setup | +|----------|--------|-------| +| **OpenAI** | `gpt-5.2`, `gpt-5.2-pro`, `gpt-5.2-codex`, `o4-mini` | API key | +| **Anthropic** | `claude-sonnet-4-5-20250929`, `claude-opus-4-5-20250929`, extended thinking | API key | +| **Google** | `gemini-3-pro`, `gemini-2.5-pro`, `gemini-2.5-flash` | API key | +| **Groq** | `llama-4-scout`, `qwen-qwq`, `deepseek-r1-distill` | API key | +| **xAI** | `grok-4`, `grok-3`, `grok-3-fast` | API key | +| **Cohere** | `command-r-plus`, `command-r` | API key | + +#### Local Models (Privacy-First) + +| Provider | Models | Setup | +|----------|--------|-------| +| **Ollama** | Llama, Qwen, Mistral, DeepSeek, etc. | Local install | +| **node-llama-cpp** | Any GGUF model | Bundled (auto GPU detection: Metal, CUDA, Vulkan) | + +#### Cloud Platforms + +| Provider | Models | Setup | +|----------|--------|-------| +| **AWS Bedrock** | Claude, Llama, Mistral | AWS credentials | +| **Google Vertex AI** | Gemini, Claude | GCP credentials | + +#### Gateway Providers + +| Provider | Access | Setup | +|----------|--------|-------| +| **OpenRouter** | 100+ models from multiple providers | API key | +| **LiteLLM** | Unified API for any provider | Self-hosted or API key | +| **Glama** | Multi-provider gateway | API key | + +```bash +# Switch models via CLI +dexto -m claude-sonnet-4-5-20250929 +dexto -m gemini-2.5-pro +dexto -m llama-4-scout +``` + +Switch within interactive CLI (`/model`) or Web UI without config changes. + +See the [Configuration Guide](https://docs.dexto.ai/docs/category/agent-configuration-guide). + +--- + +## Demos & Examples + +| Image Editor | MCP Store | Portable Agents | +|:---:|:---:|:---:| +| Image Editor | MCP Store | Portable Agents | +| Face detection & annotation using OpenCV | Browse and add MCPs | Use agents in Cursor, Claude Code via MCP| + +
+More Examples + +### Coding Agent +Build applications from natural language: +```bash +dexto --agent coding-agent +# "Create a snake game and open it in the browser" +``` +Coding Agent Demo + +### Podcast Agent +Generate multi-speaker audio content: +```bash +dexto --agent podcast-agent +``` +Podcast Agent Demo + +### Multi-Agent Triage +Coordinate specialized agents: +```bash +dexto --agent triage-agent +``` +Triage Agent Demo + +### Memory System +Persistent context that shapes behavior: +Memory Demo + +### Dynamic Forms +Agents generate forms for structured input: +User Form Demo + +### Browser Automation + + Amazon Shopping Demo + + +### MCP Playground +Test tools before deploying: +Playground Demo + +
+ +--- + +
+CLI Reference + +```text +Usage: dexto [options] [command] [prompt...] + +Basic Usage: + dexto Start web UI (default) + dexto "query" Run one-shot query + dexto --mode cli Interactive CLI + +Session Management: + dexto -c Continue last conversation + dexto -r Resume specific session + +Options: + -a, --agent Agent config file or ID + -m, --model LLM model to use + --auto-approve Skip tool confirmations + --no-elicitation Disable elicitation prompts + --mode web | cli | server | mcp + --port Server port + +Commands: + setup Configure global preferences + install Install agents from registry + list-agents List available agents + session list|history Manage sessions + search Search conversation history +``` + +Full reference: `dexto --help` + +
+ +--- + +## Documentation + +- **[Quick Start](https://docs.dexto.ai/docs/getting-started/intro/)** – Get running in minutes +- **[Configuration Guide](https://docs.dexto.ai/docs/category/guides/)** – Agents, LLMs, tools +- **[SDK Reference](https://docs.dexto.ai/api/sdk/dexto-agent)** – Programmatic usage +- **[REST API](https://docs.dexto.ai/api/rest/)** – HTTP endpoints + +--- + +## Telemetry + +We collect anonymous usage data (no personal/sensitive info) to help improve Dexto. This includes: + +- Commands used +- Command execution time +- Error occurrences +- System information (OS, Node version) +- LLM Models used + +To opt-out: + +Set env variable `DEXTO_ANALYTICS_DISABLED=1` + +--- + +## Contributing + +See [CONTRIBUTING.md](./CONTRIBUTING.md). + +--- + +## Community + +Built by [Truffle AI](https://www.trytruffle.ai). Join [Discord](https://discord.gg/GFzWFAAZcm) for support. + +If you find Dexto useful, please give us a ⭐ on GitHub—it helps a lot! + +[![Twitter Follow](https://img.shields.io/twitter/follow/Dexto?style=social)](https://x.com/intent/user?screen_name=dexto_ai) +[![Twitter Follow](https://img.shields.io/twitter/follow/Rahul?style=social)](https://x.com/intent/user?screen_name=Road_Kill11) +[![Twitter Follow](https://img.shields.io/twitter/follow/Shaunak?style=social)](https://x.com/intent/user?screen_name=shaun5k_) + +--- + +## Contributors + +[![Contributors](https://contrib.rocks/image?repo=truffle-ai/dexto)](https://github.com/truffle-ai/dexto/graphs/contributors) + +--- + +## License + +Elastic License 2.0. See [LICENSE](LICENSE). diff --git a/dexto/VERSIONING.md b/dexto/VERSIONING.md new file mode 100644 index 00000000..81800957 --- /dev/null +++ b/dexto/VERSIONING.md @@ -0,0 +1,219 @@ +# Versioning and Release Process + +This document describes the versioning strategy and release process for the Dexto monorepo. + +## Overview + +Dexto uses a monorepo structure with multiple packages: +- `dexto` (CLI) - Main package published to npm +- `@dexto/core` - Core library published to npm +- `@dexto/webui` - Web UI (private, not published) + +We use [Changesets](https://github.com/changesets/changesets) for version management and coordinated releases. + +## Versioning Strategy + +### Fixed Versioning +The `dexto` and `@dexto/core` packages use **fixed versioning** - they always maintain the same version number. This ensures API compatibility between the CLI and core library. + + +## Automated Release Process (Recommended) + +### 1. Create a Changeset + +When you make changes that should trigger a release: + +```bash +# Create a changeset describing your changes +pnpm changeset + +# Follow the interactive prompts to: +# 1. Select which packages changed +# 2. Choose the version bump type (major/minor/patch) +# 3. Write a summary of changes +``` + +This creates a markdown file in `.changeset/` describing the changes. + +> **Why Manual Changesets?** We require manual changeset creation to ensure developers think carefully about semantic versioning and write meaningful changelog entries for users. + +### 2. Commit and Push to PR + +```bash +# Add the changeset file +git add .changeset/*.md +git commit -m "chore: add changeset for [your feature]" +git push origin your-branch +``` + +### 3. Automatic Version and Release + +When your PR with changesets is merged to `main`: + +1. **Version PR Creation** (`changesets-publish.yml` triggers automatically) + - Collects all pending changesets + - Creates a "Version Packages" PR with: + - Version bumps in package.json files + - Updated CHANGELOG.md files + - Consolidated changesets + +2. **Review the Version PR** + - Team reviews the version bumps + - Can be merged immediately or held for batching multiple changes + +3. **Automatic Publishing** (when Version PR is merged) + - `changesets-publish.yml` triggers + - Builds all packages + - Publishes to npm registry + - Creates git tags + - Removes processed changeset files + +### GitHub Workflows + +#### Active Release Workflows: +- **[`require-changeset.yml`](.github/workflows/require-changeset.yml)** - Ensures PRs include changesets when needed +- **[`changesets-publish.yml`](.github/workflows/changesets-publish.yml)** - Opens a version bump PR, and publishes it when we merge the version bump PR (triggers on push to main) + +#### Quality Check Workflows: +- **[`build_and_test.yml`](.github/workflows/build_and_test.yml)** - Runs tests on PRs +- **[`code-quality.yml`](.github/workflows/code-quality.yml)** - Runs linting and type checking + +#### Documentation Workflows: +- **[`build-docs.yml`](.github/workflows/build-docs.yml)** - Builds documentation +- **[`deploy-docs.yml`](.github/workflows/deploy-docs.yml)** - Deploys documentation site + +## Manual Release Process (Emergency Only) + +If automated release fails or for emergency patches: + +### Prerequisites + +```bash +# Ensure you're on main and up to date +git checkout main +git pull origin main + +# Install dependencies +pnpm install --frozen-lockfile + +# Run all quality checks +pnpm run build +pnpm run lint +pnpm run typecheck +pnpm test +``` + +### Option 1: Manual Changeset Release + +```bash +# 1. Create changeset manually +pnpm changeset + +# 2. Version packages +pnpm changeset version + +# 3. Commit version changes +git add -A +git commit -m "chore: version packages" + +# 4. Build all packages +pnpm run build + +# 5. Publish to npm +pnpm changeset publish + +# 6. Push changes and tags +git push --follow-tags +``` + +### Option 2: Direct Version Bump (Not Recommended) + +```bash +# 1. Update versions manually in package.json files +# IMPORTANT: Keep dexto and @dexto/core versions in sync! + +# Edit packages/cli/package.json +# Edit packages/core/package.json + +# 2. Install to update lockfile +pnpm install + +# 3. Build packages +pnpm run build + +# 4. Create git tag +git add -A +git commit -m "chore: release v1.2.0" +git tag v1.2.0 + +# 5. Publish packages +cd packages/core && pnpm publish --access public +cd ../cli && pnpm publish --access public + +# 6. Push commits and tags +git push origin main --follow-tags +``` + +## Testing Releases (Without Publishing) + +### Dry Run Commands + +```bash +# See what would be published +pnpm publish -r --dry-run --no-git-checks + +# Check changeset status +pnpm changeset status + +# Preview version changes +pnpm changeset version --dry-run + +# Test package contents +cd packages/cli && npm pack --dry-run +cd packages/core && npm pack --dry-run +``` + +### Local Testing + +```bash +# Link packages locally for testing +pnpm run link-cli + +# Test the linked CLI +dexto --version +``` + +## Release Checklist + +Before any release: +- [ ] All tests passing (`pnpm test`) +- [ ] No lint errors (`pnpm run lint`) +- [ ] TypeScript compiles (`pnpm run typecheck`) +- [ ] Build succeeds (`pnpm run build`) +- [ ] Changeset created (if using automated flow) +- [ ] Version numbers synchronized (dexto and @dexto/core) + +## Common Issues + +### Issue: Versions out of sync +**Solution**: Ensure `dexto` and `@dexto/core` have the same version in their package.json files. + +### Issue: Publish fails with "Package not found" +**Solution**: Run `pnpm run build` before publishing to ensure dist folders exist. + +### Issue: Git working directory not clean +**Solution**: Commit or stash all changes before publishing. Use `--no-git-checks` flag for testing only. + +### Issue: Authentication error when publishing +**Solution**: CI uses `NPM_TOKEN` secret (granular access token). Ensure the token is valid and has publish permissions for `@dexto` scope. For local publishing, use `npm login`. + +## Version History + +See package CHANGELOGs for detailed version history: + +- packages/cli/CHANGELOG.md +- packages/core/CHANGELOG.md + +## Questions? + +For questions about the release process, please open an issue or consult the team. diff --git a/dexto/agents/README.md b/dexto/agents/README.md new file mode 100644 index 00000000..6daa91ac --- /dev/null +++ b/dexto/agents/README.md @@ -0,0 +1,306 @@ +# Configuration Guide + +Dexto uses a YAML configuration file to define tool servers and AI settings. This guide provides detailed information on all available configuration options. + +## Configuration File Location + +By default, Dexto looks for a configuration file at `agents/coding-agent/coding-agent.yml` in the project directory. You can specify a different location using the `--agent` command-line option: + +```bash +npm start -- --agent path/to/your/agent.yml +``` + +## Configuration Structure + +The configuration file has two main sections: + +1. `mcpServers`: Defines the tool servers to connect to +2. `llm`: Configures the AI provider settings + +### Basic Example + +```yaml +mcpServers: + github: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-github" + env: + GITHUB_PERSONAL_ACCESS_TOKEN: your-github-token + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . +llm: + provider: openai + model: gpt-5 + apiKey: $OPENAI_API_KEY +``` + +## Tool Server Configuration + +Each entry under `mcpServers` defines a tool server to connect to. The key (e.g., "github", "filesystem") is used as a friendly name for the server. + +Tool servers can either be local servers (stdio) or remote servers (sse) + +### Stdio Server Options + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `type` | string | Yes | The type of the server, needs to be 'stdio' | +| `command` | string | Yes | The executable to run | +| `args` | string[] | No | Array of command-line arguments | +| `env` | object | No | Environment variables for the server process | + +### SSE Server Options + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `type` | string | Yes | The type of the server, needs to be 'sse' | +| `url` | string | Yes | The url of the server | +| `headers` | map | No | Optional headers for the url | + +## LLM Configuration + +The `llm` section configures the AI provider settings. + +### LLM Options + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `provider` | string | Yes | AI provider (e.g., "openai", "anthropic", "google") | +| `model` | string | Yes | The model to use | +| `apiKey` | string | Yes | API key or environment variable reference | +| `temperature` | number | No | Controls randomness (0-1, default varies by provider) | +| `maxInputTokens` | number | No | Maximum input tokens for context compression | +| `maxOutputTokens` | number | No | Maximum output tokens for response length | +| `baseURL` | string | No | Custom API endpoint for OpenAI-compatible providers | + +### API Key Configuration + +#### Setting API Keys + +API keys can be configured in two ways: + +1. **Environment Variables (Recommended)**: + - Add keys to your `.env` file (use `.env.example` as a template) or export environment variables + - Reference them in config with the `$` prefix + +2. **Direct Configuration** (Not recommended for security): + - Directly in the YAML file (less secure, avoid in production) + +```yaml +# Recommended: Reference environment variables +apiKey: $OPENAI_API_KEY + +# Not recommended: Direct API key in config +apiKey: sk-actual-api-key +``` + +#### Security Best Practices +- Never commit API keys to version control +- Use environment variables in production environments +- Create a `.gitignore` entry for your `.env` file + +#### API Keys for Different Providers +Each provider requires its own API key: +- OpenAI: Set `OPENAI_API_KEY` in `.env` +- Anthropic: Set `ANTHROPIC_API_KEY` in `.env` +- Google Gemini: Set `GOOGLE_GENERATIVE_AI_API_KEY` in `.env` + +#### Openai example +```yaml +llm: + provider: openai + model: gpt-5 + apiKey: $OPENAI_API_KEY +``` + +#### Anthropic example +```yaml +llm: + provider: anthropic + model: claude-sonnet-4-5-20250929 + apiKey: $ANTHROPIC_API_KEY +``` + +#### Google example +```yaml +llm: + provider: google + model: gemini-2.0-flash + apiKey: $GOOGLE_GENERATIVE_AI_API_KEY +``` + +## Optional Greeting + +Add a simple `greeting` at the root of your config to provide a default welcome text that UI layers can display when a chat starts: + +```yaml +greeting: "Hi! I’m Dexto — how can I help today?" +``` + +### Windows Support + +On Windows systems, some commands like `npx` may have different paths. The system attempts to automatically detect and uses the correct paths for these commands on Windows. If you run into any issues during server initialization, you may need to adjust the path to your `npx` command. + +## Supported Tool Servers + +Here are some commonly used MCP-compatible tool servers: + +### GitHub + +```yaml +github: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-github" + env: + GITHUB_PERSONAL_ACCESS_TOKEN: your-github-token +``` + +### Filesystem + +```yaml +filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . +``` + +### Terminal + +```yaml +terminal: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-terminal" +``` + +### Desktop Commander + +```yaml +desktop: + type: stdio + command: npx + args: + - -y + - "@wonderwhy-er/desktop-commander" +``` + +### Custom Server + +```yaml +custom: + type: stdio + command: node + args: + - --loader + - ts-node/esm + - src/servers/customServer.ts + env: + API_KEY: your-api-key +``` + +### Remote Server + +This example uses a remote github server provided by composio. +The URL is just a placeholder which won't work out of the box since the URL is customized per user. +Go to mcp.composio.dev to get your own MCP server URL. + +```yaml +github-remote: + type: sse + url: https://mcp.composio.dev/github/repulsive-itchy-alarm-ABCDE +``` + +## Command-Line Options + +Dexto supports several command-line options: + +| Option | Description | +|--------|-------------| +| `--agent` | Specify a custom agent configuration file | +| `--strict` | Require all connections to succeed | +| `--verbose` | Enable verbose logging | +| `--help` | Show help | + +## Available Agent Examples + +### Database Agent +An AI agent that provides natural language access to database operations and analytics. This approach simplifies database interaction - instead of building forms, queries, and reporting dashboards, users can simply ask for what they need in plain language. + +**Quick Start:** +```bash +cd database-agent +./setup-database.sh +npm start -- --agent database-agent.yml +``` + +**Example Interactions:** +- "Show me all users" +- "Create a new user named John Doe with email john@example.com" +- "Find products under $100" +- "Generate a sales report by category" + +This agent demonstrates intelligent database interaction through conversation. + +## Complete Example + +Here's a comprehensive configuration example using multiple tool servers: + +```yaml +mcpServers: + github: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-github" + env: + GITHUB_PERSONAL_ACCESS_TOKEN: your-github-token + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + terminal: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-terminal" + desktop: + type: stdio + command: npx + args: + - -y + - "@wonderwhy-er/desktop-commander" + custom: + type: stdio + command: node + args: + - --loader + - ts-node/esm + - src/servers/customServer.ts + env: + API_KEY: your-api-key +llm: + provider: openai + model: gpt-5 + apiKey: $OPENAI_API_KEY +``` diff --git a/dexto/agents/agent-registry.json b/dexto/agents/agent-registry.json new file mode 100644 index 00000000..a85d7337 --- /dev/null +++ b/dexto/agents/agent-registry.json @@ -0,0 +1,148 @@ +{ + "version": "1.0.0", + "agents": { + "database-agent": { + "id": "database-agent", + "name": "Database Agent", + "description": "AI agent for database operations and SQL queries", + "author": "Truffle AI", + "tags": ["database", "sql", "data", "queries"], + "source": "database-agent/", + "main": "database-agent.yml" + }, + "github-agent": { + "id": "github-agent", + "name": "GitHub Agent", + "description": "GitHub operations agent for analyzing pull requests, issues, repos and more", + "author": "Truffle AI", + "tags": ["github", "repositories", "collaboration", "devops", "mcp"], + "source": "github-agent/", + "main": "github-agent.yml" + }, + "talk2pdf-agent": { + "id": "talk2pdf-agent", + "name": "Talk2PDF Agent", + "description": "PDF document analysis and conversation", + "author": "Truffle AI", + "tags": ["pdf", "documents", "analysis", "conversation"], + "source": "talk2pdf-agent/", + "main": "talk2pdf-agent.yml" + }, + "image-editor-agent": { + "id": "image-editor-agent", + "name": "Image Editor Agent", + "description": "AI agent for image editing and manipulation", + "author": "Truffle AI", + "tags": ["images", "editing", "graphics", "visual"], + "source": "image-editor-agent/", + "main": "image-editor-agent.yml" + }, + "music-agent": { + "id": "music-agent", + "name": "Music Agent", + "description": "AI agent for music creation and audio processing", + "author": "Truffle AI", + "tags": ["music", "audio", "creation", "sound"], + "source": "music-agent/", + "main": "music-agent.yml" + }, + "product-researcher": { + "id": "product-researcher", + "name": "Product Researcher", + "description": "AI agent for product name research and branding", + "author": "Truffle AI", + "tags": ["product", "research", "branding", "naming"], + "source": "product-name-researcher/", + "main": "product-name-researcher.yml" + }, + "triage-agent": { + "id": "triage-agent", + "name": "Triage Agent", + "description": "Customer support triage system", + "author": "Truffle AI", + "tags": ["support", "triage", "routing", "multi-agent"], + "source": "triage-demo/", + "main": "triage-agent.yml" + }, + "nano-banana-agent": { + "id": "nano-banana-agent", + "name": "Nano Banana Agent", + "description": "AI agent for advanced image generation and editing using Google's Nano Banana (Gemini 2.5 Flash Image)", + "author": "Truffle AI", + "tags": ["images", "generation", "editing", "ai", "nano-banana", "gemini"], + "source": "nano-banana-agent/", + "main": "nano-banana-agent.yml" + }, + "podcast-agent": { + "id": "podcast-agent", + "name": "Podcast Agent", + "description": "Advanced podcast generation agent using Google Gemini TTS for multi-speaker audio content", + "author": "Truffle AI", + "tags": ["podcast", "audio", "tts", "speech", "multi-speaker", "gemini"], + "source": "podcast-agent/", + "main": "podcast-agent.yml" + }, + "sora-video-agent": { + "id": "sora-video-agent", + "name": "Sora Video Agent", + "description": "AI agent for video generation using OpenAI's Sora technology with comprehensive video creation and management capabilities", + "author": "Truffle AI", + "tags": ["video", "generation", "sora", "openai", "ai", "media", "creation"], + "source": "sora-video-agent/", + "main": "sora-video-agent.yml" + }, + "default-agent": { + "id": "default-agent", + "name": "Default", + "description": "Default Dexto agent with filesystem and playwright tools", + "author": "Truffle AI", + "tags": ["default", "filesystem", "playwright"], + "source": "default-agent.yml" + }, + "coding-agent": { + "id": "coding-agent", + "name": "Coding Agent", + "description": "Expert software development assistant with all internal coding tools for building, debugging, and maintaining codebases", + "author": "Truffle AI", + "tags": ["coding", "development", "software", "internal-tools", "programming"], + "source": "coding-agent/", + "main": "coding-agent.yml" + }, + "workflow-builder-agent": { + "id": "workflow-builder-agent", + "name": "Workflow Builder Agent", + "description": "AI agent for building and managing n8n automation workflows with comprehensive workflow, execution, and credential management", + "author": "Truffle AI", + "tags": ["n8n", "automation", "workflows", "integration", "mcp"], + "source": "workflow-builder-agent/", + "main": "workflow-builder-agent.yml" + }, + "product-analysis-agent": { + "id": "product-analysis-agent", + "name": "Product Analysis Agent", + "description": "AI agent for product analytics using PostHog with insights, feature flags, error tracking, and user behavior analysis", + "author": "Truffle AI", + "tags": ["posthog", "analytics", "product", "insights", "feature-flags", "mcp"], + "source": "product-analysis-agent/", + "main": "product-analysis-agent.yml" + }, + "gaming-agent": { + "id": "gaming-agent", + "name": "Gaming Agent", + "description": "AI agent that plays GameBoy games like Pokemon through an emulator with screen capture and button controls", + "author": "Truffle AI", + "tags": ["gaming", "gameboy", "pokemon", "emulator", "mcp"], + "source": "gaming-agent/", + "main": "gaming-agent.yml" + }, + "explore-agent": { + "id": "explore-agent", + "name": "Explore Agent", + "description": "Fast, read-only agent for codebase exploration. Use for: 'explore the codebase', 'what's in this folder', 'how does X work', 'find where Y is handled', 'understand the architecture'. Optimized for speed with Haiku.", + "author": "Truffle AI", + "tags": ["explore", "search", "understand", "find", "research"], + "source": "explore-agent/", + "main": "explore-agent.yml" + } + } +} diff --git a/dexto/agents/agent-template.yml b/dexto/agents/agent-template.yml new file mode 100644 index 00000000..46f6cf04 --- /dev/null +++ b/dexto/agents/agent-template.yml @@ -0,0 +1,81 @@ +# Template for dexto create-app +# describes the mcp servers to use +mcpServers: + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + playwright: + type: stdio + command: npx + args: + - -y + - "@playwright/mcp@latest" + +# System prompt configuration - defines the agent's behavior and instructions +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a helpful AI assistant with access to tools. + Use these tools when appropriate to answer user queries. + You can use multiple tools in sequence to solve complex problems. + After each tool result, determine if you need more information or can provide a final answer. + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +# LLM configuration - describes the language model to use +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +# Alternative LLM providers (replace the above llm section with one of these): +# Google Gemini: +# llm: +# provider: google +# model: gemini-2.5-pro +# apiKey: $GOOGLE_GENERATIVE_AI_API_KEY +# +# Anthropic Claude: +# llm: +# provider: anthropic +# model: claude-sonnet-4-5-20250929 +# apiKey: $ANTHROPIC_API_KEY +# + +# Logger configuration - multi-transport logging system +# The CLI automatically adds a file transport for agent-specific logs +# logger: +# level: info # error | warn | info | debug | silly +# transports: +# - type: console +# colorize: true +# - type: file +# path: ./logs/agent.log +# maxSize: 10485760 # 10MB +# maxFiles: 5 + +# Storage configuration - uses a two-tier architecture: cache (fast, ephemeral) and database (persistent, reliable) +# Memory cache with file-based database (good for development with persistence) +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: in-memory + +# Tool confirmation configuration - auto-approve for better development experience +toolConfirmation: + mode: auto-approve + # timeout: omitted = infinite wait + allowedToolsStorage: memory diff --git a/dexto/agents/coding-agent/README.md b/dexto/agents/coding-agent/README.md new file mode 100644 index 00000000..0f68e569 --- /dev/null +++ b/dexto/agents/coding-agent/README.md @@ -0,0 +1,188 @@ +# Coding Agent + +An expert software development assistant optimized for building, debugging, and maintaining codebases. This agent comes equipped with all internal coding tools and is configured to handle complex software engineering tasks efficiently. + +## What You Get + +- **All Internal Coding Tools**: Read, write, edit files, execute commands, search codebases +- **Intelligent Tool Policies**: Read operations never require approval, write operations are safely guarded +- **Comprehensive File Support**: 30+ file extensions including JS/TS, Python, Go, Rust, Java, C/C++, configs, and more +- **Enhanced Codebase Access**: Index up to 500 files with depth-10 traversal, including hidden files +- **Expert System Prompt**: Specialized instructions for software development best practices +- **Persistent Tool Approvals**: Allowed tools are saved across sessions for smoother workflows +- **Coding-Focused Starter Prompts**: Quick access to common development tasks + +## Key Capabilities + +### File Operations +- **read_file**: Read any file with pagination support +- **write_file**: Create new files (requires approval) +- **edit_file**: Modify existing files precisely (requires approval) +- **glob_files**: Find files using patterns like `**/*.ts` (no approval needed) +- **grep_content**: Search within files using regex (no approval needed) + +### Command Execution +- **bash_exec**: Run shell commands for testing, building, running code (requires approval) +- **bash_output**: Monitor output from background processes +- **kill_process**: Terminate running processes + +### Analysis & Search +- Deep codebase traversal (up to 10 levels) +- Search across 500+ files +- Pattern matching with glob and regex +- Hidden file access (.env, .gitignore, etc.) + +## Requirements + +- Node.js 18+ (if using npm/pnpm commands) +- OpenAI API key (or another configured LLM key) +- File system access to your project directory + +## Run the Agent + +```bash +# From Dexto source +npm start -- --agent agents/coding-agent/coding-agent.yml + +# Or using the Dexto CLI +dexto --agent coding-agent +``` + +## Usage Examples + +### Analyze a Codebase +``` +"Analyze this codebase. Show me the project structure, main technologies used, and provide a high-level overview." +``` + +### Debug an Error +``` +"I'm getting this error: [paste error]. Help me find and fix the issue." +``` + +### Implement a Feature +``` +"I need to add user authentication. Help me design and implement it following best practices." +``` + +### Refactor Code +``` +"This function is too complex. Help me refactor it for better readability and maintainability." +``` + +### Write Tests +``` +"Generate unit tests for the UserService class with edge case coverage." +``` + +### Code Review +``` +"Review my recent changes in src/auth/ and suggest improvements." +``` + +## Configuration + +### LLM Options + +The coding agent defaults to `gpt-4o` for powerful coding capabilities. You can switch to other models: + +**Claude Sonnet (Excellent for Coding)** +```yaml +llm: + provider: anthropic + model: claude-sonnet-4-20250514 + apiKey: $ANTHROPIC_API_KEY +``` + +**Google Gemini** +```yaml +llm: + provider: google + model: gemini-2.5-pro + apiKey: $GOOGLE_GENERATIVE_AI_API_KEY +``` + +**OpenAI o1 (For Complex Reasoning)** +```yaml +llm: + provider: openai + model: o1 + apiKey: $OPENAI_API_KEY +``` + +### Tool Policies + +The agent is pre-configured with sensible defaults: + +**Always Allowed (No Approval Needed)** +- Reading files (`internal--read_file`) +- Searching files (`internal--glob_files`, `internal--grep_content`) +- Checking process output (`internal--bash_output`) +- Killing processes (`internal--kill_process`) +- Asking questions (`internal--ask_user`) + +**Requires Approval** +- Writing files (`internal--write_file`) +- Editing files (`internal--edit_file`) +- Executing commands (`internal--bash_exec`) + +You can customize these policies in the `toolConfirmation.toolPolicies` section of `coding-agent.yml`. + +### File Extensions + +The agent indexes these file types by default: + +**Web Development**: .js, .jsx, .ts, .tsx, .html, .css, .scss, .sass, .less, .vue, .svelte + +**Backend Languages**: .py, .java, .go, .rs, .rb, .php, .c, .cpp, .h, .hpp, .cs, .swift, .kt + +**Configuration**: .json, .yaml, .yml, .toml, .xml, .ini, .env + +**Documentation**: .md, .mdx, .txt, .rst + +**Build Files**: .gradle, .maven, Makefile, Dockerfile, .dockerignore, .gitignore + +Add more extensions in the `internalResources.resources[0].includeExtensions` section. + +## Starter Prompts + +The agent includes 8 built-in starter prompts: + +1. **🔍 Analyze Codebase** - Get a project overview +2. **🐛 Debug Error** - Identify and fix bugs +3. **♻️ Refactor Code** - Improve code quality +4. **🧪 Write Tests** - Generate comprehensive tests +5. **✨ Implement Feature** - Build new functionality +6. **⚡ Optimize Performance** - Find bottlenecks +7. **🚀 Setup Project** - Initialize new projects +8. **👀 Code Review** - Review for issues and improvements + +## Best Practices + +1. **Read Before Writing**: The agent automatically searches and reads relevant code before making changes +2. **Use Glob & Grep**: Leverage pattern matching to explore unfamiliar codebases efficiently +3. **Test Changes**: Execute tests after modifications to verify correctness +4. **Follow Conventions**: The agent adapts to your project's existing code style +5. **Ask Questions**: The agent will ask for clarification when requirements are ambiguous + +## Troubleshooting + +### Agent Can't Find Files +- Ensure you're running from your project root +- Check that file extensions are included in the config +- Verify `maxDepth` is sufficient for your project structure + +### Commands Require Too Many Approvals +- Use `allowedToolsStorage: storage` to persist approvals +- Add frequently-used commands to the `alwaysAllow` list + +### Performance Issues with Large Codebases +- Increase `maxFiles` limit (default: 500) +- Reduce `maxDepth` to limit traversal +- Exclude large directories in `.gitignore` + +## Learn More + +- [Dexto Documentation](https://github.com/truffle-ai/dexto) +- [Internal Tools Reference](../../docs/internal-tools.md) +- [Agent Configuration Guide](../../docs/agent-configuration.md) diff --git a/dexto/agents/coding-agent/coding-agent.yml b/dexto/agents/coding-agent/coding-agent.yml new file mode 100644 index 00000000..6c2fadaa --- /dev/null +++ b/dexto/agents/coding-agent/coding-agent.yml @@ -0,0 +1,298 @@ +# Coding Agent Configuration +# Optimized for software development with internal coding tools + +# Image: Specifies the provider bundle for this agent +# This agent requires local development tools (filesystem, process) +image: '@dexto/image-local' + +# System prompt configuration - defines the agent's behavior as a coding assistant +systemPrompt: + contributors: + - id: date + type: dynamic + priority: 10 + source: date + - id: env + type: dynamic + priority: 15 + source: env + - id: primary + type: static + priority: 0 + content: | + You are an expert software development assistant with deep knowledge of multiple programming languages, + frameworks, and development best practices. + + Your primary goal is to help users write, debug, refactor, and understand code efficiently. + + Key capabilities: + - Read and analyze codebases using glob and grep patterns + - Write and edit files with precise, well-structured code + - Execute shell commands for testing, building, and running code + - Debug issues by examining error messages and code structure + - Refactor code following best practices and design patterns + - Explain complex code concepts clearly + - Delegate exploration tasks to specialized sub-agents + + ## Task Delegation + + You have access to spawn_agent for delegating tasks to specialized sub-agents. + + **When to delegate to explore-agent:** + - Open-ended exploration: "explore the codebase", "what's in this folder", "how does X work" + - Understanding architecture: "explain the project structure", "how are components organized" + - Finding patterns: "where is authentication handled", "find all API endpoints" + - Research tasks: "what testing framework is used", "how is state managed" + + **When to use your own tools directly:** + - Specific file operations: "read src/index.ts", "edit the config file" + - Targeted searches: "find the User class", "grep for 'TODO'" + - Writing/editing code: any task that requires modifications + - Running commands: build, test, install dependencies + + **Rule of thumb:** If the task requires understanding or exploring before you know what to do, delegate to explore-agent first. If you know exactly what file/function to target, use your tools directly. + + ## Task Tracking with Todos + + Use the todo_write tool to track progress on multi-step tasks: + - **DO use todos for:** Feature implementations, multi-file refactors, bug fixes requiring investigation + fix + tests, any task with 3+ distinct steps + - **DON'T use todos for:** Reading files, answering questions, single-file edits, quick explanations + + When using todos: + 1. Create the todo list when you identify a multi-step task + 2. Mark a task as in_progress when you start working on it + 3. Mark it as completed when done, and move to the next task + 4. Keep only ONE task in_progress at a time + + ## Bash Tool Guidelines + When using bash_exec: + - Your environment is already configured with the correct working directory. + - NEVER prefix commands to run in current working directory with cd && . + - Only use cd as command prefix if the command needs to run outside the working directory. + + ## Guidelines + - Always read relevant code before making changes to understand context + - Use glob_files to find files and grep_content to search within files + - Test changes when possible using bash_exec + - Follow the project's existing code style and conventions + - Provide clear explanations for your code decisions + - Ask for clarification when requirements are ambiguous + +# Memory configuration - controls how memories are included in system prompt +memories: + enabled: true + priority: 40 + includeTimestamps: false + includeTags: true + limit: 10 + pinnedOnly: false + +# Greeting optimized for coding tasks +greeting: "👨‍💻 Ready to code! What are we building today?" + +# LLM configuration - using a powerful model for coding tasks +llm: + provider: anthropic + model: claude-sonnet-4-5-20250929 + apiKey: $ANTHROPIC_API_KEY + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 + +toolConfirmation: + mode: manual + # timeout: omitted = infinite wait (no timeout for CLI) + allowedToolsStorage: storage # Persist allowed tools across sessions + + # Tool policies optimized for coding workflows + toolPolicies: + # Tools that never require approval (safe, read-only operations) + # Use qualified names: custom--{tool_id} for custom tools, internal--{tool_id} for internal tools + alwaysAllow: + - internal--ask_user + - custom--read_file # Read files without approval + - custom--glob_files # Search for files without approval + - custom--grep_content # Search within files without approval + - custom--bash_output # Check background process output + - custom--kill_process # Kill processes without approval (only processes started by agent) + - custom--todo_write # Todo updates don't need approval + - custom--plan_read # Read plan without approval + + # Tools that are always denied (dangerous operations) + # Uncomment to restrict certain operations + # alwaysDeny: + # - custom--process-tools--bash_exec--rm -rf* # Prevent recursive deletion + +# Compaction configuration - automatically summarizes conversation when context is full +compaction: + type: reactive-overflow + enabled: true + # Uncomment to override model's context window (useful for testing or capping usage): + # maxContextTokens: 50000 # Cap at 50K tokens regardless of model's max + # thresholdPercent: 0.9 # Trigger at 90% of context window (leaves safety buffer) + +# Elicitation configuration - required for ask_user tool +elicitation: + enabled: true + # timeout: omitted = infinite wait (no timeout for CLI) + +# Internal tools - core tools that are always available +internalTools: + - ask_user # Ask questions and collect input + - invoke_skill # Invoke skills/prompts during execution + +# Custom tools - filesystem and process tools from image-local +customTools: + - type: filesystem-tools + allowedPaths: ["."] + blockedPaths: [".git", "node_modules/.bin", ".env"] + blockedExtensions: [".exe", ".dll", ".so"] + enableBackups: false + - type: process-tools + securityLevel: moderate + - type: todo-tools + - type: plan-tools + basePath: "${{dexto.project_dir}}/plans" + - type: agent-spawner + maxConcurrentAgents: 5 + defaultTimeout: 300000 + allowSpawning: true + # List of agent IDs from the registry that can be spawned. + # Agent metadata (name, description) is pulled from the registry at runtime. + allowedAgents: + - explore-agent + # Agents with read-only tools that should have auto-approved tool calls + autoApproveAgents: + - explore-agent + +# Internal resources configuration - expanded for coding projects +internalResources: + enabled: true + resources: + # Filesystem resource - comprehensive file access for coding + - type: filesystem + paths: ["."] + maxFiles: 500 # Increased for larger codebases + maxDepth: 10 # Deeper traversal for nested projects + includeHidden: true # Include hidden files (.env, .gitignore, etc.) + includeExtensions: + # Web development + - .js + - .jsx + - .ts + - .tsx + - .html + - .css + - .scss + - .sass + - .less + - .vue + - .svelte + # Backend languages + - .py + - .java + - .go + - .rs + - .rb + - .php + - .c + - .cpp + - .h + - .hpp + - .cs + - .swift + - .kt + # Shell and config + - .sh + - .bash + - .zsh + - .fish + # Config and data + - .json + - .yaml + - .yml + - .toml + - .xml + - .ini + - .env + # Documentation + - .md + - .mdx + - .txt + - .rst + # Build and package files + - .gradle + - .maven + - .dockerignore + - .gitignore + + # Blob resource - for handling build artifacts, images, etc. + - type: blob + +# Prompts - coding-focused examples shown as clickable buttons in WebUI +prompts: + # Skills - can be invoked by the LLM via invoke_skill tool + - type: file + file: "${{dexto.agent_dir}}/skills/code-review.md" + # user-invocable: true (default) - appears as /code-review slash command + # disable-model-invocation: false (default) - LLM can invoke via invoke_skill + - type: inline + id: analyze-codebase + title: "🔍 Analyze Codebase" + description: "Get an overview of the project structure" + prompt: "Analyze this codebase. Show me the project structure, main technologies used, and provide a high-level overview." + category: analysis + priority: 10 + showInStarters: true + - type: inline + id: implement-feature + title: "✨ Implement Feature" + description: "Build a new feature from scratch" + prompt: "Help me implement a new feature. I'll describe what I need, and you can design and implement it following best practices." + category: development + priority: 9 + showInStarters: true + - type: inline + id: write-tests + title: "🧪 Write Tests" + description: "Generate unit tests for code" + prompt: "Help me write comprehensive unit tests. Identify the testing framework and create tests that cover edge cases." + category: testing + priority: 8 + showInStarters: true + - type: inline + id: refactor-code + title: "♻️ Refactor Code" + description: "Improve code quality and structure" + prompt: "Help me refactor some code to improve its structure, readability, and maintainability while preserving functionality." + category: refactoring + priority: 7 + showInStarters: true + +## Alternative LLM configurations for coding + +## Claude Sonnet (excellent for coding) +# llm: +# provider: anthropic +# model: claude-sonnet-4-20250514 +# apiKey: $ANTHROPIC_API_KEY + +## Google Gemini (strong coding capabilities) +# llm: +# provider: google +# model: gemini-2.5-pro +# apiKey: $GOOGLE_GENERATIVE_AI_API_KEY + +## OpenAI o1 (for complex reasoning tasks) +# llm: +# provider: openai +# model: o1 +# apiKey: $OPENAI_API_KEY diff --git a/dexto/agents/coding-agent/skills/code-review.md b/dexto/agents/coding-agent/skills/code-review.md new file mode 100644 index 00000000..f8ab44e8 --- /dev/null +++ b/dexto/agents/coding-agent/skills/code-review.md @@ -0,0 +1,46 @@ +--- +title: Code Review +description: Perform a thorough code review on the specified files or changes +arguments: + - name: focus + description: Optional focus area (security, performance, style, all) + required: false +--- + +# Code Review Skill + +You are now performing a code review. Follow these guidelines: + +## Review Checklist + +1. **Correctness**: Does the code do what it's supposed to do? +2. **Security**: Are there any security vulnerabilities (injection, XSS, etc.)? +3. **Performance**: Are there any obvious performance issues? +4. **Readability**: Is the code easy to understand? +5. **Maintainability**: Will this code be easy to maintain? +6. **Error Handling**: Are errors handled appropriately? +7. **Tests**: Are there adequate tests for the changes? + +## Output Format + +Structure your review as: + +### Summary +Brief overview of what the code does and overall assessment. + +### Issues Found +List any problems, categorized by severity: +- **Critical**: Must fix before merge +- **Major**: Should fix, but not blocking +- **Minor**: Nice to have improvements +- **Nitpick**: Style/preference suggestions + +### Positive Highlights +Note any particularly good patterns or practices. + +### Recommendations +Actionable suggestions for improvement. + +--- + +Begin the code review now. If no specific files were mentioned, ask the user what they'd like reviewed. diff --git a/dexto/agents/database-agent/README.md b/dexto/agents/database-agent/README.md new file mode 100644 index 00000000..08be88e4 --- /dev/null +++ b/dexto/agents/database-agent/README.md @@ -0,0 +1,35 @@ +# Database Agent + +An AI agent that provides natural language access to database operations and analytics. This approach changes how we interact with data - instead of learning SQL syntax, building query interfaces, or designing complex dashboards, users can simply ask for what they need in natural language. + +## Setup + +```bash +cd database-agent +./setup-database.sh +npm start -- --agent database-agent.yml +``` + +## Example Interactions + +- "Show me all users" +- "Create a new user named John Doe with email john@example.com" +- "Find products under $100" +- "Generate a sales report by category" + +## Capabilities + +- **Data Queries**: Natural language database queries and reporting +- **Data Management**: Create, update, and delete records +- **Analytics**: Generate insights and business intelligence +- **Schema Operations**: Table creation and database structure management + +## How It Works + +The agent connects to a SQLite database via MCP server and: +- Interprets natural language requests into SQL queries +- Validates data before operations +- Provides formatted results and insights +- Handles errors gracefully with helpful suggestions + +This agent demonstrates intelligent database interaction through conversation. \ No newline at end of file diff --git a/dexto/agents/database-agent/data/example.db b/dexto/agents/database-agent/data/example.db new file mode 100644 index 00000000..82d63056 Binary files /dev/null and b/dexto/agents/database-agent/data/example.db differ diff --git a/dexto/agents/database-agent/database-agent-example.sql b/dexto/agents/database-agent/database-agent-example.sql new file mode 100644 index 00000000..258cb0d9 --- /dev/null +++ b/dexto/agents/database-agent/database-agent-example.sql @@ -0,0 +1,98 @@ +-- Sample database schema and data for the Database Interaction Agent +-- This demonstrates the types of operations the agent can perform + +-- Create users table +CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + email TEXT UNIQUE NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_login DATETIME, + is_active BOOLEAN DEFAULT 1 +); + +-- Create products table +CREATE TABLE IF NOT EXISTS products ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + description TEXT, + price DECIMAL(10,2) NOT NULL, + category TEXT NOT NULL, + stock_quantity INTEGER DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- Create orders table +CREATE TABLE IF NOT EXISTS orders ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + total_amount DECIMAL(10,2) NOT NULL, + status TEXT DEFAULT 'pending', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) +); + +-- Create order_items table +CREATE TABLE IF NOT EXISTS order_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id INTEGER NOT NULL, + product_id INTEGER NOT NULL, + quantity INTEGER NOT NULL, + unit_price DECIMAL(10,2) NOT NULL, + FOREIGN KEY (order_id) REFERENCES orders(id), + FOREIGN KEY (product_id) REFERENCES products(id) +); + +-- Insert sample users +INSERT INTO users (name, email) VALUES + ('John Doe', 'john@example.com'), + ('Jane Smith', 'jane@example.com'), + ('Bob Johnson', 'bob@example.com'), + ('Alice Brown', 'alice@example.com'), + ('Charlie Wilson', 'charlie@example.com'); + +-- Insert sample products +INSERT INTO products (name, description, price, category, stock_quantity) VALUES + ('Laptop', 'High-performance laptop for professionals', 899.99, 'Electronics', 15), + ('Smartphone', 'Latest smartphone with advanced features', 699.99, 'Electronics', 25), + ('Coffee Maker', 'Automatic coffee maker for home use', 89.99, 'Home & Kitchen', 30), + ('Running Shoes', 'Comfortable running shoes for athletes', 129.99, 'Sports', 20), + ('Backpack', 'Durable backpack for daily use', 49.99, 'Fashion', 40), + ('Bluetooth Speaker', 'Portable wireless speaker', 79.99, 'Electronics', 18), + ('Yoga Mat', 'Non-slip yoga mat for fitness', 29.99, 'Sports', 35), + ('Desk Lamp', 'LED desk lamp with adjustable brightness', 39.99, 'Home & Kitchen', 22); + +-- Insert sample orders +INSERT INTO orders (user_id, total_amount, status) VALUES + (1, 899.99, 'completed'), + (2, 209.98, 'completed'), + (3, 159.98, 'pending'), + (4, 699.99, 'completed'), + (5, 89.99, 'shipped'); + +-- Insert sample order items +INSERT INTO order_items (order_id, product_id, quantity, unit_price) VALUES + (1, 1, 1, 899.99), -- John bought a laptop + (2, 3, 1, 89.99), -- Jane bought a coffee maker + (2, 7, 1, 29.99), -- Jane also bought a yoga mat + (2, 8, 1, 39.99), -- Jane also bought a desk lamp + (3, 5, 1, 49.99), -- Bob bought a backpack + (3, 6, 1, 79.99), -- Bob also bought a bluetooth speaker + (3, 8, 1, 39.99), -- Bob also bought a desk lamp + (4, 2, 1, 699.99), -- Alice bought a smartphone + (5, 3, 1, 89.99); -- Charlie bought a coffee maker + +-- Update some user last_login times +UPDATE users SET last_login = datetime('now', '-1 day') WHERE id = 1; +UPDATE users SET last_login = datetime('now', '-3 days') WHERE id = 2; +UPDATE users SET last_login = datetime('now', '-7 days') WHERE id = 3; +UPDATE users SET last_login = datetime('now', '-2 days') WHERE id = 4; +UPDATE users SET last_login = datetime('now', '-5 days') WHERE id = 5; + +-- Create indexes for better performance +CREATE INDEX IF NOT EXISTS idx_users_email ON users(email); +CREATE INDEX IF NOT EXISTS idx_products_category ON products(category); +CREATE INDEX IF NOT EXISTS idx_orders_user_id ON orders(user_id); +CREATE INDEX IF NOT EXISTS idx_orders_status ON orders(status); +CREATE INDEX IF NOT EXISTS idx_order_items_order_id ON order_items(order_id); +CREATE INDEX IF NOT EXISTS idx_order_items_product_id ON order_items(product_id); \ No newline at end of file diff --git a/dexto/agents/database-agent/database-agent.yml b/dexto/agents/database-agent/database-agent.yml new file mode 100644 index 00000000..e1550975 --- /dev/null +++ b/dexto/agents/database-agent/database-agent.yml @@ -0,0 +1,166 @@ +# Database Interaction Agent +# This agent demonstrates an alternative approach to building database interfaces +# Instead of traditional web UIs with forms and buttons, this agent provides +# natural language interaction with database operations through MCP tools + +mcpServers: + # SQLite database server for direct database interaction + sqlite: + type: stdio + command: npx + args: + - -y + - "@executeautomation/database-server" + - "${{dexto.agent_dir}}/data/example.db" + timeout: 30000 + connectionMode: strict + + # Filesystem access for database file management and schema inspection + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + timeout: 30000 + connectionMode: lenient + +# Optional greeting shown at chat start (UI can consume this) +greeting: "🗄️ Hi! I'm your Database Agent. What would you like to explore?" + +# System prompt that defines the agent's database interaction capabilities +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a Database Interaction Agent that provides natural language access to database operations + and analytics. You orchestrate database operations through intelligent conversation and tool usage. + + ## Your Core Capabilities + + **Database Operations:** + - Execute SQL queries and return formatted results + - Create, modify, and drop database tables + - Insert, update, and delete records + - Analyze database schema and structure + - Generate reports and data insights + - Perform data validation and integrity checks + + **Intelligent Orchestration:** + - Understand user intent from natural language + - Break down complex requests into sequential operations + - Validate data before operations + - Provide clear explanations of what you're doing + - Handle errors gracefully with helpful suggestions + + **Intelligent Data Operations:** + - Natural conversation for data access + - Intelligent data handling and validation + - Context-aware operations and insights + - Flexible querying and reporting + + ## Interaction Patterns + + **For Data Queries:** + 1. Understand what the user wants to know + 2. Formulate appropriate SQL queries + 3. Execute and format results clearly + 4. Provide insights or suggest follow-up questions + + **For Data Modifications:** + 1. Confirm the user's intent + 2. Validate data integrity + 3. Execute the operation safely + 4. Confirm success and show results + + **For Schema Operations:** + 1. Analyze current structure + 2. Plan the changes needed + 3. Execute modifications + 4. Verify the new structure + + ## Best Practices + + - Always explain what you're doing before doing it + - Show sample data when creating tables + - Validate user input before database operations + - Provide helpful error messages and suggestions + - Use transactions for multi-step operations + - Keep responses concise but informative + + ## Example Interactions + + User: "Create a users table with name, email, and created_at fields" + You: "I'll create a users table with the specified fields. Let me set this up for you..." + + User: "Show me all users who signed up this month" + You: "I'll query the users table for recent signups. Let me get that information..." + + User: "Add a new user named John Doe with email john@example.com" + You: "I'll insert a new user record for John Doe. Let me add that to the database..." + + Remember: You're demonstrating intelligent database interaction through + natural conversation and data analysis. + + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +# Storage configuration +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 + +# LLM configuration for intelligent database interactions +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + temperature: 0.1 # Lower temperature for more consistent database operations + +# Prompts - database interaction examples shown as clickable buttons in WebUI +prompts: + - type: inline + id: explore-database + title: "🔍 Explore Database" + description: "See what's in the database" + prompt: "Show me what tables exist in the database and their structure." + category: exploration + priority: 10 + showInStarters: true + - type: inline + id: create-table + title: "🗂️ Create Table" + description: "Design and create a new database table" + prompt: "Create a products table with columns for name, description, price, and stock quantity." + category: schema + priority: 9 + showInStarters: true + - type: inline + id: insert-data + title: "➕ Insert Data" + description: "Add new records to a table" + prompt: "Insert a new product into the products table with name 'Laptop', price 999.99, and stock 15." + category: data-management + priority: 8 + showInStarters: true + - type: inline + id: query-data + title: "📊 Query Data" + description: "Search and filter database records" + prompt: "Show me all products from the products table sorted by price." + category: queries + priority: 7 + showInStarters: true \ No newline at end of file diff --git a/dexto/agents/database-agent/setup-database.sh b/dexto/agents/database-agent/setup-database.sh new file mode 100755 index 00000000..f22edf27 --- /dev/null +++ b/dexto/agents/database-agent/setup-database.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Exit immediately on errors, unset variables, or pipeline failures +set -euo pipefail + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Setup script for Database Interaction Agent +# This script creates the data directory and initializes the database with sample data + +echo "🚀 Setting up Database Interaction Agent..." + +# Create data directory if it doesn't exist +echo "📁 Creating data directory..." +mkdir -p "${SCRIPT_DIR}/data" + +# Check if SQLite is available +if ! command -v sqlite3 &> /dev/null; then + echo "❌ SQLite3 is not installed. Please install SQLite3 first:" + echo " macOS: brew install sqlite3" + echo " Ubuntu/Debian: sudo apt-get install sqlite3" + echo " Windows: Download from https://www.sqlite.org/download.html" + exit 1 +fi + +# Initialize database with sample data +echo "🗄️ Initializing database with sample data..." + +# Remove existing database if it exists to avoid constraint violations +if [ -f "${SCRIPT_DIR}/data/example.db" ]; then + echo "🗑️ Removing existing database..." + rm "${SCRIPT_DIR}/data/example.db" +fi + +sqlite3 "${SCRIPT_DIR}/data/example.db" < "${SCRIPT_DIR}/database-agent-example.sql" + +# Verify the database was created successfully +if [ -f "${SCRIPT_DIR}/data/example.db" ]; then + echo "✅ Database created successfully!" + + # Show some basic stats + echo "📊 Database statistics:" + echo " Tables: $(sqlite3 "${SCRIPT_DIR}/data/example.db" "SELECT COUNT(*) FROM sqlite_master WHERE type='table';")" + echo " Users: $(sqlite3 "${SCRIPT_DIR}/data/example.db" "SELECT COUNT(*) FROM users;")" + echo " Products: $(sqlite3 "${SCRIPT_DIR}/data/example.db" "SELECT COUNT(*) FROM products;")" + echo " Orders: $(sqlite3 "${SCRIPT_DIR}/data/example.db" "SELECT COUNT(*) FROM orders;")" + + echo "" + echo "🎉 Database setup complete!" + echo "" + echo "You can now run the Database Interaction Agent with:" + echo " dexto --agent agents/database-agent.yml" + echo "" + echo "Example interactions you can try:" + echo " - 'Show me all users'" + echo " - 'List products under \$100'" + echo " - 'Create a new user named Test User with email test@example.com'" + echo " - 'Show me total sales by category'" + echo " - 'Find users who haven't logged in for more than 5 days'" +else + echo "❌ Failed to create database. Please check the SQL file and try again." + exit 1 +fi \ No newline at end of file diff --git a/dexto/agents/default-agent.yml b/dexto/agents/default-agent.yml new file mode 100644 index 00000000..4d347a44 --- /dev/null +++ b/dexto/agents/default-agent.yml @@ -0,0 +1,217 @@ +# describes the mcp servers to use +mcpServers: + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + playwright: + type: stdio + command: npx + args: + - -y + - "@playwright/mcp@latest" + +# System prompt configuration - defines the agent's behavior and instructions +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a helpful AI assistant with access to tools. + Use these tools when appropriate to answer user queries. + You can use multiple tools in sequence to solve complex problems. + After each tool result, determine if you need more information or can provide a final answer. + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +# Memory configuration - controls how memories are included in system prompt +memories: + enabled: true + priority: 40 + includeTimestamps: false + includeTags: true + limit: 10 + pinnedOnly: false + +# Optional greeting shown at chat start (UI can consume this) +greeting: "Hi! I'm Dexto — how can I help today?" + +# LLM configuration - describes the language model to use +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + + # Optional: Control which media types are expanded for LLM consumption + # If omitted, uses model capabilities from registry (recommended default) + # Supports MIME patterns with wildcards (e.g., "image/*", "video/mp4") + # allowedMediaTypes: + # - "image/*" # All images (png, jpg, gif, etc.) + # - "application/pdf" # PDF documents + # - "audio/*" # All audio formats + # Note: Unsupported types become descriptive placeholders like: [Video: demo.mp4 (5.2 MB)] + +# Logger configuration - multi-transport logging system +# The CLI automatically adds a file transport for agent-specific logs +# logger: +# level: info # error | warn | info | debug | silly +# transports: +# - type: console +# colorize: true +# - type: file +# path: ./logs/agent.log +# maxSize: 10485760 # 10MB +# maxFiles: 5 + +storage: + cache: + type: in-memory + database: + type: sqlite + # path: ./data/dexto.db # Optional: customize database location + blob: + type: local # CLI provides storePath automatically + # storePath: ./data/blobs # Optional: customize blob storage location (defaults to agent-specific path) + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 + +toolConfirmation: + mode: manual + # timeout: # Time to wait for approval (ms). Omit for no timeout (wait indefinitely) + allowedToolsStorage: memory # 'memory' or 'storage' for persisting allowed tools + + # Optional: Static tool policies for fine-grained allow/deny control + toolPolicies: + # Tools that never require approval (low-risk, common operations) + alwaysAllow: + - internal--ask_user + - mcp--read_file + - mcp--list_directory + - mcp--list_allowed_directories + + # Tools that are always denied (high-risk, destructive operations) + # Deny list takes precedence over allow list + # alwaysDeny: + # - mcp--filesystem--delete_file + # - mcp--playwright--execute_script + +# Elicitation configuration - separate from tool confirmation +# Allows auto-approve for tools while still supporting user input requests +elicitation: + enabled: true # Enable ask_user tool and MCP server elicitations + # timeout: # Time to wait for user input (ms). Omit for no timeout (wait indefinitely) + +# Internal tools - built-in Dexto capabilities +internalTools: + - ask_user # Allows the agent to ask you questions and collect structured input + - delegate_to_url + - list_resources + - get_resource + +# Internal resources configuration - manages file system access and blob storage +# NOTE: Blob storage capacity and backend settings are in the 'storage.blob' section above +internalResources: + enabled: true + resources: + # Filesystem resource - provides read access to local files for the agent + - type: filesystem + paths: ["."] # Directories to expose + maxFiles: 50 # Maximum number of files to index + maxDepth: 3 # Maximum directory depth to traverse + includeHidden: false # Include hidden files/directories + includeExtensions: [".txt", ".md", ".json", ".yaml", ".yml", ".js", ".ts", ".py", ".html", ".css"] + + # Blob resource - enables large file upload/storage (settings in storage.blob section above) + - type: blob + +# Plugin system - built-in plugins for content policy and response sanitization +# plugins: +# # ContentPolicy - validates and sanitizes input before sending to LLM +# contentPolicy: +# priority: 10 # Lower priority = runs first +# blocking: true # Blocks execution if validation fails +# maxInputChars: 50000 # Maximum input length (characters) +# redactEmails: true # Redact email addresses from input +# redactApiKeys: true # Redact potential API keys from input +# enabled: true # Enable this plugin + +# # ResponseSanitizer - sanitizes LLM responses before returning to user +# responseSanitizer: +# priority: 900 # Higher priority = runs near the end +# blocking: false # Non-blocking (logs warnings but doesn't stop) +# redactEmails: true # Redact email addresses from responses +# redactApiKeys: true # Redact potential API keys from responses +# maxResponseLength: 100000 # Maximum response length (characters) +# enabled: true # Enable this plugin + +# # Custom plugins can be added here (see documentation) +# # custom: +# # - name: tenant-auth +# # module: "${{dexto.agent_dir}}/plugins/tenant-auth.ts" +# # enabled: true +# # blocking: true +# # priority: 100 +# # config: +# # enforceQuota: true + +# Prompts - inline prompts shown as clickable buttons in WebUI (showInStarters: true) +prompts: + - type: inline + id: quick-start + title: "📚 Quick Start Guide" + description: "Learn the basics and see what you can do" + prompt: "I'd like to get started quickly. Can you show me a few examples of what you can do and help me understand how to work with you?" + category: learning + priority: 9 + showInStarters: true + - type: inline + id: tool-demo + title: "⚡ Tool Demonstration" + description: "See the tools in action with practical examples" + prompt: "I'd like to see your tools in action. Can you pick one of your most interesting tools and demonstrate it with a practical example? Show me what it can do and how it works." + category: tools + priority: 5 + showInStarters: true + - type: inline + id: snake-game + title: "🐍 Create Snake Game" + description: "Build a fun interactive game with HTML, CSS, and JavaScript" + prompt: "Create a snake game in a new directory with HTML, CSS, and JavaScript, then open it in the browser for me to play." + category: coding + priority: 4 + showInStarters: true + - type: inline + id: connect-tools + title: "🔧 Connect New Tools" + description: "Browse and add MCP servers to extend capabilities" + prompt: "I want to connect new tools to expand my capabilities. Can you help me understand what MCP servers are available and how to add them?" + category: tools + priority: 3 + showInStarters: true + +# Telemetry configuration (optional) - OpenTelemetry for distributed tracing +# telemetry: +# serviceName: dexto-default-agent +# enabled: true +# tracerName: dexto-tracer +# export: +# type: otlp # 'otlp' for production, 'console' for development +# protocol: http # 'http' or 'grpc' +# endpoint: http://127.0.0.1:4318/v1/traces # OTLP collector endpoint +# headers: # Optional headers for authentication +# Authorization: Bearer + +## To use Google Gemini, replace the LLM section with Google Gemini configuration below +## Similar for anthropic/groq/etc. +# llm: +# provider: google +# model: gemini-2.5-pro +# apiKey: $GOOGLE_GENERATIVE_AI_API_KEY diff --git a/dexto/agents/examples/README.md b/dexto/agents/examples/README.md new file mode 100644 index 00000000..7db03c1f --- /dev/null +++ b/dexto/agents/examples/README.md @@ -0,0 +1,22 @@ +# Configuration Examples + +This folder contains examples for agents with different configurations. These examples demonstrate how to configure and set up various agents to handle different use cases. + +You can directly plug in these configuration files and try them out on your local system to see the power of different AI Agents! + +## Available Examples + +### `linear-task-manager.yml` +A task management agent that integrates with Linear's official MCP server to help you manage issues, projects, and team collaboration through natural language commands. Features include: +- Create, update, and search Linear issues +- Manage project status and tracking +- Add comments and collaborate with team members +- Handle task assignments and priority management + +**Setup**: Requires Linear workspace authentication when first connecting. + +### Other Examples +- `email_slack.yml` - Email and Slack integration +- `notion.yml` - Notion workspace management +- `ollama.yml` - Local LLM integration +- `website_designer.yml` - Web design assistance diff --git a/dexto/agents/examples/email_slack.yml b/dexto/agents/examples/email_slack.yml new file mode 100644 index 00000000..0d344247 --- /dev/null +++ b/dexto/agents/examples/email_slack.yml @@ -0,0 +1,26 @@ +# Email to Slack Automation Configuration +# This agent monitors emails and posts summaries to Slack +mcpServers: + gmail: + type: sse + url: "composio-url" + slack: + type: stdio + command: "npx" + args: + - -y + - "@modelcontextprotocol/server-slack" + env: + SLACK_BOT_TOKEN: "slack-bot-token" + SLACK_TEAM_ID: "slack-team-id" + +# System prompt - defines the agent's behavior for email processing +systemPrompt: | + Prompt the user to provide the information needed to answer their question or identify them on Slack. + Also let them know that they can directly update the systemPrompt in the yml if they prefer. + +# LLM Configuration +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY diff --git a/dexto/agents/examples/linear-task-manager.yml b/dexto/agents/examples/linear-task-manager.yml new file mode 100644 index 00000000..97a62ed0 --- /dev/null +++ b/dexto/agents/examples/linear-task-manager.yml @@ -0,0 +1,58 @@ +# Linear Task Management Agent +# This agent integrates with Linear's MCP server to manage tasks, issues, and projects +# through natural language commands. + +systemPrompt: | + You are a Linear Task Management Agent specialized in helping users manage their Linear workspace efficiently. You have access to Linear's official MCP server that allows you to: + + ## Your Capabilities + - **Issue Management**: Find, create, update, and manage Linear issues + - **Project Tracking**: Access and manage Linear projects and their status + - **Team Collaboration**: View team activity, assign tasks, and track progress + - **Comment Management**: Add comments to issues and participate in discussions + - **Status Updates**: Update issue status, priority, and labels + - **Search & Filter**: Find specific issues, projects, or team members + + ## How You Should Behave + - Always confirm destructive actions (deleting, major status changes) before proceeding + - Provide clear summaries when listing multiple issues or projects + - Use natural language to explain Linear concepts when needed + - Be proactive in suggesting task organization and workflow improvements + - When creating issues, ask for essential details if not provided (title, description, priority) + - Offer to set up logical task relationships (dependencies, sub-tasks) when appropriate + + ## Usage Examples + - "Create a new issue for fixing the login bug with high priority" + - "Show me all open issues assigned to me" + - "Update the API documentation task to in progress" + - "Find all issues related to the mobile app project" + - "Add a comment to issue #123 about the testing results" + - "What's the status of our current sprint?" + +mcpServers: + linear: + type: stdio + command: npx + args: + - -y + - mcp-remote + - https://mcp.linear.app/sse + connectionMode: strict + # Note: Linear MCP requires authentication through your Linear workspace + # You'll need to authenticate when first connecting + +toolConfirmation: + mode: auto-approve + allowedToolsStorage: memory + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +storage: + cache: + type: in-memory + database: + type: sqlite + path: .dexto/database/linear-task-manager.db \ No newline at end of file diff --git a/dexto/agents/examples/notion.yml b/dexto/agents/examples/notion.yml new file mode 100644 index 00000000..39778452 --- /dev/null +++ b/dexto/agents/examples/notion.yml @@ -0,0 +1,31 @@ +# Refer https://github.com/makenotion/notion-mcp-server for information on how to use notion mcp server +# mcpServers: +# notion: +# type: stdio +# url: "get-url-from-composio-or-any-other-provider" + +# System prompt configuration - defines the agent's behavior and instructions +systemPrompt: | + You are a helpful Notion AI assistant. Your primary goals are to: + 1. Help users organize and manage their Notion workspace effectively + 2. Assist with creating, editing, and organizing pages and databases + 3. Provide guidance on Notion features and best practices + 4. Help users find and retrieve information from their Notion workspace + + When interacting with users: + - Ask clarifying questions to understand their specific needs + - Provide step-by-step instructions when explaining complex tasks + - Suggest relevant Notion templates or structures when appropriate + - Explain the reasoning behind your recommendations + + If you need additional information to help the user: + - Ask for specific details about their Notion workspace + - Request clarification about their goals or requirements + - Inquire about their current Notion setup and experience level + + Remember to be concise, clear, and focus on practical solutions. + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY diff --git a/dexto/agents/examples/ollama.yml b/dexto/agents/examples/ollama.yml new file mode 100644 index 00000000..fb1e063f --- /dev/null +++ b/dexto/agents/examples/ollama.yml @@ -0,0 +1,67 @@ +# describes the mcp servers to use +mcpServers: + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + playwright: + type: stdio + command: npx + args: + - -y + - "@playwright/mcp@latest" + # hf: + # type: stdio + # command: npx + # args: + # - -y + # - "@llmindset/mcp-hfspace" + +# System prompt configuration - defines the agent's behavior and instructions +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a helpful AI assistant with access to tools. + Use these tools when appropriate to answer user queries. + You can use multiple tools in sequence to solve complex problems. + After each tool result, determine if you need more information or can provide a final answer. + + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +# first start the ollama server +# ollama run gemma3n:e2b +# then run the following command to start the agent: +# dexto --agent +# dexto --agent for web ui +llm: + provider: openai-compatible + model: gemma3n:e2b + baseURL: http://localhost:11434/v1 + apiKey: $OPENAI_API_KEY + maxInputTokens: 32768 + +# Storage configuration - uses a two-tier architecture: cache (fast, ephemeral) and database (persistent, reliable) +# Memory cache with file-based database (good for development with persistence) +# storage: +# cache: +# type: in-memory +# database: +# type: sqlite +# path: ./data/dexto.db + +## To use Google Gemini, replace the LLM section with Google Gemini configuration below +## Similar for anthropic/groq/etc. +# llm: +# provider: google +# model: gemini-2.0-flash +# apiKey: $GOOGLE_GENERATIVE_AI_API_KEY diff --git a/dexto/agents/examples/website_designer.yml b/dexto/agents/examples/website_designer.yml new file mode 100644 index 00000000..517c32d9 --- /dev/null +++ b/dexto/agents/examples/website_designer.yml @@ -0,0 +1,34 @@ +# describes the mcp servers to use +mcpServers: + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + playwright: + type: stdio + command: npx + args: + - -y + - "@playwright/mcp@latest" + +# System prompt configuration - defines the agent's behavior and instructions +systemPrompt: | + You are a professional website developer. You design beautiful, aesthetic websites. + Use these tools when appropriate to answer user queries. + You can use multiple tools in sequence to solve complex problems. + After each tool result, determine if you need more information or can provide a final answer. + When building a website, do this in a separate folder to keep it separate from the rest of the code. + The website should look clean, professional, modern and elegant. + It should be visually appealing. Carefully consider the color scheme, font choices, and layout. I like non-white backgrounds. + It should be responsive and mobile-friendly, and seem like a professional website. + After you are done building it, open it up in the browser + +# # describes the llm configuration +llm: + provider: openai + model: gpt-5 + # you can update the system prompt to change the behavior of the llm + apiKey: $OPENAI_API_KEY diff --git a/dexto/agents/explore-agent/explore-agent.yml b/dexto/agents/explore-agent/explore-agent.yml new file mode 100644 index 00000000..1b4267ab --- /dev/null +++ b/dexto/agents/explore-agent/explore-agent.yml @@ -0,0 +1,126 @@ +# Explore Agent Configuration +# Lightweight, read-only agent optimized for codebase exploration +# Designed to be spawned by other agents for research tasks + +# No image bundle - uses only explicitly defined tools below + +# System prompt optimized for exploration tasks +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a fast, focused exploration agent specialized in understanding codebases and finding information. + + ## Your Mission + Quickly and thoroughly explore codebases to answer questions, find patterns, locate files, and understand architecture. You are optimized for speed and accuracy. + + ## Available Tools + You have access to read-only tools: + - `glob_files` - Find files matching patterns (e.g., "src/**/*.ts", "*.config.js") + - `grep_content` - Search for text/patterns within files + - `read_file` - Read file contents + + ## Exploration Strategy + + ### For "quick" searches: + - Single glob or grep to find the target + - Read 1-2 most relevant files + - Return focused answer + + ### For "medium" exploration: + - Multiple search patterns to find related files + - Read key files to understand connections + - Summarize findings with file references + + ### For "very thorough" analysis: + - Comprehensive search across multiple naming conventions + - Trace imports, exports, and dependencies + - Map relationships between components + - Provide detailed analysis with evidence + + ## Guidelines + - Start with broad searches, then narrow down + - Use glob for file discovery, grep for content search + - Try multiple naming conventions (camelCase, snake_case, kebab-case, PascalCase) + - Check common locations: src/, lib/, packages/, tests/, config/ + - Report what you found AND what you didn't find + - Include file paths and line numbers in your response + - Be concise but complete + + ## Response Format + Structure your response as: + 1. **Summary** - Brief answer to the question + 2. **Key Files** - List of relevant files found + 3. **Details** - Specific findings with code references + 4. **Notes** - Any caveats or areas that need further exploration + +# LLM configuration - Haiku for speed and cost efficiency +# When spawned as a sub-agent, this config may be overridden by parent's LLM +# if the API key is not available (with a warning to use 'dexto login') +llm: + provider: anthropic + model: claude-haiku-4-5-20251001 + apiKey: $ANTHROPIC_API_KEY + +# Minimal storage - in-memory only for ephemeral use +storage: + cache: + type: in-memory + database: + type: in-memory + blob: + type: in-memory + +# Auto-approve all tools since they're read-only +toolConfirmation: + mode: auto-approve + allowedToolsStorage: memory + +# No internal tools needed for exploration +internalTools: [] + +# Read-only filesystem tools only - explicitly enable only read operations +customTools: + - type: filesystem-tools + enabledTools: ["read_file", "glob_files", "grep_content"] # Read-only tools only + allowedPaths: ["."] + blockedPaths: + # Version control + - ".git" + # Binaries + - "node_modules/.bin" + # Environment files with secrets + - ".env" + - ".env.local" + - ".env.production" + - ".env.development" + - ".env.test" + - ".env.staging" + # Package manager credentials + - ".npmrc" + - ".yarnrc" + - ".pypirc" + # Git credentials + - ".git-credentials" + - ".gitconfig" + # SSH keys + - ".ssh" + # Cloud provider credentials + - ".aws" + - ".gcp" + - ".azure" + # Kubernetes config + - ".kube" + # Docker credentials + - ".docker" + blockedExtensions: [".exe", ".dll", ".so", ".dylib"] + maxFileSize: 10485760 # 10MB + enableBackups: false + +# Note: This agent intentionally excludes: +# - write_file, edit_file (write operations) - not in enabledTools +# - process-tools (bash execution) - provider not included +# - agent-spawner (no sub-agent spawning) - provider not included +# This ensures it remains a safe, read-only exploration tool diff --git a/dexto/agents/gaming-agent/gaming-agent.yml b/dexto/agents/gaming-agent/gaming-agent.yml new file mode 100644 index 00000000..41d8e837 --- /dev/null +++ b/dexto/agents/gaming-agent/gaming-agent.yml @@ -0,0 +1,109 @@ +# Gaming Agent - Plays GameBoy games with save state support + +mcpServers: + gameboy: + type: stdio + command: npx + args: + - -y + - "@truffle-ai/gameboy-server@0.1.0" + timeout: 60000 + connectionMode: strict + +greeting: "Hey! I'm your Gaming Agent. Want to continue a saved game or start something new?" + +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a Gaming Agent that plays GameBoy games. You see the screen and press buttons. + + ## On Startup + **ALWAYS check first:** Call `list_games` to see installed games, then `list_states` for any game with saves. Ask user if they want to continue from a save or start fresh. + + ## Tools + + **Games:** `list_games`, `load_rom` (game name or path), `install_rom` + **Saves:** `save_state`, `load_state`, `list_states`, `delete_state` + **Controls:** `press_up/down/left/right/a/b/start/select` (duration_frames default: 25) + **Speed:** `set_speed(multiplier)` - 1=normal, 2-4x for walking/grinding, 1x for battles + **View:** `start_live_view` (use after loading), `stop_live_view`, `get_screen`, `wait_frames` + + ## Save States + - **Save often** - before battles, important choices, risky moves + - Save states capture exact emulator state - restore anytime + - Games stored in `~/.gameboy-mcp/games/`, saves persist across sessions + - Loading a ROM from a path auto-installs it + + ## Workflow + 1. Check for existing games/saves → offer to continue + 2. `load_rom` (by name or path) → `start_live_view` + 3. Play with button presses, save frequently + 4. Before anything risky: `save_state` + + ## Pokemon Tips + - Title: START → Menus: D-pad + A/B → Text: spam A + - Use 3-4x speed for walking/exploring, 1x for battles and menus + - Save before gym battles, catching legendaries, tough trainers + + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local + maxBlobSize: 52428800 + maxTotalSize: 1073741824 + cleanupAfterDays: 30 + +llm: + provider: anthropic + model: claude-haiku-4-5-20251001 + apiKey: $ANTHROPIC_API_KEY + +toolConfirmation: + mode: auto-approve + allowedToolsStorage: memory + +prompts: + - type: inline + id: continue-save + title: "Continue from Save" + description: "Resume from a previous save state" + prompt: "Check my installed games and save states. Show me what I can continue from." + category: gaming + priority: 10 + showInStarters: true + - type: inline + id: new-game + title: "Start New Game" + description: "Load a ROM and start fresh" + prompt: "I want to start a new game. Show me my installed games or I'll give you a ROM path." + category: gaming + priority: 9 + showInStarters: true + - type: inline + id: play-pokemon + title: "Play Pokemon" + description: "Play Pokemon (checks for existing saves)" + prompt: "Let's play Pokemon! Check if I have any Pokemon saves to continue, or start a new game." + category: gaming + priority: 8 + showInStarters: true + - type: inline + id: save-progress + title: "Save Progress" + description: "Create a save state" + prompt: "Save my current progress with a descriptive name." + category: gaming + priority: 7 + showInStarters: true diff --git a/dexto/agents/github-agent/README.md b/dexto/agents/github-agent/README.md new file mode 100644 index 00000000..6cc91ed5 --- /dev/null +++ b/dexto/agents/github-agent/README.md @@ -0,0 +1,58 @@ +# GitHub Integration Agent + +Bring GitHub context into any workspace. This agent starts the `@truffle-ai/github-mcp-server` in `stdio` mode so the assistant can explore repositories, manage pull requests, and automate GitHub project workflows directly from chat. + +## What You Get +- Full coverage of GitHub toolsets (repos, pull requests, issues, actions, discussions, notifications, security, projects, and more) +- Automatic OAuth device-flow login with cached tokens, no personal access token required for most users +- Safe-guarded write operations (issues, PRs, workflow runs, comments, etc.) that the agent confirms before executing +- Optional read-only or scoped toolsets to limit the surface area for sensitive environments + +## Requirements +- Node.js 18+ with access to `npx` +- A GitHub account with access to the repositories you plan to manage +- Browser access to complete the one-time device-code OAuth prompt (opened automatically) +- `OPENAI_API_KEY` (or another configured LLM key) exported in your shell for the agent + +## Run the Agent +```bash +npm start -- --agent agents/github-agent/github-agent.yml +``` + +The CLI launches `npx -y @truffle-ai/github-mcp-server stdio`. On first run you will see a device code and a browser window prompting you to authorize the "GitHub MCP Server" application. Approving the flow stores an access token at: + +``` +~/.config/truffle/github-mcp/-token.json +``` + +Subsequent sessions reuse the cached token unless it expires or you delete the file. Once the server reports it is ready, start chatting with Dexto about repositories, issues, CI failures, releases, or team activity. + +## Optional Configuration +You can tailor the underlying MCP server by exporting environment variables before starting Dexto: + +- `GITHUB_PERSONAL_ACCESS_TOKEN`: Provide a PAT (with `repo` and `read:user` at minimum) if you need to bypass OAuth or run in headless environments. +- `GITHUB_OAUTH_SCOPES="repo read:user"`: Override the scopes requested during the OAuth device flow (space or comma separated). Use this to request additional permissions (e.g., `gist`) or limit scopes in tightly controlled environments. +- `GITHUB_TOOLSETS="repos,issues,pull_requests,actions"`: Restrict which groups of tools the agent loads. Available groups include `repos`, `issues`, `pull_requests`, `actions`, `notifications`, `discussions`, `projects`, `code_security`, `dependabot`, `secret_protection`, `security_advisories`, `users`, `orgs`, `gists`, `context`, and `experiments`. +- `GITHUB_READ_ONLY=1`: Offer only read-only tools; write operations will be hidden. +- `GITHUB_DYNAMIC_TOOLSETS=1`: Enable on-demand toolset discovery so the model only activates tools as needed. +- `GITHUB_HOST=https://github.mycompany.com`: Point the agent at GitHub Enterprise Server or ghe.com tenants. +- `GITHUB_LOG_FILE=~/github-mcp.log` and `GITHUB_ENABLE_COMMAND_LOGGING=1`: Persist detailed MCP command logs for auditing. +- `GITHUB_CONTENT_WINDOW_SIZE=7500`: Increase the amount of content retrieved for large diffs or logs. + +Refer to the upstream [`github-mcp-server`](https://github.com/github/github-mcp-server) documentation for the full flag list (every CLI flag is mirrored as an environment variable using the `GITHUB_` prefix). + +## Switching to the Remote GitHub MCP Server (optional) +If you prefer to connect to the GitHub-hosted remote MCP server instead of running the bundled binary, replace the `mcpServers.github` block in `github-agent.yml` with: + +```yaml +mcpServers: + github: + type: http + url: https://api.githubcopilot.com/mcp/ + connectionMode: strict +``` + +You can optionally add `headers` for PAT authentication if your host does not support OAuth. Restart Dexto after saving the change. + +## Resetting Authorization +To force a new OAuth login, delete the cached token file (`rm ~/.config/truffle/github-mcp/*-token.json`) and relaunch the agent. The next startup will trigger a fresh device flow. diff --git a/dexto/agents/github-agent/github-agent.yml b/dexto/agents/github-agent/github-agent.yml new file mode 100644 index 00000000..4bf45cd7 --- /dev/null +++ b/dexto/agents/github-agent/github-agent.yml @@ -0,0 +1,107 @@ +# GitHub Integration Agent configuration +# Connects Dexto to GitHub via the bundled GitHub MCP server binary + +mcpServers: + github: + # type: http + # url: https://api.githubcopilot.com/mcp/ + # # timeout: 45000 + # connectionMode: strict + type: stdio + command: npx + args: + - -y + - '@truffle-ai/github-mcp-server' + - stdio + # Optional: uncomment to override the default OAuth scopes requested during the device flow + # env: + # GITHUB_OAUTH_SCOPES: 'repo read:user' + +# Optional greeting shown at chat start (UI can consume this) +greeting: "🐙 Hello! I'm your GitHub Agent. How can I help with your repositories?" + +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are the GitHub Integration Agent for Dexto. Collaborate with users on GitHub repositories by leveraging the GitHub tools available in this runtime. Inspect repositories, manage pull requests and issues, coordinate releases, and share actionable guidance while honoring GitHub permissions and organizational policies. + + ## Core Responsibilities + - Surface repository structure, history, and code insights that help users understand the current state of their projects + - Draft, triage, and refine issues, pull requests, project boards, and release notes on request + - Perform focused code reviews, flag risky changes, and suggest remediation steps or next actions + - Present step-by-step plans before executing write operations or multi-step procedures + + ## Interaction Guidelines + - For queries related to starring repos, do not ask for confirmation, just star the repo + - Confirm the repository, branch, or resource when a request is ambiguous or spans multiple contexts + - For read-only requests, respond concisely and include relevant references, links, or identifiers. + - Before taking any action that modifies GitHub state, summarize the intended change and obtain confirmation + - If you are blocked (missing scopes, inaccessible repository, conflicting state), say so clearly and offer next steps or alternatives + + ## Tool Usage + - Prefer live GitHub data from the provided tools instead of relying on local filesystem copies or stale context + - Break complex objectives into sequential tool calls, narrating progress so the user can follow along + - After each tool interaction, decide whether additional context is needed or if you can deliver a confident answer + + Stay practical, keep responses focused on the user’s goal, and highlight any assumptions or risks you notice. + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +# Storage configuration +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +# Prompts - GitHub operations examples shown as clickable buttons in WebUI +prompts: + - type: inline + id: repo-info + title: "📊 Repository Info" + description: "Get details about the Dexto repository" + prompt: "Show me information about the truffle-ai/dexto repository including stars, forks, and recent activity." + category: info + priority: 10 + showInStarters: true + - type: inline + id: list-issues + title: "📋 List Issues" + description: "View open issues in Dexto" + prompt: "List all open issues in the truffle-ai/dexto repository." + category: issues + priority: 9 + showInStarters: true + - type: inline + id: star-dexto + title: "⭐ Star Dexto" + description: "Star the Dexto repository" + prompt: "Star the truffle-ai/dexto repository on GitHub." + category: engagement + priority: 8 + showInStarters: true + - type: inline + id: recent-commits + title: "📝 Recent Commits" + description: "View latest commits to Dexto" + prompt: "Show me the recent commits to the truffle-ai/dexto repository." + category: activity + priority: 7 + showInStarters: true + diff --git a/dexto/agents/image-editor-agent/Lenna.webp b/dexto/agents/image-editor-agent/Lenna.webp new file mode 100644 index 00000000..7871386b Binary files /dev/null and b/dexto/agents/image-editor-agent/Lenna.webp differ diff --git a/dexto/agents/image-editor-agent/README.md b/dexto/agents/image-editor-agent/README.md new file mode 100644 index 00000000..899f9acb --- /dev/null +++ b/dexto/agents/image-editor-agent/README.md @@ -0,0 +1,435 @@ +# Image Editor Agent + +A comprehensive AI agent for image editing and processing using the [Image Editor MCP Server](https://github.com/truffle-ai/mcp-servers/tree/main/src/image-editor). + +This agent provides a complete suite of image manipulation tools through a Python-based MCP server built with OpenCV and Pillow. + +## Features + +### 🖼️ **Viewing & Preview** +- **Image Preview**: Get base64 previews for display in chat interfaces +- **System Viewer**: Open images in the system's default image viewer +- **Image Details**: Show detailed information in a user-friendly format +- **Thumbnail Generation**: Create quick thumbnail versions +- **Image Comparison**: Compare two images and highlight differences +- **Detailed Analysis**: Comprehensive image statistics and color analysis + +### ✂️ **Basic Operations** +- **Resize**: Resize images with aspect ratio preservation +- **Crop**: Crop images to specified dimensions +- **Format Conversion**: Convert between JPG, PNG, WebP, BMP, TIFF + +### 🎨 **Filters & Effects** +- **Basic**: Blur, sharpen, grayscale, invert +- **Artistic**: Sepia, vintage, cartoon, sketch +- **Detection**: Edge detection, emboss + +### ⚙️ **Adjustments** +- **Brightness & Contrast**: Fine-tune image appearance +- **Color Analysis**: Detailed color statistics and histograms + +### 📝 **Drawing & Annotations** +- **Basic Shapes**: Draw rectangles, circles, lines, and arrows +- **Text Overlay**: Add text with customizable font, size, color, and position +- **Annotations**: Add text with background for better visibility +- **Shape Properties**: Control thickness, fill, and positioning + +### 🔍 **Computer Vision** +- **Object Detection**: Detect faces, edges, contours, circles, lines +- **Image Analysis**: Detailed statistics, color analysis, histogram data + +### 🎯 **Advanced Features** +- **Collage Creation**: Create collages with multiple layout types and templates +- **Batch Processing**: Process multiple images with the same operation +- **Filter Discovery**: List all available filters and effects +- **Template System**: Predefined layouts for professional collages + +## Quick Start + +### Prerequisites +- **Node.js 20+**: For the Dexto framework +- **Python 3.10+**: Automatically managed by the MCP server + +### Installation + +1. **Run the Agent**: + ```bash + # From the dexto project root + dexto --agent agents/image-editor-agent/image-editor-agent.yml + ``` + +That's it! The MCP server will be automatically downloaded and installed via `uvx` on first run. + +## Configuration + +The agent is configured to use the published MCP server: + +```yaml +mcpServers: + image_editor: + type: stdio + command: uvx + args: + - truffle-ai-image-editor-mcp + connectionMode: strict +``` + +## MCP Server + +This agent uses the **Image Editor MCP Server**, which is maintained separately at: + +**🔗 [https://github.com/truffle-ai/mcp-servers/tree/main/src/image-editor](https://github.com/truffle-ai/mcp-servers/tree/main/src/image-editor)** + +The MCP server repository provides: +- Complete technical documentation +- Development and contribution guidelines +- Server implementation details +- Advanced configuration options + +## Available Tools + +### Viewing & Preview Tools + +#### `preview_image` +Get a base64 preview of an image for display in chat interfaces. + +**Parameters:** +- `filePath` (string): Path to the image file +- `maxSize` (integer, optional): Maximum size for preview (default: 800) + +#### `open_image_viewer` +Open an image in the system's default image viewer. + +**Parameters:** +- `filePath` (string): Path to the image file + +#### `show_image_details` +Display detailed information about an image in a user-friendly format. + +**Parameters:** +- `filePath` (string): Path to the image file + +#### `create_thumbnail` +Create a thumbnail version of an image for quick preview. + +**Parameters:** +- `filePath` (string): Path to the image file +- `size` (integer, optional): Thumbnail size (default: 150) +- `outputPath` (string, optional): Path for the output thumbnail + +#### `compare_images` +Compare two images and show differences. + +**Parameters:** +- `image1Path` (string): Path to the first image +- `image2Path` (string): Path to the second image + +### Basic Image Operations + +#### `get_image_info` +Get detailed information about an image file. + +**Parameters:** +- `filePath` (string): Path to the image file to analyze + +#### `resize_image` +Resize an image to specified dimensions. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `width` (integer, optional): Target width in pixels +- `height` (integer, optional): Target height in pixels +- `maintainAspectRatio` (boolean, optional): Whether to maintain aspect ratio (default: true) +- `quality` (integer, optional): Output quality 1-100 (default: 90) + +#### `crop_image` +Crop an image to specified dimensions. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `x` (integer): Starting X coordinate for cropping +- `y` (integer): Starting Y coordinate for cropping +- `width` (integer): Width of the crop area +- `height` (integer): Height of the crop area + +#### `convert_format` +Convert an image to a different format. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `format` (string): Target format (jpg, jpeg, png, webp, bmp, tiff) +- `quality` (integer, optional): Output quality 1-100 for lossy formats (default: 90) + +### Filters & Effects + +#### `apply_filter` +Apply various filters and effects to an image. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `filter` (string): Type of filter (blur, sharpen, grayscale, sepia, invert, edge_detection, emboss, vintage, cartoon, sketch) +- `intensity` (number, optional): Filter intensity 0.1-5.0 (default: 1.0) + +#### `list_available_filters` +List all available image filters and effects. + +**Parameters:** None + +### Adjustments + +#### `adjust_brightness_contrast` +Adjust brightness and contrast of an image. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `brightness` (number, optional): Brightness adjustment -100 to 100 (default: 0) +- `contrast` (number, optional): Contrast multiplier 0.1 to 3.0 (default: 1.0) + +### Drawing & Annotations + +#### `draw_rectangle` +Draw a rectangle on an image. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `x` (integer): X coordinate of top-left corner +- `y` (integer): Y coordinate of top-left corner +- `width` (integer): Width of the rectangle +- `height` (integer): Height of the rectangle +- `color` (string, optional): Color in hex format (default: "#FF0000") +- `thickness` (integer, optional): Line thickness (default: 3) +- `filled` (boolean, optional): Whether to fill the rectangle (default: false) + +#### `draw_circle` +Draw a circle on an image. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `centerX` (integer): X coordinate of circle center +- `centerY` (integer): Y coordinate of circle center +- `radius` (integer): Radius of the circle +- `color` (string, optional): Color in hex format (default: "#00FF00") +- `thickness` (integer, optional): Line thickness (default: 3) +- `filled` (boolean, optional): Whether to fill the circle (default: false) + +#### `draw_line` +Draw a line on an image. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `startX` (integer): X coordinate of line start +- `startY` (integer): Y coordinate of line start +- `endX` (integer): X coordinate of line end +- `endY` (integer): Y coordinate of line end +- `color` (string, optional): Color in hex format (default: "#0000FF") +- `thickness` (integer, optional): Line thickness (default: 2) + +#### `draw_arrow` +Draw an arrow on an image. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `startX` (integer): X coordinate of arrow start +- `startY` (integer): Y coordinate of arrow start +- `endX` (integer): X coordinate of arrow end +- `endY` (integer): Y coordinate of arrow end +- `color` (string, optional): Color in hex format (default: "#FF00FF") +- `thickness` (integer, optional): Line thickness (default: 2) +- `tipLength` (number, optional): Arrow tip length as fraction of line (default: 0.3) + +#### `add_text_to_image` +Add text overlay to an image. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `text` (string): Text to add to the image +- `x` (integer): X coordinate for text placement +- `y` (integer): Y coordinate for text placement +- `fontSize` (integer, optional): Font size in pixels (default: 30) +- `color` (string, optional): Text color in hex format (default: "#FFFFFF") + +#### `add_annotation` +Add an annotation with background to an image. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `outputPath` (string, optional): Path for the output image +- `text` (string): Text to add to the image +- `x` (integer): X coordinate for text placement +- `y` (integer): Y coordinate for text placement +- `fontSize` (integer, optional): Font size in pixels (default: 20) +- `textColor` (string, optional): Text color in hex format (default: "#FFFFFF") +- `backgroundColor` (string, optional): Background color in hex format (default: "#000000") +- `padding` (integer, optional): Padding around text (default: 5) + +### Computer Vision + +#### `detect_objects` +Detect objects in an image using OpenCV. + +**Parameters:** +- `inputPath` (string): Path to the input image file +- `detectionType` (string): Type of detection (faces, edges, contours, circles, lines) + +#### `analyze_image` +Analyze image statistics and properties. + +**Parameters:** +- `inputPath` (string): Path to the input image file + +### Advanced Features + +#### `create_collage` +Create a collage from multiple images with various layout options. + +**Parameters:** +- `imagePaths` (array): List of image file paths +- `layout` (string, optional): Layout type (grid, horizontal, vertical, mosaic, random, custom) (default: grid) +- `outputPath` (string, optional): Path for the output collage +- `maxWidth` (integer, optional): Maximum width for individual images (default: 1200) +- `spacing` (integer, optional): Spacing between images (default: 10) +- `canvasWidth` (integer, optional): Custom canvas width for mosaic/random/custom layouts +- `canvasHeight` (integer, optional): Custom canvas height for mosaic/random/custom layouts +- `backgroundColor` (string, optional): Background color in hex format (default: "#FFFFFF") +- `customPositions` (array, optional): List of {x, y} coordinates for custom layout +- `randomSeed` (integer, optional): Seed for reproducible random layouts + +#### `create_collage_template` +Create a collage using predefined templates. + +**Parameters:** +- `imagePaths` (array): List of image file paths +- `template` (string, optional): Template type (photo_wall, storyboard, featured, instagram_grid, polaroid) (default: photo_wall) +- `outputPath` (string, optional): Path for the output collage +- `maxWidth` (integer, optional): Maximum canvas width (default: 1200) +- `backgroundColor` (string, optional): Background color in hex format (default: "#FFFFFF") + +#### `list_collage_templates` +List all available collage templates and layouts. + +**Parameters:** None + +#### `batch_process` +Process multiple images with the same operation. + +**Parameters:** +- `inputPaths` (array): List of input image paths +- `operation` (string): Operation type (resize, filter, brightness_contrast, convert) +- `outputDirectory` (string, optional): Output directory for processed images +- Additional parameters depend on the operation type + +## Supported Image Formats + +- **JPEG/JPG**: Lossy compression, good for photos +- **PNG**: Lossless compression, good for graphics with transparency +- **WebP**: Modern format with good compression +- **BMP**: Uncompressed bitmap format +- **TIFF**: High-quality format for professional use + +## Dependencies + +- **OpenCV**: Computer vision operations and image processing +- **Pillow**: Image manipulation and text rendering +- **NumPy**: Numerical operations +- **MCP**: Model Context Protocol server implementation + +## Example Usage + +### Basic Image Operations +``` +"Resize the image at /path/to/image.jpg to 800x600 pixels" +"Crop the image to show only the top-left quarter" +"Convert the image to PNG format" +``` + +### Interactive Viewing +``` +"Show me a preview of the image" +"Open this image in the system viewer" +"Display detailed information about the image" +``` + +### Filters and Effects +``` +"Apply a vintage filter to the image" +"Create a cartoon effect on the image" +"Apply edge detection to find contours" +``` + +### Analysis and Detection +``` +"Analyze the color statistics of the image" +"Detect faces in the image" +"Compare two images and show differences" +``` + +### Drawing and Annotations +``` +"Draw a red rectangle around the face in the image" +"Add a circle to highlight the center point" +"Draw an arrow pointing to the important feature" +"Add an annotation saying 'Face detected' with a black background" +"Draw a line connecting two points in the image" +``` + +### Advanced Features +``` +"Create a mosaic collage from these images" +"Create a featured layout collage with one large image" +"Create an Instagram grid from 9 photos" +"Create a custom collage with specific positions" +"List all available collage templates" +"Batch process all images in the folder to apply a blur filter" +"Show me a preview of the image" +``` + +## Troubleshooting + +### Common Issues + +1. **OpenCV Installation**: If you encounter issues with OpenCV, ensure you have the required system dependencies: + ```bash + # macOS + brew install opencv + + # Ubuntu/Debian + sudo apt-get install libopencv-dev + ``` + +2. **Font Issues**: If text rendering fails, the server will fall back to the default font. + +3. **Memory Issues**: For large images, consider resizing before processing to avoid memory constraints. + +4. **Path Issues**: Ensure all file paths are absolute or correctly relative to the working directory. + +## Troubleshooting + +### Common Issues + +1. **Server Installation**: The MCP server will be automatically installed via `uvx` on first run. No manual setup required. + +2. **OpenCV Installation**: The server includes OpenCV installation - this may take a moment on first run due to the large download (35MB+). + +3. **Memory Issues**: For large images, consider resizing before processing to avoid memory constraints. + +4. **Path Issues**: Ensure all file paths are absolute or correctly relative to the working directory. + +## Getting Help + +- **MCP Server Issues**: Report at the [mcp-servers repository](https://github.com/truffle-ai/mcp-servers/issues) +- **Agent Configuration**: Report at the main Dexto repository +- **Feature Requests**: Use the mcp-servers repository for tool-related requests + +## License + +This project is part of the Dexto AI agent framework. \ No newline at end of file diff --git a/dexto/agents/image-editor-agent/image-editor-agent.yml b/dexto/agents/image-editor-agent/image-editor-agent.yml new file mode 100644 index 00000000..a5115950 --- /dev/null +++ b/dexto/agents/image-editor-agent/image-editor-agent.yml @@ -0,0 +1,52 @@ +# Dexto Agent Configuration for Python Image Editor MCP Server +# Generated on 2025-07-18T19:30:00.000Z + +# Optional greeting shown at chat start (UI can consume this) +greeting: "🎨 Hello! I'm your Image Editor. What image shall we create or edit today?" + +systemPrompt: | + You are an AI assistant specialized in image editing and processing. You have access to a comprehensive set of tools for manipulating images including: + + - **Basic Operations**: Resize, crop, convert formats + - **Filters & Effects**: Blur, sharpen, grayscale, sepia, invert, edge detection, emboss, vintage + - **Adjustments**: Brightness, contrast, color adjustments + - **Text & Overlays**: Add text to images with customizable fonts and colors + - **Computer Vision**: Face detection, edge detection, contour analysis, circle detection, line detection + - **Analysis**: Detailed image statistics, color analysis, histogram data + + When working with images: + 1. Always validate that the input image exists and is in a supported format + 2. Provide clear feedback about what operations you're performing + 3. Save processed images with descriptive names + 4. Include image information (dimensions, file size, format) in your responses + 5. Suggest additional enhancements when appropriate + + Supported image formats: JPG, JPEG, PNG, BMP, TIFF, WebP + +mcpServers: + image_editor: + type: stdio + command: uvx + args: + - truffle-ai-image-editor-mcp + connectionMode: strict + +toolConfirmation: + mode: "auto-approve" + allowedToolsStorage: "memory" + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 \ No newline at end of file diff --git a/dexto/agents/image-gen-agent/image-gen-agent.yml b/dexto/agents/image-gen-agent/image-gen-agent.yml new file mode 100644 index 00000000..0179a163 --- /dev/null +++ b/dexto/agents/image-gen-agent/image-gen-agent.yml @@ -0,0 +1,70 @@ +# Dexto Agent Configuration for OpenAI Image Generation MCP Server +# Uses GPT Image API (gpt-image-1) for high-quality image generation + +greeting: "Hello! I'm your AI Image Generator powered by OpenAI's GPT Image API. What would you like me to create?" + +systemPrompt: | + You are an expert image generation assistant powered by OpenAI's GPT Image API (gpt-image-1). + + **YOUR CAPABILITIES:** + - Generate images from detailed text descriptions + - Edit existing images with masks for precise modifications + - Create transparent backgrounds for icons, sprites, and overlays + - Produce images in various formats (PNG, JPEG, WebP) + - Multiple quality levels: low (~$0.02), medium (~$0.07), high (~$0.19) + + **WHEN GENERATING IMAGES:** + 1. Craft detailed prompts - include style, composition, lighting, colors, and mood + 2. Ask clarifying questions if the request is vague + 3. Suggest appropriate settings based on use case: + - Icons/sprites: background="transparent", size="1024x1024" + - Hero images: quality="high", size="1536x1024" (landscape) + - Portrait images: size="1024x1536" + - Quick drafts: quality="low" + - Photos: output_format="jpeg" + - Graphics with transparency: output_format="png" + + **WHEN EDITING IMAGES:** + 1. Explain how masks work (transparent areas = regions to edit) + 2. Provide clear descriptions of desired changes + 3. Suggest generating multiple variations (n=2-3) for comparison + + **BEST PRACTICES:** + - Default to quality="medium" for good balance of cost and quality + - Use quality="high" only for final production assets + - Confirm user's intent before generating expensive high-quality images + - Always provide the output file path after generation + - For transparent backgrounds, remind users to use PNG or WebP format + + Be creative, helpful, and guide users to get the best possible results. + +mcpServers: + openai_image: + type: stdio + command: npx + args: + - -y + - '@truffle-ai/openai-image-server' + env: + OPENAI_API_KEY: $OPENAI_API_KEY + connectionMode: strict + +toolConfirmation: + mode: "auto-approve" + allowedToolsStorage: "memory" + +llm: + provider: openai + model: gpt-5.2 + apiKey: $OPENAI_API_KEY + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 diff --git a/dexto/agents/logger-agent/logger-agent.yml b/dexto/agents/logger-agent/logger-agent.yml new file mode 100644 index 00000000..a9346935 --- /dev/null +++ b/dexto/agents/logger-agent/logger-agent.yml @@ -0,0 +1,157 @@ +# Request Logger Agent Configuration +# Demonstrates custom plugin integration with complete lifecycle testing +# Logs all user requests, tool calls, and assistant responses to ~/.dexto/logs/request-logger.log + +# MCP Servers - basic filesystem and browser tools +mcpServers: + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + playwright: + type: stdio + command: npx + args: + - -y + - "@playwright/mcp@latest" + +# System prompt configuration +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a helpful AI assistant with comprehensive logging enabled. + All your interactions (requests, tool calls, and responses) are being logged to help understand your behavior. + + Use tools when appropriate to answer user queries. You can use multiple tools in sequence to solve complex problems. + After each tool result, determine if you need more information or can provide a final answer. + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +# Memory configuration - controls how memories are included in system prompt +memories: + enabled: true + priority: 40 + includeTimestamps: false + includeTags: true + limit: 10 + pinnedOnly: false + +# Optional greeting shown at chat start +greeting: "Hi! I'm the Logger Agent — all interactions are being logged for analysis. How can I help?" + +# LLM configuration +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +# Storage configuration +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 + maxTotalSize: 1073741824 + cleanupAfterDays: 30 + +# Tool confirmation settings +toolConfirmation: + mode: manual + # timeout: omitted = infinite wait + allowedToolsStorage: memory + +# Elicitation configuration - required for ask_user tool +elicitation: + enabled: true + # timeout: omitted = infinite wait + +# Internal tools +internalTools: + - ask_user + +# Internal resources configuration +internalResources: + enabled: true + resources: + - type: filesystem + paths: ["."] + maxFiles: 50 + maxDepth: 3 + includeHidden: false + includeExtensions: [".txt", ".md", ".json", ".yaml", ".yml", ".js", ".ts", ".py", ".html", ".css"] + - type: blob + +# Plugin system configuration +plugins: + # Built-in plugins + contentPolicy: + priority: 10 + blocking: true + maxInputChars: 50000 + redactEmails: true + redactApiKeys: true + enabled: true + + responseSanitizer: + priority: 900 + blocking: false + redactEmails: true + redactApiKeys: true + maxResponseLength: 100000 + enabled: true + + # Custom Request Logger Plugin + custom: + - name: request-logger + module: "${{dexto.agent_dir}}/plugins/request-logger.ts" + enabled: true + blocking: false # Non-blocking - we just want to observe, not interfere + priority: 5 # Run early to capture original data before other plugins modify it + config: {} # Empty config uses defaults: ~/.dexto/logs/request-logger.log + +# Prompts - shown as clickable buttons in WebUI +prompts: + - type: inline + id: simple-question + title: "🤔 Ask a Simple Question" + description: "Test basic request/response logging" + prompt: "What is the capital of France?" + category: learning + priority: 9 + showInStarters: true + - type: inline + id: tool-usage + title: "🔧 Use a Tool" + description: "Test tool call and result logging" + prompt: "List the files in the current directory" + category: tools + priority: 8 + showInStarters: true + - type: inline + id: multi-step + title: "🎯 Multi-Step Task" + description: "Test logging across multiple tool calls" + prompt: "Create a new file called test.txt with the content 'Hello from Logger Agent' and then read it back to me" + category: tools + priority: 7 + showInStarters: true + - type: inline + id: check-logs + title: "📋 Check the Logs" + description: "View the request logger output" + prompt: "Can you read the file at ~/.dexto/logs/request-logger.log and show me the last 50 lines?" + category: tools + priority: 6 + showInStarters: true diff --git a/dexto/agents/logger-agent/plugins/request-logger.ts b/dexto/agents/logger-agent/plugins/request-logger.ts new file mode 100644 index 00000000..708126ae --- /dev/null +++ b/dexto/agents/logger-agent/plugins/request-logger.ts @@ -0,0 +1,186 @@ +import type { + DextoPlugin, + BeforeLLMRequestPayload, + BeforeResponsePayload, + BeforeToolCallPayload, + AfterToolResultPayload, + PluginResult, + PluginExecutionContext, +} from '@core/plugins/types.js'; +import { promises as fs } from 'fs'; +import { homedir } from 'os'; +import { join } from 'path'; + +/** + * Request Logger Plugin + * + * Logs all user requests and assistant responses to a file for debugging and analysis. + * Demonstrates the complete plugin lifecycle including resource management. + * + * Features: + * - Logs user input (text, images, files) + * - Logs tool calls and results + * - Logs assistant responses with token usage + * - Proper resource cleanup on shutdown + */ +export class RequestLoggerPlugin implements DextoPlugin { + private logFilePath: string = ''; + private logFileHandle: fs.FileHandle | null = null; + private requestCount: number = 0; + + /** + * Initialize plugin - create log directory and open log file + */ + async initialize(config: Record): Promise { + // Default log path: ~/.dexto/logs/request-logger.log + const logDir = config.logDir || join(homedir(), '.dexto', 'logs'); + const logFileName = config.logFileName || 'request-logger.log'; + this.logFilePath = join(logDir, logFileName); + + // Ensure log directory exists + await fs.mkdir(logDir, { recursive: true }); + + // Open log file in append mode + this.logFileHandle = await fs.open(this.logFilePath, 'a'); + + // Write initialization header + await this.writeLog('='.repeat(80)); + await this.writeLog(`Request Logger initialized at ${new Date().toISOString()}`); + await this.writeLog(`Log file: ${this.logFilePath}`); + await this.writeLog('='.repeat(80)); + } + + /** + * Log user input before it's sent to the LLM + */ + async beforeLLMRequest( + payload: BeforeLLMRequestPayload, + context: PluginExecutionContext + ): Promise { + this.requestCount++; + + await this.writeLog(''); + await this.writeLog(`[${this.requestCount}] USER REQUEST at ${new Date().toISOString()}`); + await this.writeLog(`Session: ${payload.sessionId || 'unknown'}`); + await this.writeLog(`User: ${context.userId || 'anonymous'}`); + await this.writeLog(`Model: ${context.llmConfig.provider}/${context.llmConfig.model}`); + await this.writeLog('-'.repeat(40)); + await this.writeLog(`Text: ${payload.text}`); + + if (payload.imageData) { + await this.writeLog( + `Image: ${payload.imageData.mimeType} (${payload.imageData.image.length} chars)` + ); + } + + if (payload.fileData) { + await this.writeLog( + `File: ${payload.fileData.filename || 'unknown'} (${payload.fileData.mimeType})` + ); + } + + await this.writeLog('-'.repeat(40)); + + return { ok: true }; + } + + /** + * Log tool calls before execution + */ + async beforeToolCall( + payload: BeforeToolCallPayload, + context: PluginExecutionContext + ): Promise { + await this.writeLog(''); + await this.writeLog(`[${this.requestCount}] TOOL CALL at ${new Date().toISOString()}`); + await this.writeLog(`Tool: ${payload.toolName}`); + await this.writeLog(`Call ID: ${payload.callId || 'unknown'}`); + await this.writeLog(`Arguments: ${JSON.stringify(payload.args, null, 2)}`); + + return { ok: true }; + } + + /** + * Log tool results after execution + */ + async afterToolResult( + payload: AfterToolResultPayload, + context: PluginExecutionContext + ): Promise { + await this.writeLog(''); + await this.writeLog(`[${this.requestCount}] TOOL RESULT at ${new Date().toISOString()}`); + await this.writeLog(`Tool: ${payload.toolName}`); + await this.writeLog(`Call ID: ${payload.callId || 'unknown'}`); + await this.writeLog(`Success: ${payload.success}`); + + const resultStr = + typeof payload.result === 'string' + ? payload.result.substring(0, 500) + (payload.result.length > 500 ? '...' : '') + : JSON.stringify(payload.result, null, 2).substring(0, 500); + + await this.writeLog(`Result: ${resultStr}`); + + return { ok: true }; + } + + /** + * Log assistant response before it's sent to the user + */ + async beforeResponse( + payload: BeforeResponsePayload, + context: PluginExecutionContext + ): Promise { + await this.writeLog(''); + await this.writeLog( + `[${this.requestCount}] ASSISTANT RESPONSE at ${new Date().toISOString()}` + ); + await this.writeLog(`Session: ${payload.sessionId || 'unknown'}`); + await this.writeLog(`Model: ${payload.provider}/${payload.model || 'unknown'}`); + + if (payload.tokenUsage) { + await this.writeLog( + `Tokens: ${payload.tokenUsage.input} input, ${payload.tokenUsage.output} output` + ); + } + + await this.writeLog('-'.repeat(40)); + await this.writeLog(`Content: ${payload.content}`); + + if (payload.reasoning) { + await this.writeLog('-'.repeat(40)); + await this.writeLog(`Reasoning: ${payload.reasoning}`); + } + + await this.writeLog('-'.repeat(40)); + + return { ok: true }; + } + + /** + * Cleanup - close log file handle + */ + async cleanup(): Promise { + await this.writeLog(''); + await this.writeLog('='.repeat(80)); + await this.writeLog(`Request Logger shutting down at ${new Date().toISOString()}`); + await this.writeLog(`Total requests logged: ${this.requestCount}`); + await this.writeLog('='.repeat(80)); + + if (this.logFileHandle) { + await this.logFileHandle.close(); + this.logFileHandle = null; + } + } + + /** + * Helper method to write to log file + */ + private async writeLog(message: string): Promise { + if (this.logFileHandle) { + await this.logFileHandle.write(message + '\n'); + } + } +} + +// Export the plugin class directly for the plugin manager to instantiate +export default RequestLoggerPlugin; diff --git a/dexto/agents/music-agent/README.md b/dexto/agents/music-agent/README.md new file mode 100644 index 00000000..3ea07ac5 --- /dev/null +++ b/dexto/agents/music-agent/README.md @@ -0,0 +1,294 @@ +# Music Creator Agent + +A comprehensive AI agent for music creation, editing, and audio processing using the [Music Creator MCP Server](https://github.com/truffle-ai/mcp-servers/tree/main/src/music). + +> **⚠️ Experimental Status**: This agent is currently in experimental development. The tools have not been extensively tested in production environments and may have limitations or bugs. We're actively seeking feedback and improvements from users. +## 🧪 Experimental Features + +- **Limited Testing**: Tools have been tested in controlled environments but may behave differently with various audio formats, file sizes, or system configurations +- **Active Development**: Features are being refined based on user feedback and real-world usage +- **Feedback Welcome**: We encourage users to report issues, suggest improvements, and share use cases +- **Breaking Changes**: API and tool behavior may change as we improve the implementation + +## Overview + +This agent provides access to professional-grade music production tools through a clean conversational interface. Built with industry-standard libraries like librosa, pydub, and music21, it offers comprehensive audio processing capabilities using the published `truffle-ai-music-creator-mcp` package. + +## Features + +### 🎵 Audio Analysis +- **Tempo Detection**: Automatically detect BPM and beat positions +- **Key Detection**: Identify musical key and mode +- **Spectral Analysis**: Analyze frequency spectrum, MFCC features, and audio characteristics +- **Comprehensive Analysis**: Get detailed audio information including duration, sample rate, and format + +### 🎼 Music Generation +- **Melody Creation**: Generate melodies in any key and scale +- **Chord Progressions**: Create chord progressions using Roman numeral notation +- **Drum Patterns**: Generate drum patterns for rock, jazz, and funk styles +- **MIDI Export**: All generated music exports to MIDI format for further editing + +### 🔊 Audio Processing +- **Format Conversion**: Convert between MP3, WAV, FLAC, OGG, M4A, AIFF, WMA +- **Volume Control**: Adjust audio levels with precise dB control +- **Audio Normalization**: Normalize audio to target levels +- **Audio Trimming**: Cut audio to specific time ranges +- **Audio Effects**: Apply reverb, echo, distortion, and filters + +### 🎚️ Mixing & Arrangement +- **Audio Merging**: Combine multiple audio files with crossfade support +- **Multi-track Mixing**: Mix multiple audio tracks with individual volume control +- **Batch Processing**: Process multiple files with the same operation + +## Quick Start + +### Prerequisites +- **Node.js 18+**: For the Dexto framework +- **Python 3.10+**: Automatically managed by the MCP server +- **FFmpeg**: For audio processing (optional, but recommended) + +### Installation + +1. **Install FFmpeg** (recommended): + ```bash + # macOS + brew install ffmpeg + + # Ubuntu/Debian + sudo apt update && sudo apt install ffmpeg + ``` + +2. **Run the Agent**: + ```bash + # From the project root + dexto --agent agents/music-agent/music-agent.yml + ``` + +That's it! The MCP server will be automatically downloaded and installed via `uvx` on first run. + +## Usage Examples + +### Audio Analysis +``` +"Analyze the tempo and key of my song.mp3" +"What's the BPM of this track?" +"What key is this song in?" +``` + +### Music Generation +``` +"Create a melody in G major at 140 BPM for 15 seconds" +"Create a I-IV-V-I chord progression in D major" +"Create a basic rock drum pattern" +``` + +### Audio Processing +``` +"Convert my song.wav to MP3 format" +"Convert my MIDI melody to WAV format" +"Increase the volume of my vocals by 3dB" +"Normalize my guitar track to -18dB" +"Trim my song from 30 seconds to 2 minutes" +``` + +### Audio Effects +``` +"Add reverb to my guitar with 200ms reverb time" +"Add echo to my vocals with 500ms delay and 0.7 decay" +"Add some distortion to my bass track" +``` + +### Mixing & Playback +``` +"Mix my vocals, guitar, and drums together with the vocals at +3dB" +"Mix a MIDI melody with an MP3 drum loop" +"Create a melody in G major and play it for 5 seconds" +"Play my song.mp3 starting from 30 seconds for 10 seconds" +``` + +## Available Tools + +### Music Generation +- `create_melody` - Generate melodies in any key and scale +- `create_chord_progression` - Create chord progressions using Roman numerals +- `create_drum_pattern` - Generate drum patterns for different styles + +### Audio Analysis +- `analyze_audio` - Comprehensive audio analysis +- `detect_tempo` - Detect BPM and beat positions +- `detect_key` - Identify musical key and mode +- `get_audio_info` - Get detailed audio file information +- `get_midi_info` - Get detailed MIDI file information + +### Audio Processing +- `convert_audio_format` - Convert between audio formats +- `convert_midi_to_audio` - Convert MIDI files to high-quality audio format (WAV, 44.1kHz, 16-bit) +- `adjust_volume` - Adjust audio levels in dB +- `normalize_audio` - Normalize audio to target levels +- `trim_audio` - Cut audio to specific time ranges +- `apply_audio_effect` - Apply reverb, echo, distortion, filters + +### Mixing & Arrangement +- `merge_audio_files` - Combine multiple audio files +- `mix_audio_files` - Mix tracks with individual volume control (supports both audio and MIDI files) + +### Playback +- `play_audio` - Play audio files with optional start time and duration +- `play_midi` - Play MIDI files with optional start time and duration + +### Utility +- `list_available_effects` - List all audio effects +- `list_drum_patterns` - List available drum patterns + +## Supported Formats + +### Audio Formats +- **MP3**: Most common compressed format +- **WAV**: Uncompressed high-quality audio +- **FLAC**: Lossless compressed audio +- **OGG**: Open-source compressed format +- **M4A**: Apple's compressed format +- **AIFF**: Apple's uncompressed format +- **WMA**: Windows Media Audio + +### MIDI Formats +- **MID**: Standard MIDI files +- **MIDI**: Alternative MIDI extension + +## Configuration + +### Agent Configuration +The agent is configured to use the published MCP server: + +```yaml +systemPrompt: | + You are an AI assistant specialized in music creation, editing, and production... + +mcpServers: + music_creator: + type: stdio + command: uvx + args: + - truffle-ai-music-creator-mcp + connectionMode: strict + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY +``` + +### Environment Variables +Set your OpenAI API key: + +```bash +export OPENAI_API_KEY="your-api-key-here" +``` + +Or create a `.env` file in the project root: + +```bash +OPENAI_API_KEY=your-api-key-here +``` + +## Use Cases + +### Music Production +- Create backing tracks and accompaniments +- Generate drum patterns for different genres +- Compose melodies and chord progressions +- Mix and master audio tracks + +### Audio Editing +- Clean up audio recordings +- Normalize volume levels +- Apply professional effects +- Convert between formats + +### Music Analysis +- Analyze existing music for tempo and key +- Extract musical features for machine learning +- Study musical patterns and structures +- Compare different audio files + +### Educational +- Learn about musical theory through generation +- Study different musical styles and patterns +- Experiment with composition techniques +- Understand audio processing concepts + +## MCP Server + +This agent uses the **Music Creator MCP Server**, which is maintained separately at: + +**🔗 [https://github.com/truffle-ai/mcp-servers/tree/main/src/music](https://github.com/truffle-ai/mcp-servers/tree/main/src/music)** + +The MCP server repository provides: +- Complete technical documentation +- Development and contribution guidelines +- Server implementation details +- Advanced configuration options + +## Troubleshooting + +### Common Issues + +#### 1. Server Installation +The MCP server will be automatically installed via `uvx` on first run. No manual setup required. + +#### 2. "FFmpeg not found" warnings +These warnings can be safely ignored. The agent includes fallback methods using librosa and soundfile for audio processing when FFmpeg is not available. + +```bash +# Optional: Install FFmpeg for optimal performance +brew install ffmpeg # macOS +sudo apt install ffmpeg # Ubuntu/Debian +``` + +#### 3. Large Audio Files +Consider trimming or converting to smaller formats for faster processing. + +#### 4. Memory Usage +Monitor system memory during heavy audio operations. + +### Performance Tips + +1. **Large Audio Files**: Consider trimming or converting to smaller formats for faster processing +2. **Memory Usage**: Monitor system memory during heavy audio operations +3. **Batch Processing**: Use batch operations for multiple files to improve efficiency +4. **FFmpeg**: Install FFmpeg for optimal audio processing performance (optional - fallback methods available) + +## Technical Details + +### Dependencies +The MCP server uses industry-standard libraries: +- **librosa**: Audio analysis and music information retrieval +- **pydub**: Audio file manipulation and processing +- **music21**: Music notation and analysis +- **pretty_midi**: MIDI file handling +- **numpy**: Numerical computing +- **scipy**: Scientific computing +- **matplotlib**: Plotting and visualization + +### Architecture +The agent uses a Python-based MCP server that provides: +- Fast audio processing with optimized libraries +- Memory-efficient handling of large audio files +- Thread-safe operations for concurrent processing +- Comprehensive error handling and validation + +### Performance +- Supports audio files up to several hours in length +- Efficient processing of multiple file formats +- Optimized algorithms for real-time analysis +- Minimal memory footprint for batch operations + +## Getting Help + +- **MCP Server Issues**: Report at the [mcp-servers repository](https://github.com/truffle-ai/mcp-servers/issues) +- **Agent Configuration**: Report at the main Dexto repository +- **Feature Requests**: Use the mcp-servers repository for tool-related requests + +## License + +This agent configuration is part of the Dexto AI Agent framework. The MCP server is distributed under the MIT license. \ No newline at end of file diff --git a/dexto/agents/music-agent/music-agent.yml b/dexto/agents/music-agent/music-agent.yml new file mode 100644 index 00000000..fe377417 --- /dev/null +++ b/dexto/agents/music-agent/music-agent.yml @@ -0,0 +1,85 @@ +# Dexto Agent Configuration for Music Creation and Editing MCP Server + +# Optional greeting shown at chat start (UI can consume this) +greeting: "🎵 Hi! I'm your Music Agent. Let's make some beautiful sounds together!" + +systemPrompt: | + You are an AI assistant specialized in music creation, editing, and production. You have access to a comprehensive set of tools for working with audio and music including: + + - **Audio Analysis**: Analyze audio files for tempo, key, BPM, frequency spectrum, and audio characteristics + - **Audio Processing**: Convert formats, adjust volume, normalize, apply effects (reverb, echo, distortion, etc.) + - **Music Generation**: Create melodies, chord progressions, drum patterns, and complete compositions + - **Audio Manipulation**: Trim, cut, splice, loop, and arrange audio segments + - **Effects & Filters**: Apply various audio effects and filters for creative sound design + - **Mixing & Mastering**: Balance levels, apply compression, EQ, and mastering effects + - **File Management**: Organize, convert, and manage audio files in various formats + + When working with music and audio: + 1. Always validate that input audio files exist and are in supported formats + 2. Provide clear feedback about what operations you're performing + 3. Save processed audio with descriptive names and appropriate formats + 4. Include audio information (duration, sample rate, bit depth, format) in your responses + 5. Suggest additional enhancements and creative possibilities when appropriate + 6. Consider musical theory and composition principles in your suggestions + + Supported audio formats: MP3, WAV, FLAC, OGG, M4A, AIFF, WMA + Supported MIDI formats: MID, MIDI + +mcpServers: + music_creator: + type: stdio + command: uvx + args: + - truffle-ai-music-creator-mcp + connectionMode: strict + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 + +# Prompts - music creation examples shown as clickable buttons in WebUI +prompts: + - type: inline + id: create-melody + title: "🎼 Create Melody" + description: "Generate a musical melody" + prompt: "Create a cheerful melody in G major at 140 BPM that lasts 15 seconds." + category: generation + priority: 10 + showInStarters: true + - type: inline + id: create-chords + title: "🎹 Create Chord Progression" + description: "Generate chord progressions" + prompt: "Create a I-IV-V-I chord progression in D major." + category: generation + priority: 9 + showInStarters: true + - type: inline + id: create-drums + title: "🥁 Create Drum Pattern" + description: "Generate drum patterns" + prompt: "Create a basic rock drum pattern at 120 BPM." + category: generation + priority: 8 + showInStarters: true + - type: inline + id: list-effects + title: "🎚️ List Available Effects" + description: "See what audio effects are available" + prompt: "Show me all available audio effects and how to use them." + category: discovery + priority: 7 + showInStarters: true \ No newline at end of file diff --git a/dexto/agents/nano-banana-agent/README.md b/dexto/agents/nano-banana-agent/README.md new file mode 100644 index 00000000..24d78643 --- /dev/null +++ b/dexto/agents/nano-banana-agent/README.md @@ -0,0 +1,200 @@ +# Nano Banana Agent + +A Dexto agent that provides access to Google's **Gemini 2.5 Flash Image** model for image generation and editing through a lean, powerful MCP server. + +## 🎯 What is Gemini 2.5 Flash Image? + +Gemini 2.5 Flash Image is Google's cutting-edge AI model that enables: +- **Near-instantaneous** image generation and editing +- **Object removal** with perfect background preservation +- **Background alteration** while maintaining subject integrity +- **Image fusion** for creative compositions +- **Style modification** with character consistency +- **Visible and invisible watermarks** (SynthID) for digital safety + +## 🚀 Key Features + +### Core Capabilities +- **Image Generation**: Create images from text prompts with various styles and aspect ratios +- **Image Editing**: Modify existing images based on natural language descriptions +- **Object Removal**: Remove unwanted objects while preserving the background +- **Background Changes**: Replace backgrounds while keeping subjects intact +- **Image Fusion**: Combine multiple images into creative compositions +- **Style Transfer**: Apply artistic styles to images + +### Advanced Features +- **Character Consistency**: Maintain facial features and identities across edits +- **Scene Preservation**: Seamless blending with original lighting and composition +- **Multi-Image Processing**: Handle batch operations and complex compositions +- **Safety Features**: Built-in safety filters and provenance signals + +## 🛠️ Setup + +### Prerequisites +- Dexto framework installed +- Google AI API key (Gemini API access) +- Node.js 20.0.0 or higher + +### Installation +1. **Set up environment variables**: + ```bash + export GOOGLE_GENERATIVE_AI_API_KEY="your-google-ai-api-key" + # or + export GEMINI_API_KEY="your-google-ai-api-key" + ``` + +2. **Run the agent** (the MCP server will be automatically downloaded via npx): + ```bash + # From the dexto repository root + npx dexto -a agents/nano-banana-agent/nano-banana-agent.yml + ``` + +The agent configuration uses `npx @truffle-ai/nano-banana-server` to automatically download and run the latest version of the MCP server. + +## 📋 Available Tools + +The agent provides access to 3 essential tools: + +### 1. `generate_image` +Generate new images from text prompts. + +**Example:** +``` +Generate a majestic mountain landscape at sunset in realistic style with 16:9 aspect ratio +``` + +### 2. `process_image` +Process existing images based on detailed instructions. This tool can handle any image editing task including object removal, background changes, style transfer, adding elements, and more. + +**Example:** +``` +Remove the red car in the background from /path/to/photo.jpg +``` + +**Example:** +``` +Change the background of /path/to/portrait.jpg to a beach sunset with palm trees +``` + +**Example:** +``` +Apply Van Gogh painting style with thick brushstrokes to /path/to/photo.jpg +``` + +### 3. `process_multiple_images` +Process multiple images together based on detailed instructions. This tool can combine images, create collages, blend compositions, or perform any multi-image operation. + +**Example:** +``` +Place the person from /path/to/person.jpg into the landscape from /path/to/landscape.jpg as if they were standing there +``` + +## 📤 Response Format + +Successful operations return both image data and metadata: +```json +{ + "content": [ + { + "type": "image", + "data": "base64-encoded-image-data", + "mimeType": "image/png" + }, + { + "type": "text", + "text": "{\n \"output_path\": \"/absolute/path/to/saved/image.png\",\n \"size_bytes\": 12345,\n \"format\": \"image/png\"\n}" + } + ] +} +``` + +## 🎨 Popular Use Cases + +### 1. **Selfie Enhancement** +- Remove blemishes and unwanted objects +- Change backgrounds for professional photos +- Apply artistic filters and styles +- Create figurine effects (Nano Banana's signature feature) + +### 2. **Product Photography** +- Remove backgrounds for clean product shots +- Add or remove objects from scenes +- Apply consistent styling across product images + +### 3. **Creative Compositions** +- Fuse multiple images into unique scenes +- Apply artistic styles to photos +- Create imaginative scenarios from real photos + +### 4. **Content Creation** +- Generate images for social media +- Create variations of existing content +- Apply brand-consistent styling + +## 🔧 Configuration + +### Environment Variables +- `GOOGLE_GENERATIVE_AI_API_KEY` or `GEMINI_API_KEY`: Your Google AI API key (required) + +### Agent Settings +- **LLM Provider**: Google Gemini 2.5 Flash +- **Storage**: In-memory cache with SQLite database +- **Tool Confirmation**: Auto-approve mode for better development experience + +## 📁 Supported Formats + +**Input/Output Formats:** +- JPEG (.jpg, .jpeg) +- PNG (.png) +- WebP (.webp) +- GIF (.gif) + +**File Size Limits:** +- Maximum: 20MB per image +- Recommended: Under 10MB for optimal performance + +## 🎯 Example Interactions + +### Generate a Creative Image +``` +User: "Generate a futuristic cityscape at night with flying cars and neon lights" +Agent: I'll create a futuristic cityscape image for you using Nano Banana's image generation capabilities. +``` + +### Remove Unwanted Objects +``` +User: "Remove the power lines from this photo: /path/to/landscape.jpg" +Agent: I'll remove the power lines from your landscape photo while preserving the natural background. +``` + +### Create Figurine Effect +``` +User: "Transform this selfie into a mini figurine on a desk: /path/to/selfie.jpg" +Agent: I'll create Nano Banana's signature figurine effect, transforming your selfie into a mini figurine displayed on a desk. +``` + +### Change Background +``` +User: "Change the background of this portrait to a professional office setting: /path/to/portrait.jpg" +Agent: I'll replace the background with a professional office setting while keeping you as the main subject. +``` + +## 🔒 Safety & Ethics + +Nano Banana includes built-in safety features: +- **SynthID Watermarks**: Invisible provenance signals +- **Safety Filters**: Content moderation and filtering +- **Character Consistency**: Maintains identity integrity +- **Responsible AI**: Designed to prevent misuse + +## 🤝 Contributing + +We welcome contributions! Please see our [Contributing Guidelines](../../CONTRIBUTING.md) for details. + +## 📄 License + +This project is licensed under the MIT License - see the [LICENSE](../../LICENSE) file for details. + +--- + +**Note**: This agent provides access to Google's Gemini 2.5 Flash Image model through the MCP protocol. The implementation returns both image content (base64-encoded) and text metadata according to MCP specifications, allowing for direct image display in compatible clients. A valid Google AI API key is required and usage is subject to Google's terms of service and usage limits. diff --git a/dexto/agents/nano-banana-agent/nano-banana-agent.yml b/dexto/agents/nano-banana-agent/nano-banana-agent.yml new file mode 100644 index 00000000..33145c9f --- /dev/null +++ b/dexto/agents/nano-banana-agent/nano-banana-agent.yml @@ -0,0 +1,110 @@ +# Dexto Agent Configuration for Nano Banana (Gemini 2.5 Flash Image) MCP Server +# Generated on 2025-01-27T00:00:00.000Z + +# Optional greeting shown at chat start (UI can consume this) +greeting: "🍌 Hi! I'm your Nano Banana Agent. Let's create something amazing together!" + +systemPrompt: | + You are an AI assistant specialized in advanced image generation and editing using Google's Nano Banana (Gemini 2.5 Flash Image) model. You have access to cutting-edge AI tools for: + + - **Image Generation**: Create stunning images from text prompts with various styles and aspect ratios + - **Image Editing**: Modify existing images using natural language descriptions + - **Object Removal**: Remove unwanted objects while perfectly preserving the background + - **Background Changes**: Replace backgrounds seamlessly while keeping subjects intact + - **Image Fusion**: Combine multiple images into creative compositions + - **Style Transfer**: Apply artistic styles to images with character consistency + - **Advanced Features**: Character consistency, scene preservation, and multi-image processing + + When working with images: + 1. Always validate that input images exist and are in supported formats (JPG, PNG, WebP, GIF) + 2. Provide clear feedback about what operations you're performing + 3. Save processed images with descriptive names + 4. Include image information (dimensions, file size, format) in your responses + 5. Suggest additional enhancements and creative possibilities when appropriate + 6. Leverage Nano Banana's signature features like the figurine effect and character consistency + + Key Nano Banana Capabilities: + - **Near-instantaneous** processing with high visual coherence + - **Character consistency** across multiple edits + - **Scene preservation** with seamless background blending + - **Safety features** including SynthID watermarks + - **Multi-image processing** for complex compositions + + Popular use cases: + - Selfie enhancement and creative variations + - Product photography with clean backgrounds + - Artistic style applications + - Object removal from photos + - Background replacement for portraits + - Creating figurine effects (Nano Banana's signature feature) + - Image fusion for creative compositions + + Supported image formats: JPG, JPEG, PNG, WebP, GIF + Maximum file size: 20MB per image + +mcpServers: + nano_banana: + type: stdio + command: npx + args: + - -y + - "@truffle-ai/nano-banana-server" + connectionMode: strict + env: + GEMINI_API_KEY: $GOOGLE_GENERATIVE_AI_API_KEY + timeout: 60000 + +toolConfirmation: + mode: "auto-approve" + allowedToolsStorage: "memory" + +llm: + provider: google + model: gemini-2.5-flash + apiKey: $GOOGLE_GENERATIVE_AI_API_KEY + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 + +# Prompts - image generation and editing examples shown as clickable buttons in WebUI +prompts: + - type: inline + id: generate-landscape + title: "🎨 Generate Landscape" + description: "Create a scenic image from text" + prompt: "Generate a stunning image of a majestic mountain landscape at sunset with vibrant colors and dramatic clouds." + category: generation + priority: 10 + showInStarters: true + - type: inline + id: generate-portrait + title: "👤 Generate Portrait" + description: "Create portrait images" + prompt: "Generate a professional portrait of a person in business attire with a clean studio background." + category: generation + priority: 9 + showInStarters: true + - type: inline + id: generate-abstract + title: "🌀 Generate Abstract Art" + description: "Create abstract artistic images" + prompt: "Generate an abstract art piece with swirling colors and geometric patterns inspired by Kandinsky." + category: generation + priority: 8 + showInStarters: true + - type: inline + id: generate-product + title: "📦 Generate Product Image" + description: "Create product photography" + prompt: "Generate a professional product photo of a sleek modern smartphone on a minimalist white background." + category: generation + priority: 7 + showInStarters: true \ No newline at end of file diff --git a/dexto/agents/podcast-agent/README.md b/dexto/agents/podcast-agent/README.md new file mode 100644 index 00000000..93e6fd11 --- /dev/null +++ b/dexto/agents/podcast-agent/README.md @@ -0,0 +1,168 @@ +# Advanced Podcast Generation Agent + +An AI agent for creating multi-speaker audio content using the Gemini TTS MCP server. + +## Overview + +This agent uses the refactored Gemini TTS MCP server to generate high-quality speech with advanced multi-speaker capabilities. It supports 30 prebuilt voices, natural language tone control, and can generate entire conversations with multiple speakers in a single request. The server now returns audio content that can be played directly in web interfaces. + +## Key Features + +### 🎤 **Native Multi-Speaker Support** +- Generate conversations with multiple speakers in one request +- No need for separate audio files or post-processing +- Natural conversation flow with different voices per speaker + +### 🎵 **30 Prebuilt Voices** +- **Zephyr** - Bright and energetic +- **Puck** - Upbeat and cheerful +- **Charon** - Informative and clear +- **Kore** - Firm and authoritative +- **Fenrir** - Excitable and dynamic +- **Leda** - Youthful and fresh +- **Orus** - Firm and confident +- **Aoede** - Breezy and light +- **Callirrhoe** - Easy-going and relaxed +- **Autonoe** - Bright and optimistic +- **Enceladus** - Breathy and intimate +- **Iapetus** - Clear and articulate +- **Umbriel** - Easy-going and friendly +- **Algieba** - Smooth and polished +- **Despina** - Smooth and elegant +- **Erinome** - Clear and precise +- **Algenib** - Gravelly and distinctive +- **Rasalgethi** - Informative and knowledgeable +- **Laomedeia** - Upbeat and lively +- **Achernar** - Soft and gentle +- **Alnilam** - Firm and steady +- **Schedar** - Even and balanced +- **Gacrux** - Mature and experienced +- **Pulcherrima** - Forward and engaging +- **Achird** - Friendly and warm +- **Zubenelgenubi** - Casual and approachable +- **Vindemiatrix** - Gentle and soothing +- **Sadachbia** - Lively and animated +- **Sadaltager** - Knowledgeable and wise +- **Sulafat** - Warm and inviting + +### 🌐 **WebUI Compatible** +- Returns audio content that can be played directly in web interfaces +- Base64-encoded WAV audio data +- Structured content with both text summaries and audio data + +### 🎭 **Natural Language Tone Control** +- "Say cheerfully: Welcome to our show!" +- "Speak in a formal tone: Welcome to our meeting" +- "Use an excited voice: This is amazing news!" +- "Speak slowly and clearly: This is important information" + +## Setup + +1. **Get API Keys**: + ```bash + export GEMINI_API_KEY="your-gemini-api-key" + export OPENAI_API_KEY="your-openai-api-key" + ``` + +2. **Run the Agent**: + ```bash + dexto -a agents/podcast-agent/podcast-agent.yml + ``` + +The agent will automatically install the Gemini TTS MCP server from npm when needed. + +## Usage Examples + +### Single Speaker +``` +"Generate speech: 'Welcome to our podcast' with voice 'Kore'" +"Create audio: 'Say cheerfully: Have a wonderful day!' with voice 'Puck'" +"Make a formal announcement: 'Speak in a formal tone: Important news today' with voice 'Zephyr'" +``` + +### Multi-Speaker Conversations +``` +"Generate a conversation between Dr. Anya (voice: Kore) and Liam (voice: Puck) about AI" +"Create an interview with host (voice: Zephyr) and guest (voice: Orus) discussing climate change" +"Make a story with narrator (voice: Schedar) and character (voice: Laomedeia)" +"Generate a podcast with three speakers: host (Zephyr), expert (Kore), and interviewer (Puck)" +``` + +### Podcast Types +``` +"Create an educational podcast about AI with clear, professional voices" +"Generate a storytelling podcast with expressive character voices" +"Make a news podcast with authoritative, formal delivery" +"Create an interview with host and guest using different voices" +``` + +## Available Tools + +### **Gemini TTS Tools** +- `generate_speech` - Single-speaker audio generation +- `generate_conversation` - Multi-speaker conversations +- `list_voices` - Browse available voices with characteristics + +### **File Management** +- `list_files` - Browse audio files +- `read_file` - Access file information +- `write_file` - Save generated content +- `delete_file` - Clean up files + +## Voice Selection Guide + +### **Professional Voices** +- **Kore** - Firm, authoritative (great for hosts, experts) +- **Orus** - Firm, professional (business content) +- **Zephyr** - Bright, engaging (news, announcements) +- **Schedar** - Even, balanced (narrators, guides) + +### **Expressive Voices** +- **Puck** - Upbeat, enthusiastic (entertainment, stories) +- **Laomedeia** - Upbeat, energetic (dynamic content) +- **Fenrir** - Excitable, passionate (exciting topics) +- **Achird** - Friendly, warm (casual conversations) + +### **Character Voices** +- **Umbriel** - Easy-going, relaxed (casual hosts) +- **Erinome** - Clear, articulate (educational content) +- **Autonoe** - Bright, optimistic (positive content) +- **Leda** - Youthful, fresh (younger audiences) + +## Multi-Speaker Configuration + +### **Example Speaker Setup** +```json +{ + "speakers": [ + { + "name": "Dr. Anya", + "voice": "Kore", + "characteristics": "Firm, professional" + }, + { + "name": "Liam", + "voice": "Puck", + "characteristics": "Upbeat, enthusiastic" + } + ] +} +``` + +### **Conversation Format** +``` +Dr. Anya: Welcome to our science podcast! +Liam: Thanks for having me, Dr. Anya! +Dr. Anya: Today we're discussing artificial intelligence. +Liam: It's such an exciting field! +``` + +## Advanced Features + +- **Rate Limit Handling**: Graceful fallbacks with dummy audio when API limits are hit +- **Controllable Style**: Accent, pace, and tone control +- **High-Quality Audio**: Studio-grade WAV output +- **Efficient Processing**: Single request for complex conversations +- **Structured Responses**: Both text summaries and audio data in responses + +Simple, powerful, and focused on creating engaging multi-speaker audio content! \ No newline at end of file diff --git a/dexto/agents/podcast-agent/podcast-agent.yml b/dexto/agents/podcast-agent/podcast-agent.yml new file mode 100644 index 00000000..f47c0499 --- /dev/null +++ b/dexto/agents/podcast-agent/podcast-agent.yml @@ -0,0 +1,209 @@ +# Advanced Podcast Generation Agent +# Uses Gemini TTS for multi-speaker audio generation + +mcpServers: + gemini_tts: + type: stdio + command: npx + args: + - -y + - "@truffle-ai/gemini-tts-server" + env: + GEMINI_API_KEY: $GOOGLE_GENERATIVE_AI_API_KEY + timeout: 60000 + connectionMode: strict + + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + +# Optional greeting shown at chat start (UI can consume this) +greeting: "🎙️ Hello! I'm your Podcast Agent. Let's create some amazing audio together!" + +systemPrompt: | + You are an advanced podcast generation agent that creates multi-speaker audio content using Google Gemini TTS. + + ## Your Capabilities + - Generate high-quality speech from text using Gemini TTS + - Create multi-speaker conversations in a single generation + - Use 30 different prebuilt voices with unique characteristics + - Apply natural language tone control (e.g., "Say cheerfully:") + - Save audio files with descriptive names + + ## Gemini TTS MCP Usage + + ### Single Speaker Generation + - Use `generate_speech` to generate single-speaker audio + - Choose from 30 prebuilt voices (Zephyr, Puck, Kore, etc.) + - Apply natural language tone instructions + + ### Multi-Speaker Generation + - Use `generate_conversation` for multi-speaker conversations + - Configure different voices for each speaker + - Generate entire conversations in one request + + ### Voice Discovery + - Use `list_voices` to get a complete list of all available voices with their characteristics + - This tool helps you choose the right voice for different content types + + ### Voice Selection + Available voices with characteristics: + - **Zephyr** - Bright and energetic + - **Puck** - Upbeat and cheerful + - **Charon** - Informative and clear + - **Kore** - Firm and authoritative + - **Fenrir** - Excitable and dynamic + - **Leda** - Youthful and fresh + - **Orus** - Firm and confident + - **Aoede** - Breezy and light + - **Callirrhoe** - Easy-going and relaxed + - **Autonoe** - Bright and optimistic + - **Enceladus** - Breathy and intimate + - **Iapetus** - Clear and articulate + - **Umbriel** - Easy-going and friendly + - **Algieba** - Smooth and polished + - **Despina** - Smooth and elegant + - **Erinome** - Clear and precise + - **Algenib** - Gravelly and distinctive + - **Rasalgethi** - Informative and knowledgeable + - **Laomedeia** - Upbeat and lively + - **Achernar** - Soft and gentle + - **Alnilam** - Firm and steady + - **Schedar** - Even and balanced + - **Gacrux** - Mature and experienced + - **Pulcherrima** - Forward and engaging + - **Achird** - Friendly and warm + - **Zubenelgenubi** - Casual and approachable + - **Vindemiatrix** - Gentle and soothing + - **Sadachbia** - Lively and animated + - **Sadaltager** - Knowledgeable and wise + - **Sulafat** - Warm and inviting + + ### Natural Language Tone Control + You can use natural language to control tone: + - "Say cheerfully: Welcome to our show!" + - "Speak in a formal tone: Welcome to our meeting" + - "Use an excited voice: This is amazing news!" + - "Speak slowly and clearly: This is important information" + + ## Podcast Creation Guidelines + + ### Voice Selection + - Choose appropriate voices for different speakers + - Use consistent voices for recurring characters + - Consider the content type when selecting voices + + ### Content Types + - **Educational**: Clear, professional voices (Kore, Orus, Charon, Rasalgethi) + - **Storytelling**: Expressive voices (Puck, Laomedeia, Fenrir, Sadachbia) + - **News/Current Events**: Authoritative voices (Zephyr, Schedar, Alnilam) + - **Interview**: Different voices for host and guest (Achird, Autonoe, Umbriel) + - **Fiction**: Character voices with distinct personalities (Gacrux, Leda, Algenib) + + ### Multi-Speaker Conversations - IMPORTANT + When users ask for multi-speaker content (like podcast intros, conversations, interviews): + + 1. **Always use `generate_conversation` for conversations with multiple people** + 2. **Format the text with speaker labels**: "Speaker1: [text] Speaker2: [text]" + 3. **Create ONE audio file with ALL speakers**, not separate files per speaker + 4. **REQUIRED: Always define all speakers in the speakers array** - This parameter is mandatory and cannot be omitted + 5. **Never call generate_conversation without the speakers parameter** - it will fail + + **Example for podcast intro:** + ``` + Text: "Alex: Hello everyone, and welcome to our podcast! I'm Alex, your friendly host. Jamie: And I'm Jamie! I'm thrilled to be here with you all today." + Speakers: [ + {"name": "Alex", "voice": "Achird"}, + {"name": "Jamie", "voice": "Autonoe"} + ] + ``` + + **TOOL USAGE RULE**: When using `generate_conversation`, you MUST include both: + - `text`: The conversation with speaker labels + - `speakers`: Array of all speakers with their voice assignments + + **DO NOT** call the tool without the speakers parameter - it will result in an error. + + ### Multi-Speaker Examples + ``` + "Generate a conversation between Dr. Anya (voice: Kore) and Liam (voice: Puck) about AI" + "Create an interview with host (voice: Zephyr) and guest (voice: Orus) discussing climate change" + "Make a story with narrator (voice: Schedar) and character (voice: Laomedeia)" + "Create a podcast intro with Alex (voice: Achird) and Jamie (voice: Autonoe)" + ``` + + ### Single Speaker Examples + ``` + "Generate speech: 'Welcome to our podcast' with voice 'Kore'" + "Create audio: 'Say cheerfully: Have a wonderful day!' with voice 'Puck'" + "Make a formal announcement: 'Speak in a formal tone: Important news today' with voice 'Zephyr'" + ``` + + ### File Management + - Save audio files with descriptive names + - Organize files by episode or content type + - Use appropriate file formats (WAV) + + Always provide clear feedback about what you're creating and explain your voice choices. + + **CRITICAL**: For multi-speaker requests, always generate ONE cohesive audio file with ALL speakers, never split into separate files. + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 + +toolConfirmation: + mode: auto-approve + # timeout: omitted = infinite wait + allowedToolsStorage: memory + +# Prompts - podcast and audio generation examples shown as clickable buttons in WebUI +prompts: + - type: inline + id: create-intro + title: "🎙️ Create Podcast Intro" + description: "Generate a multi-speaker podcast introduction" + prompt: "Create a podcast intro with two hosts, Alex (voice: Achird) and Jamie (voice: Autonoe), welcoming listeners to a tech podcast." + category: podcasting + priority: 10 + showInStarters: true + - type: inline + id: generate-conversation + title: "💬 Generate Conversation" + description: "Create multi-speaker dialogue" + prompt: "Generate a 2-minute conversation between Dr. Anya (voice: Kore) and Liam (voice: Puck) discussing the future of artificial intelligence." + category: conversation + priority: 9 + showInStarters: true + - type: inline + id: list-voices + title: "🔊 Explore Voices" + description: "Browse available voice options" + prompt: "Show me all available voices with their characteristics to help me choose the right ones for my podcast." + category: discovery + priority: 8 + showInStarters: true + - type: inline + id: single-speaker + title: "🗣️ Generate Speech" + description: "Create single-speaker audio" + prompt: "Generate a cheerful welcome announcement using voice Puck saying 'Welcome to our amazing podcast! We're thrilled to have you here today.'" + category: speech + priority: 7 + showInStarters: true \ No newline at end of file diff --git a/dexto/agents/product-analysis-agent/product-analysis-agent.yml b/dexto/agents/product-analysis-agent/product-analysis-agent.yml new file mode 100644 index 00000000..ea50ca36 --- /dev/null +++ b/dexto/agents/product-analysis-agent/product-analysis-agent.yml @@ -0,0 +1,152 @@ +# Product Analysis Agent +# AI agent for product analytics and insights using PostHog +# Uses the official PostHog MCP server to interact with your analytics data + +mcpServers: + posthog: + type: stdio + command: npx + args: + - -y + - mcp-remote@0.1.31 + - https://mcp.posthog.com/sse + - --header + - "Authorization:Bearer $POSTHOG_API_KEY" + timeout: 60000 + connectionMode: strict + +greeting: "Hi! I'm your Product Analysis Agent. I can help you understand your users, track feature adoption, analyze errors, and uncover insights from your PostHog data. What would you like to explore?" + +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a Product Analysis Agent specialized in helping teams understand their product usage, user behavior, and application health through PostHog analytics. You have direct access to PostHog data, enabling data-driven product decisions. + + ## Your Capabilities + + **Project Management:** + - List and view all PostHog projects in your organization + - Access project metadata and configurations + - Navigate between different project contexts + + **Analytics & Insights:** + - Query and analyze user behavior patterns + - Track key metrics like user signups, retention, and engagement + - Analyze funnels and conversion rates + - Identify trends and anomalies in product usage + - Generate insights about user segments + + **Feature Flags:** + - List all active feature flags in a project + - Query feature flag configurations and rollout percentages + - Analyze feature flag impact on user behavior + - Generate code snippets for feature flag implementation + + **Error Tracking:** + - Retrieve recent application errors and exceptions + - Analyze error patterns and frequency + - Access stack traces for debugging + - Identify error trends and affected user segments + + **Annotations:** + - Create timestamped annotations for significant events + - Mark product launches, marketing campaigns, or incidents + - Track how events correlate with metric changes + + ## Analysis Guidelines + + When analyzing product data, follow these best practices: + + 1. **Context First**: Understand what the user is trying to learn before diving into data + 2. **Ask Clarifying Questions**: Confirm time ranges, user segments, and specific metrics + 3. **Provide Actionable Insights**: Don't just report numbers - explain what they mean + 4. **Compare & Contrast**: Show trends over time and compare to benchmarks + 5. **Identify Patterns**: Look for correlations and potential causations + 6. **Highlight Anomalies**: Point out unusual patterns that may need attention + + ## Common Analysis Scenarios + + - **User Growth**: Track signup trends, activation rates, and cohort retention + - **Feature Adoption**: Measure how users engage with new features + - **Funnel Analysis**: Identify where users drop off in key flows + - **Error Impact**: Understand how errors affect user experience + - **Experiment Results**: Analyze A/B test outcomes and feature flag impact + - **User Segmentation**: Compare behavior across different user groups + + ## Interaction Guidelines + + - Present data in clear, digestible formats (summaries, comparisons, trends) + - Explain technical metrics in business terms when appropriate + - Proactively suggest related analyses that might be valuable + - Recommend actions based on the insights discovered + - Be transparent about data limitations or gaps + + ## Privacy & Security + + - Never expose personally identifiable information (PII) + - Aggregate user data appropriately + - Respect data access permissions + - Be mindful of sensitive business metrics in shared contexts + + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local + maxBlobSize: 52428800 + maxTotalSize: 1073741824 + cleanupAfterDays: 30 + +llm: + provider: anthropic + model: claude-sonnet-4-5-20250929 + apiKey: $ANTHROPIC_API_KEY + +toolConfirmation: + mode: manual + allowedToolsStorage: memory + +prompts: + - type: inline + id: user-growth + title: "User Growth" + description: "Analyze user signup and growth trends" + prompt: "Show me user growth trends over the past 30 days. Include signups, activations, and any notable patterns." + category: analytics + priority: 10 + showInStarters: true + - type: inline + id: recent-errors + title: "Recent Errors" + description: "View recent application errors" + prompt: "What are the top recent errors in my application? Show me the most common ones with their frequency." + category: errors + priority: 9 + showInStarters: true + - type: inline + id: feature-flags + title: "Feature Flags" + description: "List and analyze feature flags" + prompt: "List all active feature flags and their current rollout status." + category: features + priority: 8 + showInStarters: true + - type: inline + id: user-behavior + title: "User Behavior" + description: "Analyze how users interact with your product" + prompt: "Help me understand how users are engaging with my product. What are the key behavior patterns?" + category: analytics + priority: 7 + showInStarters: true diff --git a/dexto/agents/product-name-researcher/README.md b/dexto/agents/product-name-researcher/README.md new file mode 100644 index 00000000..344ffc3c --- /dev/null +++ b/dexto/agents/product-name-researcher/README.md @@ -0,0 +1,98 @@ +# Product Name Research Agent + +An AI agent specialized in comprehensive product name research and brand validation. Combines domain availability checking, search engine analysis, developer platform collision detection, and competitive intelligence to provide thorough name validation. + +## 📖 Tutorial + +For a complete walkthrough of building and using this agent, see the [Product Name Scout Agent Tutorial](../../docs/docs/tutorials/product-name-scout-agent.md). + +## Features + +- **Domain Availability**: Check multiple TLD extensions (.com, .io, .app, .dev, etc.) +- **SERP Competition Analysis**: Analyze search engine results for brand competition +- **Autocomplete Intelligence**: Assess name recognition and spelling patterns +- **Developer Platform Collision Detection**: Check GitHub, npm, and PyPI for conflicts +- **Competitive Research**: DuckDuckGo-powered market intelligence +- **Comprehensive Scoring**: Weighted algorithms for brand viability assessment +- **Batch Comparison**: Compare multiple names with detailed scoring breakdown + +## Prerequisites + +Ensure you have the domain checker MCP server available: + +```bash +# Install the domain checker MCP server +uvx truffle-ai-domain-checker-mcp +``` + +## Usage + +### Start the Agent + +```bash +# From the dexto root directory +dexto -a agents/product-name-researcher/product-name-researcher.yml +``` + +### Example Interactions + +**Single Product Name Research:** +``` +User: I want to research the name "CloudSync" for my new file sync product +Agent: [Performs comprehensive research including domain availability, trademark search, social media handles, and competitive analysis] +``` + +**Compare Multiple Names:** +``` +User: Help me choose between "DataFlow", "InfoStream", and "SyncHub" for my data management tool +Agent: [Compares all three names across multiple criteria and provides recommendations] +``` + +**Domain-Focused Research:** +``` +User: Check domain availability for "myawesomeapp" across all major TLDs +Agent: [Uses domain checker to verify availability across .com, .net, .org, .io, .app, etc.] +``` + +## Configuration + +The agent uses: +- **Domain Checker MCP Server**: For domain availability checking +- **DuckDuckGo MCP Server**: For web search and competitive research +- **Product Name Scout MCP Server**: For SERP analysis, autocomplete, and developer collision detection + +## Research Report + +The agent generates comprehensive reports including: + +1. **Domain Availability Summary** + - Available domains with recommendations + - Pricing information where available + - Alternative TLD suggestions + +2. **Trademark Analysis** + - Similar trademarks found + - Risk assessment + - Recommendations for trademark clearance + +3. **Developer Platform Analysis** + - GitHub repository conflicts + - NPM package collisions + - PyPI package conflicts + +4. **Competitive Landscape** + - Existing products with similar names + - Market positioning analysis + - Differentiation opportunities + +5. **Overall Recommendation** + - Scoring across all criteria + - Risk assessment + - Next steps recommendations + +## Tips for Best Results + +- **Be specific about your product**: Include the product category and target market +- **Provide alternatives**: Give multiple name options for comparison +- **Consider your priorities**: Mention if domain availability, trademark clearance, or developer platform conflicts are most important +- **Think internationally**: Consider how the name works in different languages and markets \ No newline at end of file diff --git a/dexto/agents/product-name-researcher/product-name-researcher.yml b/dexto/agents/product-name-researcher/product-name-researcher.yml new file mode 100644 index 00000000..83bf6df3 --- /dev/null +++ b/dexto/agents/product-name-researcher/product-name-researcher.yml @@ -0,0 +1,207 @@ +# Product Name Research Agent Configuration +# Specializes in comprehensive product name validation including domain availability, +# trademark searching, social media handle checking, and competitive analysis + +mcpServers: + # Domain availability checking + domain-checker: + type: stdio + command: uvx + args: + - truffle-ai-domain-checker-mcp + + # Web search for competitive research and trademark checking + duckduckgo: + type: stdio + command: uvx + args: + - duckduckgo-mcp-server + + # Advanced product name research tools (SERP analysis, autocomplete, dev collisions, scoring) + product-name-scout: + type: stdio + command: npx + args: + - "@truffle-ai/product-name-scout-mcp" + +# Optional greeting shown at chat start (UI can consume this) +greeting: "🔍 Hi! I'm your Product Name Researcher. What name shall we explore today?" + +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a specialized Product Name Research Agent focused on helping entrepreneurs, product managers, and marketing teams validate potential product names through comprehensive research. Your expertise combines domain availability checking with competitive landscape analysis and market research. + + ## Your Core Capabilities + + ### 1. Domain Availability Research + - Use the domain-checker tools to verify domain availability across multiple TLDs + - Prioritize .com, .ai, .dev, .org, .io, .app and .tech extensions, in that order. + - Provide recommendations on domain alternatives and pricing considerations + - Check domain variations (with/without hyphens, plurals, abbreviations) + - Compare multiple domain options for product names + + ### 2. Competitive Research & Market Analysis + - Use DuckDuckGo search to research existing products/companies with similar names + - Search for trademark conflicts and existing brand usage + - Analyze competitive landscape and market positioning + - Research industry-specific naming conventions and trends + - Identify potential brand confusion or conflicts + + ### 3. Comprehensive Brand Validation + - Combine domain availability with competitive research + - Assess market saturation for similar product names + - Evaluate naming conflicts across different industries + - Research social media presence and brand mentions + - Provide risk assessments for trademark and competitive conflicts + + ## Research Methodology + + ### For Single Name Research: + 1. **Domain Availability Check**: Use domain-checker to verify availability across key TLDs + 2. **Competitive Analysis**: Search DuckDuckGo for existing companies/products with similar names + 3. **Trademark Research**: Search for trademark conflicts and existing brand usage + 4. **Market Context**: Research industry usage and naming patterns + 5. **Risk Assessment**: Evaluate potential conflicts and brand confusion risks + 6. **Strategic Recommendations**: Provide actionable recommendations based on all findings + + ### For Multiple Name Comparison: + 1. **Batch Domain Analysis**: Check all names across key TLD extensions + 2. **Competitive Research**: Search each name for existing market presence + 3. **Comparison Matrix**: Create comprehensive comparison including domains and competitive landscape + 4. **Scoring & Ranking**: Rank names based on availability, competitive conflicts, and strategic value + 5. **Final Recommendation**: Provide clear recommendation with detailed reasoning + + ## Key Guidelines + + ### Domain Research Best Practices: + - Always start with .com availability as the highest priority + - Check .ai, .dev, .io, .app, .tech for tech/startup products + - Consider .org for non-profits or community-focused products + - Test common misspellings and character variations + - Look for patterns in domain availability that might indicate trademark issues + + ### Competitive Research Best Practices: + - Search for exact name matches and close variations + - Research across different industries and markets + - Look for existing trademarks and brand registrations + - Check for social media presence and brand mentions + - Identify potential customer confusion risks + - Consider international markets and global brand presence + + ### Search Strategy Guidelines: + - Use specific search queries: "[name] company", "[name] trademark", "[name] brand" + - Search for industry-specific usage: "[name] [industry]", "[name] product" + - Look for legal conflicts: "[name] lawsuit", "[name] trademark dispute" + - Check domain parking and cybersquatting patterns + - Research naming trends in the target industry + + ### Interaction Guidelines: + - **Ask targeted questions**: Product category, target market, budget considerations + - **Provide context**: Explain why domain choices matter for business success + - **Be practical**: Focus on actionable domain strategies within user's budget + - **Offer alternatives**: When preferred domains are taken, suggest creative variations + - **Think holistically**: Consider how domain choice impacts overall brand strategy + + ## Available Tools + + ### Domain Checking Tools: + 1. **check_domain(domain)**: Check a single domain's availability + 2. **check_multiple_domains(domains)**: Check multiple domains at once + 3. **check_domain_variations(base_name, extensions)**: Check a name across multiple TLD extensions + + ### Advanced Name Analysis Tools: + 1. **check_brand_serp(name, engine, limit)**: Analyze search engine results for brand competition and searchability + 2. **get_autocomplete(name)**: Get search engine autocomplete suggestions to assess name recognition + 3. **check_dev_collisions(name, platforms)**: Check for existing projects on GitHub, npm, PyPI + 4. **score_name(name, weights, rawSignals)**: Get comprehensive scoring across multiple factors + + ### Research Tools: + 1. **search(query, max_results)**: Search DuckDuckGo for competitive research and market analysis + 2. **get_content(url)**: Extract and analyze content from specific web pages + + ## Tool Usage Strategy: + + **For Comprehensive Research:** + 1. Start with domain availability using domain-checker tools + 2. Analyze search competition: `check_brand_serp(name)` to assess existing brand presence + 3. Check autocomplete patterns: `get_autocomplete(name)` to understand search behavior + 4. Identify developer conflicts: `check_dev_collisions(name)` for tech product names + 5. Search for existing companies/products: `search("[name] company")` + 6. Check for trademarks: `search("[name] trademark")` + 7. Research industry usage: `search("[name] [industry] product")` + 8. Look for legal issues: `search("[name] lawsuit trademark dispute")` + 9. Get comprehensive scoring: `score_name(name)` for overall assessment + 10. Get detailed content from relevant pages using `get_content(url)` + + Use these tools strategically to provide comprehensive product name validation that combines technical availability with market intelligence. + + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +# LLM configuration +llm: + provider: anthropic + model: claude-sonnet-4-5-20250929 + apiKey: $ANTHROPIC_API_KEY + +# Storage configuration +storage: + cache: + type: in-memory + database: + type: sqlite + +# Tool confirmation - auto-approve for seamless domain checking +toolConfirmation: + mode: auto-approve + allowedToolsStorage: memory + +# Prompts - product name research examples shown as clickable buttons in WebUI +prompts: + - type: inline + id: research-name + title: "🔍 Research Product Name" + description: "Comprehensive name validation" + prompt: "I want to research the name 'CloudSync' for my new file sync product. Check domain availability, trademark conflicts, and competitive landscape." + category: research + priority: 10 + showInStarters: true + - type: inline + id: compare-names + title: "⚖️ Compare Multiple Names" + description: "Compare and score name options" + prompt: "Help me choose between 'DataFlow', 'InfoStream', and 'SyncHub' for my data management tool. Compare them across all criteria." + category: comparison + priority: 9 + showInStarters: true + - type: inline + id: check-domains + title: "🌐 Check Domain Availability" + description: "Verify domain availability across TLDs" + prompt: "Check if 'myawesomeapp' is available across .com, .io, .app, and .dev domains." + category: domains + priority: 8 + showInStarters: true + - type: inline + id: trademark-search + title: "™️ Search Trademarks" + description: "Find potential trademark conflicts" + prompt: "Search for existing trademarks similar to 'TechFlow' in the software industry." + category: legal + priority: 7 + showInStarters: true + - type: inline + id: dev-collisions + title: "💻 Check Developer Platforms" + description: "Find conflicts on GitHub, npm, PyPI" + prompt: "Check if 'awesome-toolkit' is available on GitHub, npm, and PyPI." + category: development + priority: 6 + showInStarters: true \ No newline at end of file diff --git a/dexto/agents/sora-video-agent/README.md b/dexto/agents/sora-video-agent/README.md new file mode 100644 index 00000000..1226d8b7 --- /dev/null +++ b/dexto/agents/sora-video-agent/README.md @@ -0,0 +1,122 @@ +# Sora Video Agent + +A Dexto agent specialized in AI video generation using OpenAI's Sora technology. This agent provides a comprehensive interface for creating, managing, and manipulating AI-generated videos. + +## Features + +- **🎬 Video Generation**: Create videos from text prompts with custom settings +- **📊 Progress Monitoring**: Real-time status updates during video generation +- **🎭 Video Remixing**: Create variations and extensions of existing videos +- **📁 File Management**: Automatic download and organization of generated videos +- **🖼️ Reference Support**: Use images or videos as reference for consistent style +- **🗂️ Video Library**: List and manage all your generated videos + +## Capabilities + +### Video Creation +- Generate videos from detailed text prompts +- Customize duration (4s, 8s, 16s, 32s) +- Choose resolution for different platforms: + - **Vertical (9:16)**: 720x1280, 1024x1808 - Perfect for social media + - **Horizontal (16:9)**: 1280x720, 1808x1024 - Great for YouTube + - **Square (1:1)**: 1024x1024 - Ideal for Instagram posts + +### Reference-Based Generation +- Use existing images or videos as style references +- Maintain character consistency across multiple videos +- Apply specific visual styles or aesthetics + +### Video Management +- Monitor generation progress with real-time updates +- List all your videos with status information +- Download completed videos automatically +- Delete unwanted videos to manage storage + +### Creative Workflows +- Create video series with consistent characters +- Generate multiple variations of the same concept +- Extend existing videos with new scenes +- Build comprehensive video libraries + +## Usage Examples + +### Basic Video Generation +```text +Create a video of a cat playing piano in a cozy living room +``` + +### Custom Settings +```text +Generate an 8-second video in 16:9 format showing a sunset over mountains +``` + +### Reference-Based Creation +```text +Create a video using this image as reference, showing the character walking through a forest +``` + +### Video Remixing +```text +Create a remix of video_123 showing the same character but in a different setting +``` + +## Best Practices + +1. **Detailed Prompts**: Be specific about characters, settings, actions, and mood +2. **Platform Optimization**: Choose the right aspect ratio for your target platform +3. **Progressive Creation**: Start with shorter videos for testing, then create longer versions +4. **Style Consistency**: Use reference images/videos for character or style continuity +5. **Library Management**: Regularly organize and clean up your video collection + +## Technical Requirements + +- OpenAI API key with Sora access +- Node.js 18+ for running npx +- Sufficient storage space for video downloads + +## Setup + +### Default Setup (Recommended) + +By default, this agent uses the published `@truffle-ai/sora-video-server` NPM package via `npx`. No additional installation is required - the package will be automatically fetched and run when the agent starts. + +```yaml +mcpServers: + sora_video: + type: stdio + command: npx + args: + - -y + - "@truffle-ai/sora-video-server" + connectionMode: strict + env: + OPENAI_API_KEY: $OPENAI_API_KEY +``` + +### Local Development Setup (Optional) + +If you're developing or modifying the Sora agent locally, you can override the default behavior: + +1. Clone and build the MCP Sora server locally +2. Set the environment variable to point to your local installation: + +```bash +export MCP_SORA_VIDEO_PATH="/path/to/mcp-servers/src/sora-video/dist/index.js" +``` + +3. Update the agent YAML to use the local path instead of npx: + + + +Add the environment variable to your shell profile (`.bashrc`, `.zshrc`, etc.) to persist across sessions. + +## Workflow + +1. **Plan**: Define your video concept and requirements +2. **Generate**: Create the initial video with your prompt +3. **Monitor**: Check progress and wait for completion +4. **Download**: Save the completed video to your device +5. **Iterate**: Create variations or remixes as needed +6. **Organize**: Manage your video library efficiently + +This agent makes AI video generation accessible and efficient, providing all the tools you need to create professional-quality videos with OpenAI's Sora technology. diff --git a/dexto/agents/sora-video-agent/sora-video-agent.yml b/dexto/agents/sora-video-agent/sora-video-agent.yml new file mode 100644 index 00000000..26622495 --- /dev/null +++ b/dexto/agents/sora-video-agent/sora-video-agent.yml @@ -0,0 +1,106 @@ +# Dexto Agent Configuration for Sora Video Generation MCP Server + +# Optional greeting shown at chat start (UI can consume this) +greeting: "🎬 Hello! I'm your Sora Video Agent. Let's create some amazing videos together!" + +systemPrompt: | + You are an AI assistant specialized in video generation using OpenAI's Sora technology. You have access to a comprehensive set of tools for creating, managing, and manipulating AI-generated videos including: + + - **Video Generation**: Create videos from text prompts with custom duration, resolution, and style + - **Reference-Based Creation**: Use images or videos as reference for more precise generation + - **Video Management**: Monitor generation progress, list all videos, and organize your creations + - **Video Remixing**: Create variations and extensions of existing videos with new prompts + - **File Management**: Automatically download and organize generated videos + - **Quality Control**: Delete unwanted videos and manage storage efficiently + + When working with video generation: + 1. Always provide clear, detailed prompts for the best results + 2. Consider the target audience and use case when choosing duration and resolution + 3. Monitor video generation progress and provide status updates + 4. Suggest creative variations and remixes when appropriate + 5. Help users organize and manage their video library + 6. Provide guidance on optimal video settings for different use cases + + **Supported Video Specifications:** + - **Durations**: 4s, 8s, 16s, 32s + - **Resolutions**: 720x1280 (9:16), 1280x720 (16:9), 1024x1024 (1:1), 1024x1808 (9:16 HD), 1808x1024 (16:9 HD) + - **Reference Formats**: JPG, PNG, WebP, MP4, MOV, AVI, WebM + + **Best Practices:** + - Use descriptive, specific prompts for better results + - Consider the aspect ratio for your intended platform (vertical for social media, horizontal for YouTube) + - Start with shorter durations for testing, then create longer versions + - Use reference images/videos for consistent style or character continuity + - Monitor generation progress as it typically takes 1-3 minutes + - Save completed videos promptly to avoid losing access + +mcpServers: + sora_video: + type: stdio + command: npx + args: + - -y + - "@truffle-ai/sora-video-server" + connectionMode: strict + env: + OPENAI_API_KEY: $OPENAI_API_KEY + +toolConfirmation: + mode: "auto-approve" + allowedToolsStorage: "memory" + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +internalResources: + enabled: true + resources: + - type: blob + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 209715200 # 200MB per blob (for HD videos up to 32 seconds) + maxTotalSize: 2147483648 # 2GB total storage + cleanupAfterDays: 30 + +# Prompts - video generation examples shown as clickable buttons in WebUI +prompts: + - type: inline + id: create-video + title: "🎬 Create Video" + description: "Generate video from text prompt" + prompt: "Create an 8-second video of a cat playing piano in a cozy living room with warm lighting." + category: generation + priority: 10 + showInStarters: true + - type: inline + id: nature-video + title: "🌄 Nature Video" + description: "Generate scenic landscape video" + prompt: "Generate an 8-second video in 16:9 format showing a breathtaking sunset over mountains with clouds moving across the sky." + category: nature + priority: 9 + showInStarters: true + - type: inline + id: social-video + title: "📱 Social Media Video" + description: "Create vertical video for social platforms" + prompt: "Create a 4-second vertical video (9:16) showing a product reveal with dramatic lighting." + category: social-media + priority: 8 + showInStarters: true + - type: inline + id: futuristic-video + title: "🚀 Futuristic Scene" + description: "Generate sci-fi themed video" + prompt: "Create a 16-second video showing a futuristic cityscape with flying cars and neon lights at night." + category: sci-fi + priority: 7 + showInStarters: true \ No newline at end of file diff --git a/dexto/agents/talk2pdf-agent/README.md b/dexto/agents/talk2pdf-agent/README.md new file mode 100644 index 00000000..5514d8e2 --- /dev/null +++ b/dexto/agents/talk2pdf-agent/README.md @@ -0,0 +1,166 @@ +# Talk2PDF Agent + +A comprehensive AI agent for parsing and analyzing PDF documents using the [Talk2PDF MCP Server](https://github.com/truffle-ai/mcp-servers/tree/main/src/talk2pdf). + +This agent provides intelligent PDF document processing through a TypeScript-based MCP server that can extract text, metadata, and search for specific content within PDF files. + +## Features + +### 📄 **PDF Parsing & Text Extraction** +- **Full Document Parsing**: Extract complete text content from PDF files +- **Metadata Extraction**: Get document information (title, author, page count, creation date) +- **Format Support**: Handle various PDF versions and structures +- **Error Handling**: Graceful handling of corrupted or protected PDFs + +### 🔍 **Content Search & Analysis** +- **Section Extraction**: Search for and extract specific content sections +- **Intelligent Filtering**: Find content containing specific terms or patterns +- **Context Preservation**: Maintain document structure and formatting +- **Multi-page Support**: Process documents of any length + +### 🧠 **AI-Powered Analysis** +- **Document Summarization**: Generate intelligent summaries of PDF content +- **Key Information Extraction**: Identify and extract important details +- **Question Answering**: Answer questions about document content +- **Content Classification**: Analyze document type and structure + +## Quick Start + +### Prerequisites +- **Node.js 20+**: For the Dexto framework +- **TypeScript**: Automatically managed by the MCP server + +### Installation + +1. **Run the Agent**: + ```bash + # From the dexto project root + dexto --agent agents/talk2pdf-agent/talk2pdf-agent.yml + ``` + +That's it! The MCP server will be automatically downloaded and installed via `npx` on first run. + +## Configuration + +The agent is configured to use the published MCP server: + +```yaml +mcpServers: + talk2pdf: + type: stdio + command: npx + args: + - "@truffle-ai/talk2pdf-mcp" + timeout: 30000 + connectionMode: strict +``` + +## MCP Server + +This agent uses the **Talk2PDF MCP Server**, which is maintained separately at: + +**🔗 [https://github.com/truffle-ai/mcp-servers/tree/main/src/talk2pdf](https://github.com/truffle-ai/mcp-servers/tree/main/src/talk2pdf)** + +The MCP server repository provides: +- Complete technical documentation +- Development and contribution guidelines +- Server implementation details +- Advanced configuration options + +## Available Tools + +### PDF Processing Tools + +#### `parse_pdf` +Extract complete text content and metadata from a PDF file. + +**Parameters:** +- `filePath` (string): Path to the PDF file to parse + +**Returns:** +- Full text content of the document +- Document metadata (title, author, page count, creation date, etc.) +- File information (size, format) + +#### `extract_section` +Search for and extract specific content sections from a PDF. + +**Parameters:** +- `filePath` (string): Path to the PDF file +- `searchTerms` (string): Terms or patterns to search for +- `maxResults` (number, optional): Maximum number of results to return + +**Returns:** +- Matching content sections with context +- Page numbers and locations +- Relevance scoring + +## Supported PDF Features + +- **Standard PDF formats**: PDF 1.4 through 2.0 +- **Text-based PDFs**: Documents with extractable text content +- **Multi-page documents**: No page limit restrictions +- **Metadata support**: Title, author, creation date, modification date +- **Various encodings**: UTF-8, Latin-1, and other standard encodings + +## Example Usage + +### Basic PDF Parsing +``` +"Parse the PDF at /path/to/document.pdf and show me the full content" +"Extract all text and metadata from my research paper" +"What's in this PDF file?" +``` + +### Content Search +``` +"Find all sections about 'machine learning' in the PDF" +"Extract the introduction and conclusion from this document" +"Search for mentions of 'budget' in the financial report" +``` + +### Document Analysis +``` +"Summarize the main points from this PDF" +"What is this document about?" +"Extract the key findings from the research paper" +"List all the recommendations mentioned in the report" +``` + +### Intelligent Q&A +``` +"What are the main conclusions of this study?" +"Who are the authors of this document?" +"When was this document created?" +"How many pages does this PDF have?" +``` + +## Troubleshooting + +### Common Issues + +1. **Server Installation**: The MCP server will be automatically installed via `npx` on first run. No manual setup required. + +2. **PDF Access Issues**: Ensure the PDF file path is correct and the file is readable. Protected or encrypted PDFs may require special handling. + +3. **Memory Issues**: For very large PDFs (100+ pages), processing may take longer. Consider breaking large documents into sections. + +4. **Text Extraction**: If text appears garbled, the PDF may use non-standard encoding or be scanned image-based (OCR not supported). + +### Error Handling + +The agent provides clear error messages for common issues: +- File not found or inaccessible +- Invalid PDF format +- Corrupted PDF files +- Permission-protected documents + +## Getting Help + +- **MCP Server Issues**: Report at the [mcp-servers repository](https://github.com/truffle-ai/mcp-servers/issues) +- **Agent Configuration**: Report at the main Dexto repository +- **Feature Requests**: Use the mcp-servers repository for tool-related requests + +## License + +This project is part of the Dexto AI agent framework. \ No newline at end of file diff --git a/dexto/agents/talk2pdf-agent/talk2pdf-agent.yml b/dexto/agents/talk2pdf-agent/talk2pdf-agent.yml new file mode 100644 index 00000000..858051a6 --- /dev/null +++ b/dexto/agents/talk2pdf-agent/talk2pdf-agent.yml @@ -0,0 +1,57 @@ +# Talk2PDF Agent +# This agent provides natural language access to PDF parsing tools via a custom MCP server + +mcpServers: + talk2pdf: + type: stdio + command: npx + args: + - "@truffle-ai/talk2pdf-mcp" + timeout: 30000 + connectionMode: strict + +# Optional greeting shown at chat start (UI can consume this) +greeting: "📄 Hi! I'm your PDF Agent. Share a document and let's explore it together!" + +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a Talk2PDF Agent. You can parse PDF files, extract their text, metadata, and provide summaries or extract specific sections for LLM consumption. + + ## Your Capabilities + - Parse PDF files and extract all text content and metadata + - Extract specific sections or search for terms within a PDF + - Provide intelligent analysis, summarization, and insights based on the extracted content + - Handle errors gracefully and provide clear feedback + + ## Usage Examples + - "Parse the PDF at /path/to/file.pdf and show me the text." + - "Analyze and summarize the document at /path/to/file.pdf." + - "Extract all content containing 'invoice' from /path/to/file.pdf and explain what you found." + + Always ask for the file path if not provided. If a file is not a PDF or does not exist, inform the user. + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +# Storage configuration +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY \ No newline at end of file diff --git a/dexto/agents/triage-demo/README.md b/dexto/agents/triage-demo/README.md new file mode 100644 index 00000000..cdf94020 --- /dev/null +++ b/dexto/agents/triage-demo/README.md @@ -0,0 +1,337 @@ +# TeamFlow Customer Support Triage Agent System + +This demonstration showcases an intelligent **Customer Support Triage System** built with Dexto agents for **TeamFlow**, a cloud-based project management and team collaboration platform. The system automatically analyzes customer inquiries, routes them to specialized support agents, and provides complete customer support responses. + +## 🏢 About TeamFlow (Demo Business Context) + +TeamFlow is a fictional cloud-based project management platform used for this demonstration. It offers three service tiers: + +- **Basic Plan ($9/user/month)**: Up to 10 team members, 5GB storage, basic features +- **Pro Plan ($19/user/month)**: Up to 100 team members, 100GB storage, advanced integrations (Slack, GitHub, Salesforce) +- **Enterprise Plan ($39/user/month)**: Unlimited users, 1TB storage, SSO, dedicated support + +Key features include project management, team collaboration, time tracking, mobile apps, and a comprehensive API. The platform integrates with popular tools like Slack, GitHub, Salesforce, and Google Workspace. + +This realistic business context allows the agents to provide specific, accurate responses about pricing, features, technical specifications, and policies using the FileContributor system to access comprehensive documentation. + +## 🏗️ Architecture Overview + +``` +Customer Request + ↓ + Triage Agent (Main Coordinator) + ↓ + [Analyzes, Routes & Executes via MCP] + ↓ +┌─────────────────────────────────────────┐ +│ Technical Support │ Billing Agent │ +│ Agent │ │ +├─────────────────────┼──────────────────┤ +│ Product Info │ Escalation │ +│ Agent │ Agent │ +└─────────────────────────────────────────┘ + ↓ + Complete Customer Response +``` + +The triage agent doesn't just route requests - it **executes the specialized agents as MCP servers** and provides complete, integrated customer support responses that combine routing intelligence with expert answers. + +## 🤖 Agent Roles + +### 1. **Triage Agent** (`triage-agent.yml`) +- **Primary Role**: Intelligent routing coordinator AND customer response provider +- **Capabilities**: + - Analyzes requests and categorizes issues + - Routes to specialists via `chat_with_agent` tool calls + - **Executes specialist agents directly through MCP connections** + - **Provides complete customer responses** combining routing + specialist answers +- **Tools**: Filesystem, web research, **chat_with_agent** (connects to all specialists) +- **Tool Confirmation**: Auto-approve mode for seamless delegation + +### 2. **Technical Support Agent** (`technical-support-agent.yml`) +- **Specialization**: Bug fixes, troubleshooting, system issues +- **Tools**: Filesystem, terminal, browser automation +- **Model**: GPT-5 (higher capability for complex technical issues) +- **Connection**: Available as MCP server via stdio + +### 3. **Billing Agent** (`billing-agent.yml`) +- **Specialization**: Payments, subscriptions, financial inquiries +- **Tools**: Browser automation, filesystem for policy docs +- **Model**: GPT-5 Mini (efficient for structured billing processes) +- **Connection**: Available as MCP server via stdio + +### 4. **Product Info Agent** (`product-info-agent.yml`) +- **Specialization**: Features, comparisons, documentation +- **Tools**: Web research (Tavily), filesystem, browser automation +- **Model**: GPT-5 Mini (efficient for information retrieval) +- **Connection**: Available as MCP server via stdio + +### 5. **Escalation Agent** (`escalation-agent.yml`) +- **Specialization**: Complex issues, Enterprise customers, management approval +- **Tools**: Filesystem, web research for compliance/legal info +- **Model**: GPT-5 (higher capability for sensitive issues) +- **Connection**: Available as MCP server via stdio + +## 📚 Business Context Documentation + +Each agent has access to relevant TeamFlow documentation via the FileContributor system: + +### Documentation Files (`docs/` folder) +- **`company-overview.md`**: General company information, plans, SLAs, contact info +- **`technical-documentation.md`**: API docs, system requirements, troubleshooting guides +- **`billing-policies.md`**: Pricing, refund policies, billing procedures, payment methods +- **`product-features.md`**: Feature descriptions, plan comparisons, integrations +- **`escalation-policies.md`**: Escalation procedures, contact information, incident templates + +### Agent-Specific Context +- **Technical Support**: Company overview + technical documentation +- **Billing Agent**: Company overview + billing policies +- **Product Info**: Company overview + product features +- **Escalation**: Company overview + escalation policies +- **Triage Agent**: Company overview for routing context + +## 🚀 Getting Started + +### Quick Start - Integrated Triage System + +The **recommended way** to run the triage system is using the main triage agent, which automatically connects to all specialists: + +```bash +# Run the complete triage system (connects to all specialist agents automatically) +npx dexto --agent agents/triage-demo/triage-agent.yml + +# Test with a customer inquiry +npx dexto --agent agents/triage-demo/triage-agent.yml "I want to upgrade from Basic to Pro but confused about pricing" +``` + +This will: +1. **Auto-connect** to all 4 specialist agents as MCP servers +2. **Analyze** your request and route to the appropriate specialist +3. **Execute** the specialist agent to get the expert answer +4. **Respond** with a complete customer support response + +### Running Individual Agents (Advanced) + +For testing individual specialist agents: + +```bash +# Run specialized agents individually +npx dexto --agent agents/triage-demo/technical-support-agent.yml +npx dexto --agent agents/triage-demo/billing-agent.yml +npx dexto --agent agents/triage-demo/product-info-agent.yml +npx dexto --agent agents/triage-demo/escalation-agent.yml +``` + +### Environment Setup + +Ensure you have the required API keys in your `.env` file: + +```env +OPENAI_API_KEY=your_openai_api_key +TAVILY_API_KEY=your_tavily_api_key # For web research +``` + +## 📋 Example Scenarios + +### Scenario 1: Technical Issue +**Customer Input**: *"My app keeps crashing when I try to upload files. Getting error code 500."* + +**Triage Agent Response**: +- **Analysis**: Technical Issue → Route to Technical Support Agent +- **Execution**: Calls `chat_with_agent` → Technical Support provides debugging steps +- **Customer Response**: *"I've routed your issue to our Technical Support team. Here's how to resolve the 500 error when uploading files: [detailed technical troubleshooting steps from specialist]"* + +### Scenario 2: Billing Question +**Customer Input**: *"I want to upgrade from Basic to Pro but confused about pricing. Will I be charged the full amount?"* + +**Triage Agent Response**: +- **Analysis**: Billing Question → Route to Billing Agent +- **Execution**: Calls `chat_with_agent` → Billing Agent explains prorated pricing +- **Customer Response**: *"I've routed your inquiry to our Billing team. When upgrading from Basic to Pro, you'll only be charged a prorated amount for the remaining billing period - just the difference between plans for the time left in your cycle. The upgrade takes effect immediately."* + +### Scenario 3: Product Information +**Customer Input**: *"What's the difference between your Pro and Enterprise plans? I need API access."* + +**Triage Agent Response**: +- **Analysis**: Product Inquiry → Route to Product Info Agent +- **Execution**: Calls `chat_with_agent` → Product Info Agent provides detailed comparison +- **Customer Response**: *"I've connected you with our Product team. Both Pro and Enterprise plans include full API access. The main differences are: Pro supports up to 100 users with 100GB storage, while Enterprise offers unlimited users, 1TB storage, SSO, and dedicated support. [detailed feature comparison]"* + +### Scenario 4: Escalation Required +**Customer Input**: *"This is my third time contacting support about data privacy concerns. I'm considering legal action."* + +**Triage Agent Response**: +- **Analysis**: Legal/Compliance Issue → Route to Escalation Agent +- **Execution**: Calls `chat_with_agent` → Escalation Agent provides management contact +- **Customer Response**: *"I've escalated your data privacy concerns to our senior support team. Given the legal implications, I'm connecting you directly with our compliance manager Sarah Johnson (sarah.johnson@teamflow.com). She'll address your concerns within 2 hours."* + +## 🔧 Advanced Configuration + +### Current MCP Server Configuration (Automatic) + +The triage agent automatically connects to specialists via stdio MCP servers: + +```yaml +# Current configuration in triage-agent.yml +mcpServers: + technical_support: + type: stdio + command: npx + args: [dexto, --mode, mcp, --agent, agents/triage-demo/technical-support-agent.yml] + + billing_support: + type: stdio + command: npx + args: [dexto, --mode, mcp, --agent, agents/triage-demo/billing-agent.yml] + + # Similar configuration for product_info and escalation agents... +``` + +### Production Configuration (Distributed Servers) + +For production deployment, you would run each specialist as a separate server: + +```yaml +# triage-agent.yml - Production Configuration +mcpServers: + technical_support: + type: sse + url: "http://localhost:3001/mcp" + headers: + Authorization: "Bearer your-auth-token" + + billing_support: + type: sse + url: "http://localhost:3002/mcp" + headers: + Authorization: "Bearer your-auth-token" + + product_info: + type: sse + url: "http://localhost:3003/mcp" + headers: + Authorization: "Bearer your-auth-token" + + escalation: + type: sse + url: "http://localhost:3004/mcp" + headers: + Authorization: "Bearer your-auth-token" +``` + +### Running Distributed Servers + +```bash +# Terminal 1: Technical Support Server +npx dexto --agent agents/triage-demo/technical-support-agent.yml --mode server --port 3001 + +# Terminal 2: Billing Support Server +npx dexto --agent agents/triage-demo/billing-agent.yml --mode server --port 3002 + +# Terminal 3: Product Info Server +npx dexto --agent agents/triage-demo/product-info-agent.yml --mode server --port 3003 + +# Terminal 4: Escalation Server +npx dexto --agent agents/triage-demo/escalation-agent.yml --mode server --port 3004 + +# Terminal 5: Main Triage Coordinator +npx dexto --agent agents/triage-demo/triage-agent.yml --mode server --port 3000 +``` + +## 🎯 Key Features Demonstrated + +### 1. **Intelligent Routing with Execution** +- Natural language analysis to determine issue category +- **Automatic execution** of specialist agents via MCP +- **Complete customer responses** combining routing + expert answers +- Seamless tool confirmation with auto-approve mode + +### 2. **Specialized Expertise Integration** +- Each agent has domain-specific knowledge and tools +- **Real-time coordination** between triage and specialists +- **Unified customer experience** despite multi-agent backend + +### 3. **Scalable MCP Architecture** +- **Stdio connections** for local development and testing +- **SSE connections** for distributed production deployment +- **Tool-based delegation** using `chat_with_agent` + +### 4. **Comprehensive Tool Access** +- Filesystem access for documentation and logging +- Web research capabilities for up-to-date information +- Browser automation for testing and demonstrations +- **Agent-to-agent communication** via MCP tools + +## 🔍 Testing the System + +### Interactive Testing + +1. **Start the complete triage system**: + ```bash + npx dexto --agent agents/triage-demo/triage-agent.yml + ``` + +2. **Test with various customer scenarios** and observe: + - **Routing analysis** (which specialist is chosen) + - **Tool execution** (`chat_with_agent` calls) + - **Complete responses** (routing confirmation + specialist answer) + +### Sample Test Cases + +``` +Test 1: "API returns 401 unauthorized error" +Expected: Technical Support Agent → Complete troubleshooting response + +Test 2: "Cancel my subscription immediately" +Expected: Billing Agent → Complete cancellation process and policy info + +Test 3: "Do you have a mobile app?" +Expected: Product Info Agent → Complete feature details and download links + +Test 4: "Your service caused my business to lose $10,000" +Expected: Escalation Agent → Complete escalation with management contact +``` + +### One-Shot Testing + +```bash +# Test billing scenario +npx dexto --agent agents/triage-demo/triage-agent.yml "I was charged twice this month" + +# Test technical scenario +npx dexto --agent agents/triage-demo/triage-agent.yml "Getting 500 errors on file upload" + +# Test product scenario +npx dexto --agent agents/triage-demo/triage-agent.yml "What integrations do you support?" +``` + +## 🚦 Production Considerations + +### Security +- Implement proper authentication between agents +- Secure API key management +- Customer data privacy controls +- **Tool confirmation policies** for sensitive operations + +### Monitoring +- Log all routing decisions and tool executions +- Track resolution times by agent type +- Monitor escalation patterns +- **Tool usage analytics** for optimization + +### Scaling +- Load balance multiple instances of specialist agents +- Implement request queuing for high volume +- **Distributed MCP server deployment** +- Add more specialized agents as needed (e.g., Sales, Onboarding) + +## 🤝 Contributing + +To extend this triage system: + +1. **Add new specialist agents** by creating new YAML configs +2. **Update triage routing logic** in the main agent's system prompt +3. **Configure new agents as MCP servers** in the triage agent's mcpServers section +4. **Test end-to-end flow** including tool execution and complete responses + +This demonstration showcases the power of **multi-agent coordination with tool execution** using Dexto's MCP integration capabilities! \ No newline at end of file diff --git a/dexto/agents/triage-demo/billing-agent.yml b/dexto/agents/triage-demo/billing-agent.yml new file mode 100644 index 00000000..bc11c539 --- /dev/null +++ b/dexto/agents/triage-demo/billing-agent.yml @@ -0,0 +1,76 @@ +# Billing Agent Configuration +# Specializes in billing, payments, subscriptions, and financial inquiries + +systemPrompt: + contributors: + - id: base-prompt + type: static + priority: 0 + content: | + You are a specialized Billing Agent for TeamFlow, responsible for handling payment issues, subscription management, and financial inquiries. + + Your primary responsibilities: + - Process billing inquiries, payment issues, and subscription changes + - Explain charges, fees, and billing cycles clearly + - Help with payment method updates, refunds, and disputes + - Manage subscription upgrades, downgrades, and cancellations + - Resolve invoice discrepancies and billing errors + + Your approach: + - Always verify customer identity before discussing sensitive billing information + - Be transparent about charges, fees, and policies + - Provide clear explanations of billing cycles and payment schedules + - Handle financial disputes with empathy and professionalism + - Escalate complex billing disputes or refund requests beyond your authority + + Key information to gather: + - Customer account details and billing address + - Specific billing period or invoice number in question + - Payment method and transaction details when relevant + - Desired outcome or resolution + + You have access to comprehensive billing policies, pricing information, and refund procedures for TeamFlow's Basic ($9/user/month), Pro ($19/user/month), and Enterprise ($39/user/month) plans. + + Tools available to you: + - Web browsing for checking payment processor status and policies + - Filesystem access for accessing billing documentation and policies + + Remember: Handle all financial information with strict confidentiality and always follow TeamFlow's billing policies. + + - id: company-overview + type: file + priority: 10 + files: + - "${{dexto.agent_dir}}/docs/company-overview.md" + options: + includeFilenames: true + errorHandling: skip + + - id: billing-policies + type: file + priority: 20 + files: + - "${{dexto.agent_dir}}/docs/billing-policies.md" + options: + includeFilenames: true + errorHandling: skip + +mcpServers: + playwright: + type: stdio + command: npx + args: + - -y + - "@playwright/mcp@latest" + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY \ No newline at end of file diff --git a/dexto/agents/triage-demo/docs/billing-policies.md b/dexto/agents/triage-demo/docs/billing-policies.md new file mode 100644 index 00000000..17869704 --- /dev/null +++ b/dexto/agents/triage-demo/docs/billing-policies.md @@ -0,0 +1,246 @@ +# TeamFlow Billing Policies & Procedures + +## Pricing Plans & Features + +### Basic Plan - $9/user/month +**Billed monthly or annually ($90/user/year, save 17%)** + +**Features Included**: +- Up to 10 team members +- 5GB storage per team +- Core project management (tasks, boards, basic reporting) +- Email support (24-hour response) +- Mobile apps (iOS/Android) +- API access (1,000 requests/hour) +- Basic integrations (Google Calendar, basic file imports) + +**Usage Limits**: +- Maximum 1,000 tasks per workspace +- 50MB maximum file upload size +- Email support only + +### Pro Plan - $19/user/month +**Billed monthly or annually ($190/user/year, save 17%)** + +**Features Included**: +- Up to 100 team members +- 100GB storage per team +- Advanced project management (Gantt charts, custom fields, advanced reporting) +- Priority support (8-hour response, chat support) +- Advanced integrations (Slack, GitHub, Jira, Salesforce) +- API access (10,000 requests/hour) +- Custom workflows and automation +- Time tracking and invoicing +- Advanced security (2FA, audit logs) + +**Usage Limits**: +- Maximum 10,000 tasks per workspace +- 100MB maximum file upload size +- Priority support queue + +### Enterprise Plan - $39/user/month +**Billed annually only ($468/user/year), minimum 25 users** + +**Features Included**: +- Unlimited team members +- 1TB storage per team +- Enterprise security (SSO, SAML, advanced compliance) +- Dedicated customer success manager +- Phone support (4-hour response SLA) +- Unlimited API access +- Custom integrations and white-labeling +- Advanced admin controls +- On-premises deployment option (additional cost) +- 99.95% uptime SLA + +**Usage Limits**: +- Unlimited tasks and projects +- 500MB maximum file upload size +- Dedicated support team + +## Billing Cycles & Payment Processing + +### Billing Dates +- **Monthly Plans**: Charged on the same day each month as initial subscription +- **Annual Plans**: Charged once per year on subscription anniversary +- **Billing Time**: All charges processed at 12:00 AM UTC +- **Failed Payment Retries**: Automatic retries on days 3, 7, and 14 + +### Accepted Payment Methods +- **Credit Cards**: Visa, MasterCard, American Express, Discover +- **Debit Cards**: Visa and MasterCard debit cards +- **Enterprise Options**: Wire transfer, ACH (US only), annual invoicing +- **International**: PayPal for international customers +- **Cryptocurrency**: Not currently accepted + +### Prorated Billing +- **Plan Upgrades**: Immediate access, prorated charge for remaining billing period +- **Plan Downgrades**: Takes effect at next billing cycle, no immediate refund +- **Adding Users**: Prorated charge for new users based on remaining billing period +- **Removing Users**: Credit applied to next invoice + +## Refund Policy + +### Refund Eligibility + +#### Full Refunds (100%) +- **New Subscriptions**: Within 30 days of first payment +- **Service Outages**: If SLA uptime guarantees are not met +- **Billing Errors**: Duplicate charges, incorrect amounts +- **Technical Issues**: If service is unusable and cannot be resolved within 48 hours + +#### Partial Refunds +- **Plan Downgrades**: Credit for unused portion (Enterprise to Pro/Basic) +- **Early Cancellation**: Pro-rated refund for annual plans (minimum 90 days required) +- **User Reduction**: Credit applied to next billing cycle + +#### No Refunds +- **Basic Plan**: No refunds for monthly Basic plans after 30 days +- **Temporary Outages**: Outages under 4 hours (within SLA) +- **User Error**: Data deletion, misconfiguration, or user training issues +- **Third-party Integration Issues**: Problems with Slack, GitHub, etc. + +### Refund Processing Time +- **Credit Cards**: 5-10 business days +- **PayPal**: 24-48 hours +- **Wire Transfer/ACH**: 10-15 business days + +## Common Billing Scenarios + +### 1. Plan Upgrades + +#### Basic to Pro Upgrade +- **Timing**: Immediate access to Pro features +- **Billing**: Prorated charge for Pro plan, credit for unused Basic time +- **Example**: 15 days into Basic monthly cycle → charged $14.50 for remaining Pro time + +#### Pro to Enterprise Upgrade +- **Requirements**: Minimum 25 users, annual billing only +- **Process**: Requires sales team approval for custom Enterprise features +- **Migration**: Dedicated success manager assists with transition + +### 2. Plan Downgrades + +#### Pro to Basic Downgrade +- **Timing**: Takes effect at next billing cycle +- **Data Retention**: 90-day grace period for Pro-only data (advanced reports, etc.) +- **User Limits**: Must reduce team size to 10 users or less before downgrade + +#### Enterprise to Pro Downgrade +- **Notice Period**: 60-day notice required +- **Custom Features**: Loss of SSO, dedicated support, custom integrations +- **Partial Refund**: Available for remaining months of annual contract + +### 3. User Management + +#### Adding Users +- **Process**: Admin adds users in account settings +- **Billing**: Prorated charge for remaining billing period +- **Automatic**: Charges appear on next invoice with detailed breakdown + +#### Removing Users +- **Deactivation**: Admin can deactivate users immediately +- **Billing Impact**: Credit applied to next billing cycle +- **Data Retention**: User data retained for 30 days in case of reactivation + +### 4. Payment Failures + +#### First Failed Payment +- **Action**: Automatic retry in 3 days +- **User Impact**: No service interruption +- **Notification**: Email sent to account admin + +#### Second Failed Payment (Day 7) +- **Action**: Second automatic retry +- **User Impact**: Warning banner in app +- **Notification**: Email and in-app notification + +#### Third Failed Payment (Day 14) +- **Action**: Final automatic retry +- **User Impact**: Account enters "Past Due" status +- **Features**: Read-only access, limited functionality + +#### Account Suspension (Day 21) +- **Action**: Account suspended if payment still fails +- **User Impact**: Complete loss of access +- **Data Retention**: 30-day grace period before data deletion + +### 5. Currency & International Billing + +#### Supported Currencies +- **Primary**: USD (US Dollar) +- **Additional**: EUR (Euro), GBP (British Pound), CAD (Canadian Dollar) +- **Exchange Rates**: Updated daily, charged in customer's local currency when available + +#### International Considerations +- **VAT/Taxes**: Applied automatically based on billing address +- **Payment Methods**: PayPal preferred for international customers +- **Currency Conversion**: Customer's bank may apply additional conversion fees + +## Enterprise Billing + +### Custom Pricing +- **Volume Discounts**: Available for 100+ users +- **Multi-year Agreements**: Additional discounts for 2-3 year contracts +- **Custom Features**: Additional costs for white-labeling, on-premises deployment + +### Invoice Process +- **Net Terms**: 30-day payment terms standard +- **PO Numbers**: Purchase order numbers accepted and included on invoices +- **Multiple Billing Contacts**: Support for separate billing and technical contacts +- **Custom Payment Terms**: Negotiable for large enterprise accounts + +## Billing Support Procedures + +### Customer Identity Verification +Before discussing billing information, verify: +1. **Account Email**: Customer must provide account email address +2. **Last Payment Amount**: Ask for recent payment amount/date +3. **Billing Address**: Verify last 4 digits of ZIP/postal code +4. **Security Question**: Account-specific security question if configured + +### Common Resolution Steps + +#### Duplicate Charges +1. Verify both charges in billing system +2. Check if customer has multiple accounts +3. Process immediate refund for duplicate charge +4. Update payment method if card was charged twice due to processing error + +#### Failed Payment Recovery +1. Verify current payment method on file +2. Check for expired cards or insufficient funds +3. Update payment method if needed +4. Process manual payment if urgently needed +5. Restore account access immediately upon successful payment + +#### Subscription Cancellation +1. Confirm customer intent to cancel +2. Offer plan downgrade as alternative +3. Process cancellation for end of current billing period +4. Provide data export instructions +5. Send confirmation email with final billing details + +### Escalation Triggers + +Escalate to Finance Team when: +- Refund requests over $500 +- Enterprise contract modifications +- Custom pricing negotiations +- Legal or compliance billing questions +- Suspected fraudulent activity +- International tax questions + +### Billing System Access + +#### Internal Tools +- **Billing Dashboard**: Real-time subscription and payment status +- **Payment Processor**: Stripe dashboard for transaction details +- **Invoice System**: Generate and send custom invoices +- **Refund Portal**: Process refunds up to $500 limit + +#### Customer Tools +- **Account Billing Page**: Self-service billing management +- **Invoice Download**: PDF invoices for all past payments +- **Payment Method Update**: Credit card and PayPal management +- **Usage Reports**: Storage and API usage tracking \ No newline at end of file diff --git a/dexto/agents/triage-demo/docs/company-overview.md b/dexto/agents/triage-demo/docs/company-overview.md new file mode 100644 index 00000000..beb0d5d8 --- /dev/null +++ b/dexto/agents/triage-demo/docs/company-overview.md @@ -0,0 +1,94 @@ +# TeamFlow - Company Overview + +## About TeamFlow + +TeamFlow is a leading cloud-based project management and team collaboration platform that helps organizations streamline their workflows, improve team productivity, and deliver projects on time. Founded in 2019, we serve over 50,000 companies worldwide, from small startups to Fortune 500 enterprises. + +## Our Mission + +To empower teams to work more efficiently and collaboratively through intuitive project management tools and seamless integrations with the tools they already use. + +## Key Products & Services + +### TeamFlow Platform +- **Project Management**: Kanban boards, Gantt charts, task management +- **Team Collaboration**: Real-time chat, file sharing, video conferencing integration +- **Time Tracking**: Automated time tracking with detailed reporting +- **Resource Management**: Team capacity planning and workload balancing +- **Analytics & Reporting**: Custom dashboards and project insights + +### TeamFlow API +- RESTful API with comprehensive documentation +- Webhook support for real-time notifications +- Rate limits: 1,000 requests/hour (Basic), 10,000/hour (Pro), unlimited (Enterprise) +- SDKs available for JavaScript, Python, PHP, and Ruby + +### TeamFlow Mobile Apps +- Native iOS and Android applications +- Offline synchronization capabilities +- Push notifications for project updates + +## Service Plans + +### Basic Plan - $9/user/month +- Up to 10 team members +- 5GB storage per team +- Core project management features +- Email support +- API access (1,000 requests/hour) + +### Pro Plan - $19/user/month +- Up to 100 team members +- 100GB storage per team +- Advanced reporting and analytics +- Priority email and chat support +- Advanced integrations (Slack, GitHub, Jira) +- API access (10,000 requests/hour) +- Custom fields and workflows + +### Enterprise Plan - $39/user/month +- Unlimited team members +- 1TB storage per team +- Advanced security (SSO, 2FA) +- Dedicated customer success manager +- Phone support with 4-hour response SLA +- Unlimited API access +- Custom integrations and white-labeling +- Advanced admin controls and audit logs + +## Key Integrations + +- **Communication**: Slack, Microsoft Teams, Discord +- **Development**: GitHub, GitLab, Bitbucket, Jira +- **File Storage**: Google Drive, Dropbox, OneDrive +- **Calendar**: Google Calendar, Outlook, Apple Calendar +- **Time Tracking**: Toggl, Harvest, RescueTime +- **CRM**: Salesforce, HubSpot, Pipedrive + +## Service Level Agreements (SLA) + +### Uptime Commitments +- Basic Plan: 99.5% uptime +- Pro Plan: 99.9% uptime +- Enterprise Plan: 99.95% uptime with dedicated infrastructure + +### Support Response Times +- Basic: Email support within 24 hours +- Pro: Email/chat support within 8 hours +- Enterprise: Phone/email/chat support within 4 hours + +## Security & Compliance + +- SOC 2 Type II certified +- GDPR compliant +- ISO 27001 certified +- Enterprise-grade encryption (AES-256) +- Regular security audits and penetration testing + +## Contact Information + +- **Headquarters**: San Francisco, CA +- **Support Email**: support@teamflow.com +- **Sales Email**: sales@teamflow.com +- **Emergency Escalation**: escalations@teamflow.com +- **Phone**: 1-800-TEAMFLOW (1-800-832-6356) \ No newline at end of file diff --git a/dexto/agents/triage-demo/docs/escalation-policies.md b/dexto/agents/triage-demo/docs/escalation-policies.md new file mode 100644 index 00000000..7ed54f09 --- /dev/null +++ b/dexto/agents/triage-demo/docs/escalation-policies.md @@ -0,0 +1,301 @@ +# TeamFlow Escalation Policies & Procedures + +## Escalation Overview + +The escalation process ensures that complex, sensitive, or high-priority customer issues receive appropriate attention from senior staff and management. This document outlines when to escalate, escalation paths, and procedures for different types of issues. + +## Escalation Criteria + +### Immediate Escalation (Within 15 minutes) + +#### Security & Data Incidents +- Suspected data breach or unauthorized access +- Customer reports potential security vulnerability +- Malicious activity detected on customer accounts +- Data loss or corruption affecting customer data +- Compliance violations (GDPR, HIPAA, SOC 2) + +#### Service Outages +- Platform-wide service disruption +- API downtime affecting multiple customers +- Critical infrastructure failures +- Database connectivity issues +- CDN or hosting provider problems + +#### Legal & Compliance Issues +- Legal threats or litigation mentions +- Regulatory compliance inquiries +- Subpoenas or legal document requests +- Data deletion requests under GDPR "Right to be Forgotten" +- Intellectual property disputes + +### Priority Escalation (Within 1 hour) + +#### Enterprise Customer Issues +- Any issue affecting Enterprise customers +- SLA violations for Enterprise accounts +- Dedicated success manager requests +- Custom integration problems +- White-label deployment issues + +#### Financial Impact +- Billing system errors affecting multiple customers +- Payment processor failures +- Refund requests over $1,000 +- Revenue recognition issues +- Contract modification requests + +#### High-Value Accounts +- Customers with >$50k annual contract value +- Fortune 500 company issues +- Potential churn indicators for major accounts +- Competitive pressures from large customers +- Expansion opportunity discussions + +### Standard Escalation (Within 4 hours) + +#### Technical Issues +- Unresolved technical problems after 24 hours +- Multiple failed resolution attempts +- Customer-reported bugs affecting core functionality +- Integration partner API issues +- Performance degradation reports + +#### Customer Satisfaction +- Formal complaints about service quality +- Requests to speak with management +- Negative feedback about support experience +- Social media mentions requiring response +- Product feature requests from Pro customers + +## Escalation Paths + +### Level 1: First-Line Support +- **Technical Support Agent**: Technical issues, bugs, troubleshooting +- **Billing Agent**: Payment, subscription, pricing questions +- **Product Info Agent**: Features, plans, general information +- **Response Time**: 24 hours (Basic), 8 hours (Pro), 4 hours (Enterprise) + +### Level 2: Senior Support +- **Senior Technical Specialist**: Complex technical issues, integration problems +- **Billing Manager**: Billing disputes, refund approvals, contract changes +- **Product Manager**: Feature requests, product feedback, roadmap questions +- **Response Time**: 4 hours (all plans) + +### Level 3: Management +- **Support Manager**: Service quality issues, team performance, process improvements +- **Engineering Manager**: System outages, security incidents, technical escalations +- **Finance Director**: Large refunds, contract negotiations, revenue issues +- **Response Time**: 2 hours + +### Level 4: Executive +- **VP of Customer Success**: Enterprise customer issues, major account management +- **CTO**: Security breaches, major technical failures, architecture decisions +- **CEO**: Legal issues, major customer relationships, crisis management +- **Response Time**: 1 hour + +## Contact Information + +### Internal Emergency Contacts + +#### 24/7 On-Call Rotation +- **Primary**: +1-415-555-0199 (Support Manager) +- **Secondary**: +1-415-555-0188 (Engineering Manager) +- **Escalation**: +1-415-555-0177 (VP Customer Success) + +#### Email Escalation Lists +- **Security Incidents**: security-incident@teamflow.com +- **Service Outages**: outage-response@teamflow.com +- **Legal Issues**: legal-emergency@teamflow.com +- **Executive Escalation**: executive-escalation@teamflow.com + +#### Slack Channels +- **#support-escalation**: Real-time escalation coordination +- **#security-alerts**: Security incident response +- **#outage-response**: Service disruption coordination +- **#customer-success**: Enterprise customer issues + +### External Emergency Contacts + +#### Legal Counsel +- **Primary**: Johnson & Associates, +1-415-555-0166 +- **After Hours**: Emergency legal hotline, +1-415-555-0155 +- **International**: Global Legal Partners, +44-20-1234-5678 + +#### Public Relations +- **Crisis Communications**: PR Partners Inc., +1-415-555-0144 +- **Social Media Monitoring**: SocialWatch, +1-415-555-0133 + +## Escalation Procedures + +### 1. Security Incident Escalation + +#### Immediate Actions (0-15 minutes) +1. **Secure the Environment**: Isolate affected systems if possible +2. **Notify Security Team**: Email security-incident@teamflow.com +3. **Document Everything**: Start incident log with timeline +4. **Customer Communication**: Acknowledge receipt, avoid details +5. **Activate Incident Response**: Follow security incident playbook + +#### Follow-up Actions (15-60 minutes) +1. **Executive Notification**: Inform CTO and CEO +2. **Legal Review**: Consult with legal counsel if needed +3. **Customer Updates**: Provide status updates every 30 minutes +4. **External Notifications**: Regulatory bodies if required +5. **Media Monitoring**: Watch for public mentions + +### 2. Service Outage Escalation + +#### Immediate Actions (0-15 minutes) +1. **Status Page Update**: Update status.teamflow.com +2. **Engineering Notification**: Page on-call engineer +3. **Customer Communication**: Send service disruption notice +4. **Management Alert**: Notify Support and Engineering Managers +5. **Monitor Social Media**: Watch Twitter and community forums + +#### Follow-up Actions (15-60 minutes) +1. **Root Cause Analysis**: Begin investigating cause +2. **Vendor Communication**: Contact AWS, CloudFlare if needed +3. **Customer Success**: Notify Enterprise customer success managers +4. **Regular Updates**: Status updates every 15 minutes +5. **Post-Incident Review**: Schedule review meeting + +### 3. Legal/Compliance Escalation + +#### Immediate Actions (0-15 minutes) +1. **Preserve Records**: Do not delete any relevant data +2. **Legal Notification**: Email legal-emergency@teamflow.com +3. **Executive Alert**: Notify CEO and CTO immediately +4. **Customer Response**: Acknowledge receipt, request legal review time +5. **Document Control**: Secure all relevant documentation + +#### Follow-up Actions (15-60 minutes) +1. **Legal Counsel**: Conference call with external legal team +2. **Compliance Review**: Check against SOC 2, GDPR requirements +3. **Response Preparation**: Draft official response with legal approval +4. **Internal Communication**: Brief relevant team members +5. **Follow-up Plan**: Establish ongoing communication schedule + +### 4. Enterprise Customer Escalation + +#### Immediate Actions (0-1 hour) +1. **Account Review**: Pull complete customer history and contract +2. **Success Manager**: Notify dedicated customer success manager +3. **Management Alert**: Inform VP of Customer Success +4. **Priority Handling**: Move to front of all queues +5. **Initial Response**: Acknowledge with management involvement + +#### Follow-up Actions (1-4 hours) +1. **Executive Involvement**: Engage appropriate C-level if needed +2. **Solution Planning**: Develop comprehensive resolution plan +3. **Resource Allocation**: Assign dedicated technical resources +4. **Communication Plan**: Establish regular update schedule +5. **Relationship Review**: Assess overall account health + +## Communication Templates + +### Security Incident Notification +``` +Subject: [URGENT] Security Incident - TeamFlow Customer Data + +Priority: Critical +Incident ID: SEC-2024-001 +Reported: [Timestamp] +Affected Customer: [Company Name] +Reported By: [Customer Contact] + +Initial Report: +[Brief description of reported issue] + +Immediate Actions Taken: +- Security team notified +- Incident response activated +- Customer acknowledged +- Environment secured + +Next Steps: +- Investigation in progress +- Legal counsel engaged +- Customer updates every 30 minutes +- Executive team briefed + +Incident Commander: [Name] +Contact: [Phone/Email] +``` + +### Service Outage Alert +``` +Subject: [OUTAGE] TeamFlow Service Disruption + +Priority: High +Outage ID: OUT-2024-001 +Started: [Timestamp] +Affected Services: [List services] +Impact Scope: [Geographic/Feature scope] + +Symptoms: +[Description of user-facing issues] + +Actions Taken: +- Status page updated +- Engineering team engaged +- Root cause investigation started +- Customer notifications sent + +ETA for Resolution: [Time estimate] +Next Update: [Time] + +Incident Commander: [Name] +Contact: [Phone/Email] +``` + +## Escalation Metrics & SLAs + +### Response Time SLAs +- **Security Incidents**: 15 minutes initial response +- **Service Outages**: 15 minutes status update +- **Legal Issues**: 30 minutes acknowledgment +- **Enterprise Customer**: 1 hour initial response +- **Standard Escalation**: 4 hours initial response + +### Resolution Time Targets +- **Critical Issues**: 4 hours +- **High Priority**: 24 hours +- **Standard Escalation**: 72 hours +- **Complex Issues**: 1 week with daily updates + +### Escalation Success Metrics +- **Customer Satisfaction**: >95% for escalated issues +- **First-Call Resolution**: >80% for escalations +- **SLA Compliance**: >99% for response times +- **Escalation Rate**: <5% of total support tickets + +## Training & Certification + +### Escalation Team Requirements +- **Security Awareness**: Annual security training certification +- **Legal Compliance**: GDPR and privacy law training +- **Customer Success**: Enterprise account management training +- **Communication Skills**: Crisis communication workshop +- **Technical Knowledge**: Platform architecture certification + +### Regular Training Sessions +- **Monthly**: Escalation scenario drills +- **Quarterly**: Legal update sessions +- **Bi-annually**: Crisis communication training +- **Annually**: Complete escalation process review + +## Post-Escalation Process + +### Incident Review +1. **Root Cause Analysis**: Complete within 48 hours +2. **Process Review**: Evaluate escalation handling +3. **Customer Follow-up**: Satisfaction survey and feedback +4. **Documentation**: Update knowledge base and procedures +5. **Team Debrief**: Discuss lessons learned and improvements + +### Continuous Improvement +- **Monthly Metrics Review**: Escalation trends and patterns +- **Quarterly Process Updates**: Refine procedures based on feedback +- **Annual Training Updates**: Update training materials and scenarios +- **Customer Feedback Integration**: Incorporate customer suggestions \ No newline at end of file diff --git a/dexto/agents/triage-demo/docs/product-features.md b/dexto/agents/triage-demo/docs/product-features.md new file mode 100644 index 00000000..92d21901 --- /dev/null +++ b/dexto/agents/triage-demo/docs/product-features.md @@ -0,0 +1,253 @@ +# TeamFlow Product Features & Information + +## Core Platform Features + +### Project Management + +#### Task Management +- **Create & Organize**: Unlimited task creation with custom categories and tags +- **Task Dependencies**: Link tasks with predecessor/successor relationships +- **Subtasks**: Break down complex tasks into manageable subtasks (up to 5 levels deep) +- **Priority Levels**: Critical, High, Medium, Low priority with visual indicators +- **Due Dates**: Set deadlines with automatic reminders and escalation +- **Custom Fields**: Add custom properties (text, numbers, dates, dropdowns, checkboxes) +- **Task Templates**: Save and reuse common task structures + +#### Project Views +- **Kanban Boards**: Drag-and-drop task management with customizable columns +- **Gantt Charts**: Timeline view with critical path analysis (Pro/Enterprise) +- **List View**: Traditional task list with sorting and filtering +- **Calendar View**: Tasks and deadlines in calendar format +- **Dashboard View**: Project overview with progress metrics and team activity + +#### Collaboration Tools +- **Comments**: Real-time commenting on tasks and projects with @mentions +- **File Attachments**: Attach files directly to tasks with version control +- **Activity Feed**: Real-time updates on project activity +- **Team Chat**: Built-in messaging with project-specific channels +- **Screen Sharing**: Integrated video conferencing for remote teams (Pro/Enterprise) + +### Time Tracking & Reporting + +#### Time Tracking +- **Manual Entry**: Log time spent on tasks manually +- **Timer Integration**: Start/stop timers directly from tasks +- **Automatic Tracking**: AI-powered time detection based on activity (Pro/Enterprise) +- **Time Approval**: Manager approval workflow for billable hours +- **Offline Tracking**: Mobile app continues tracking without internet connection + +#### Reporting & Analytics +- **Project Reports**: Progress, budget, and timeline analysis +- **Team Performance**: Individual and team productivity metrics +- **Time Reports**: Detailed time tracking and billing reports +- **Custom Dashboards**: Build personalized views with key metrics +- **Export Options**: PDF, Excel, CSV export for all reports + +### Storage & File Management + +#### File Storage +- **Basic Plan**: 5GB total storage per team +- **Pro Plan**: 100GB total storage per team +- **Enterprise Plan**: 1TB total storage per team +- **File Types**: Support for all major file formats +- **Version Control**: Automatic versioning with rollback capability + +#### File Sharing +- **Public Links**: Share files with external stakeholders +- **Permission Control**: Read-only, edit, or full access permissions +- **Expiring Links**: Set expiration dates for shared files +- **Download Tracking**: Monitor who downloads shared files + +## Advanced Features by Plan + +### Basic Plan Features +- Up to 10 team members +- Core project management (tasks, lists, basic boards) +- 5GB file storage +- Mobile apps (iOS/Android) +- Email support +- Basic integrations (Google Calendar, CSV import) +- API access (1,000 requests/hour) + +### Pro Plan Additional Features +- Up to 100 team members +- Advanced project views (Gantt charts, advanced dashboards) +- 100GB file storage +- Custom fields and workflows +- Time tracking and invoicing +- Advanced integrations (Slack, GitHub, Jira, Salesforce) +- Priority support (chat + email) +- API access (10,000 requests/hour) +- Team workload balancing +- Advanced reporting and analytics +- Custom branding (logo, colors) + +### Enterprise Plan Additional Features +- Unlimited team members +- 1TB file storage +- Advanced security (SSO, SAML, 2FA enforcement) +- Dedicated customer success manager +- Phone support with 4-hour SLA +- Unlimited API access +- Custom integrations and white-labeling +- Advanced admin controls and audit logs +- On-premises deployment option +- 99.95% uptime SLA +- Custom workflow automation +- Advanced permissions and role management + +## Integration Ecosystem + +### Communication Platforms + +#### Slack Integration (Pro/Enterprise) +- **Two-way Sync**: Create tasks from Slack messages, get updates in channels +- **Notification Control**: Choose which updates appear in Slack +- **Slash Commands**: Quick task creation with `/teamflow create` command +- **File Sync**: Automatically sync files shared in Slack to project storage + +#### Microsoft Teams (Pro/Enterprise) +- **Tab Integration**: Embed TeamFlow projects directly in Teams channels +- **Bot Commands**: Create and update tasks via Teams bot +- **Calendar Sync**: Sync project deadlines with Teams calendar +- **File Integration**: Access TeamFlow files from Teams file browser + +#### Discord (Pro/Enterprise) +- **Channel Integration**: Link Discord channels to specific projects +- **Role Sync**: Sync Discord roles with TeamFlow permissions +- **Voice Channel Links**: Start Discord voice calls directly from tasks + +### Development Tools + +#### GitHub Integration (Pro/Enterprise) +- **Commit Linking**: Link commits to specific tasks automatically +- **Pull Request Tracking**: Track PR status within TeamFlow tasks +- **Branch Management**: Create branches directly from tasks +- **Release Planning**: Plan releases using TeamFlow milestones + +#### GitLab Integration (Pro/Enterprise) +- **Issue Sync**: Two-way sync between GitLab issues and TeamFlow tasks +- **Pipeline Status**: View CI/CD pipeline status in project dashboard +- **Merge Request Workflow**: Track code reviews within project context + +#### Jira Integration (Pro/Enterprise) +- **Epic/Story Mapping**: Map Jira epics to TeamFlow projects +- **Sprint Planning**: Import Jira sprints as TeamFlow milestones +- **Status Sync**: Automatically update task status based on Jira workflow + +### CRM & Sales Tools + +#### Salesforce Integration (Pro/Enterprise) +- **Lead-to-Project**: Convert Salesforce leads into TeamFlow projects +- **Account Sync**: Link projects to Salesforce accounts +- **Opportunity Tracking**: Track project delivery against sales opportunities + +#### HubSpot Integration (Pro/Enterprise) +- **Contact Sync**: Import HubSpot contacts as team members +- **Deal Pipeline**: Track project delivery stages aligned with deal stages +- **Marketing Campaign Tracking**: Link projects to marketing campaigns + +### File Storage & Productivity + +#### Google Workspace +- **Google Drive**: Direct file access and sync with Google Drive +- **Google Calendar**: Two-way calendar sync for deadlines and meetings +- **Gmail**: Create tasks from emails with Gmail browser extension +- **Google Sheets**: Import/export project data to Google Sheets + +#### Microsoft 365 +- **OneDrive**: Seamless file sync and storage integration +- **Outlook**: Email-to-task conversion and calendar integration +- **Excel**: Advanced reporting with Excel integration +- **SharePoint**: Enterprise file management and compliance + +#### Dropbox +- **File Sync**: Automatic sync of project files with Dropbox +- **Paper Integration**: Convert Dropbox Paper docs to project documentation +- **Team Folders**: Organize project files in shared Dropbox folders + +## Mobile Applications + +### iOS App Features +- **Native Design**: Full iOS design guidelines compliance +- **Offline Support**: Continue working without internet connection +- **Push Notifications**: Real-time updates for mentions, deadlines, and assignments +- **Touch ID/Face ID**: Biometric authentication for security +- **Widgets**: Quick access to tasks and notifications from home screen +- **Apple Watch**: Task completion and notifications on Apple Watch + +### Android App Features +- **Material Design**: Full Android Material Design implementation +- **Battery Optimization**: Efficient background sync and battery usage +- **Quick Settings**: Add tasks and check notifications from notification panel +- **Google Assistant**: Voice commands for task creation and status updates +- **Adaptive Icons**: Support for Android adaptive icon system + +### Cross-Platform Features +- **Real-time Sync**: Instant synchronization across all devices +- **Offline Mode**: Full functionality without internet connection +- **File Download**: Download and view attachments offline +- **Voice Notes**: Record voice memos and attach to tasks +- **Photo Capture**: Take photos and attach directly to tasks + +## API & Developer Tools + +### REST API +- **Full Coverage**: Complete access to all platform features via API +- **Rate Limits**: 1,000/hour (Basic), 10,000/hour (Pro), unlimited (Enterprise) +- **Authentication**: OAuth 2.0 and API key authentication +- **Webhooks**: Real-time event notifications for integrations +- **GraphQL**: Alternative GraphQL endpoint for efficient data fetching + +### SDKs & Libraries +- **JavaScript/Node.js**: Full-featured SDK with TypeScript support +- **Python**: Comprehensive Python library with async support +- **PHP**: Laravel and standard PHP integration library +- **Ruby**: Ruby gem with Rails integration helpers +- **REST Clients**: Postman collection and OpenAPI specification + +### Webhook Events +- **Task Events**: Created, updated, completed, deleted +- **Project Events**: Created, archived, member changes +- **Comment Events**: New comments, mentions, reactions +- **File Events**: Uploaded, updated, shared, deleted +- **Team Events**: Member added, removed, role changes + +## Security & Compliance + +### Data Security +- **Encryption**: AES-256 encryption for all data at rest and in transit +- **HTTPS**: TLS 1.3 for all client connections +- **API Security**: Rate limiting, request signing, and token management +- **Database Security**: Encrypted backups with geographic redundancy + +### Access Control +- **Role-Based Permissions**: Admin, Manager, Member, Viewer roles +- **Project-Level Permissions**: Fine-grained control over project access +- **Two-Factor Authentication**: SMS, authenticator app, hardware keys (Pro/Enterprise) +- **Single Sign-On**: SAML 2.0 and OAuth integration (Enterprise) + +### Compliance Standards +- **SOC 2 Type II**: Annual compliance audits and certification +- **GDPR**: Full compliance with European data protection regulations +- **ISO 27001**: Information security management certification +- **HIPAA**: Healthcare compliance option for Enterprise customers + +## Getting Started Resources + +### Onboarding +- **Interactive Tutorial**: Step-by-step guide for new users +- **Sample Projects**: Pre-built templates for common use cases +- **Video Library**: Comprehensive training videos for all features +- **Webinar Training**: Live training sessions twice weekly + +### Documentation +- **Knowledge Base**: Searchable help articles and guides +- **API Documentation**: Complete developer reference with examples +- **Video Tutorials**: Feature-specific how-to videos +- **Community Forum**: User community for tips and best practices + +### Support Channels +- **Basic Plan**: Email support with 24-hour response +- **Pro Plan**: Email and chat support with 8-hour response +- **Enterprise Plan**: Phone, email, and chat with 4-hour response and dedicated success manager \ No newline at end of file diff --git a/dexto/agents/triage-demo/docs/technical-documentation.md b/dexto/agents/triage-demo/docs/technical-documentation.md new file mode 100644 index 00000000..200b95e9 --- /dev/null +++ b/dexto/agents/triage-demo/docs/technical-documentation.md @@ -0,0 +1,226 @@ +# TeamFlow Technical Documentation + +## System Requirements + +### Web Application +- **Supported Browsers**: Chrome 90+, Firefox 88+, Safari 14+, Edge 90+ +- **Minimum Screen Resolution**: 1024x768 +- **Internet Connection**: Broadband (1 Mbps minimum, 5 Mbps recommended) +- **JavaScript**: Must be enabled + +### Mobile Applications + +#### iOS App +- **Minimum Version**: iOS 13.0 or later +- **Compatible Devices**: iPhone 6s and newer, iPad Air 2 and newer +- **Storage**: 150MB free space required +- **Network**: 3G/4G/5G or Wi-Fi connection + +#### Android App +- **Minimum Version**: Android 8.0 (API level 26) +- **RAM**: 2GB minimum, 4GB recommended +- **Storage**: 200MB free space required +- **Network**: 3G/4G/5G or Wi-Fi connection + +## API Documentation + +### Authentication +``` +Authorization: Bearer +Content-Type: application/json +``` + +### Base URL +- **Production**: `https://api.teamflow.com/v1` +- **Sandbox**: `https://sandbox-api.teamflow.com/v1` + +### Rate Limits +- **Basic Plan**: 1,000 requests/hour +- **Pro Plan**: 10,000 requests/hour +- **Enterprise Plan**: Unlimited +- **Rate Limit Headers**: + - `X-RateLimit-Limit`: Total requests allowed + - `X-RateLimit-Remaining`: Requests remaining in current window + - `X-RateLimit-Reset`: Unix timestamp when window resets + +### Common Error Codes +- **400**: Bad Request - Invalid parameters or request format +- **401**: Unauthorized - Invalid or missing API token +- **403**: Forbidden - Insufficient permissions +- **404**: Not Found - Resource doesn't exist +- **422**: Unprocessable Entity - Validation errors +- **429**: Too Many Requests - Rate limit exceeded +- **500**: Internal Server Error - Contact support +- **502/503**: Service Unavailable - Temporary outage + +## Common Technical Issues & Solutions + +### 1. Login and Authentication Issues + +#### "Invalid credentials" error +**Symptoms**: User cannot log in, receives "Invalid email or password" message +**Common Causes**: +- Incorrect email/password combination +- Account locked due to multiple failed attempts +- Browser caching old session data + +**Solutions**: +1. Verify email address (check for typos, extra spaces) +2. Try password reset flow +3. Clear browser cookies and cache +4. Try incognito/private browsing mode +5. Check if account is locked (wait 15 minutes or contact support) + +#### Two-Factor Authentication (2FA) issues +**Symptoms**: 2FA code not working or not received +**Solutions**: +1. Ensure device clock is synchronized +2. Try generating a new code (codes expire every 30 seconds) +3. Check authenticator app is configured correctly +4. Use backup codes if available +5. Contact support to reset 2FA if backup codes exhausted + +### 2. Performance Issues + +#### Slow loading pages +**Symptoms**: Pages take >10 seconds to load, timeouts +**Troubleshooting Steps**: +1. Check internet connection speed (minimum 1 Mbps required) +2. Test on different networks (mobile data vs. Wi-Fi) +3. Clear browser cache and cookies +4. Disable browser extensions temporarily +5. Try different browser +6. Check TeamFlow status page: status.teamflow.com + +#### Mobile app crashes +**Symptoms**: App closes unexpectedly, freezes during use +**Solutions**: +1. Force close and restart the app +2. Restart device +3. Update app to latest version +4. Clear app cache (Android) or offload/reinstall app (iOS) +5. Check available storage space (minimum 500MB recommended) +6. Report crash with device logs + +### 3. API Integration Issues + +#### 401 Unauthorized errors +**Diagnostic Steps**: +1. Verify API token is correct and not expired +2. Check token has required permissions +3. Ensure proper Authorization header format +4. Test with different API endpoints + +#### 429 Rate limit exceeded +**Solutions**: +1. Implement exponential backoff in API calls +2. Check current rate limit status in response headers +3. Consider upgrading plan for higher limits +4. Cache responses when possible to reduce API calls + +#### Webhook delivery failures +**Common Issues**: +- Endpoint URL not accessible from internet +- SSL certificate issues +- Timeout (webhook endpoint must respond within 10 seconds) +- Incorrect response status (must return 2xx status code) + +### 4. File Upload Issues + +#### "File too large" errors +**File Size Limits**: +- Basic Plan: 25MB per file +- Pro Plan: 100MB per file +- Enterprise Plan: 500MB per file + +**Solutions**: +1. Compress files using zip/rar +2. Use cloud storage links for large files +3. Split large files into smaller chunks +4. Consider plan upgrade for larger limits + +#### Unsupported file formats +**Supported Formats**: +- Images: JPG, PNG, GIF, SVG, WebP +- Documents: PDF, DOC, DOCX, XLS, XLSX, PPT, PPTX +- Archives: ZIP, RAR, 7Z +- Text: TXT, CSV, MD +- Code: JS, HTML, CSS, JSON, XML + +### 5. Integration Problems + +#### Slack integration not working +**Setup Requirements**: +1. Slack workspace admin permissions +2. TeamFlow Pro or Enterprise plan +3. Proper webhook configuration + +**Troubleshooting**: +1. Verify Slack workspace URL is correct +2. Check webhook permissions in Slack admin +3. Test with simple message first +4. Ensure both apps are updated to latest versions + +#### GitHub integration issues +**Common Problems**: +- Repository access permissions +- Webhook authentication failures +- Branch protection rules blocking commits + +**Solutions**: +1. Verify GitHub personal access token has correct scopes +2. Check repository permissions for TeamFlow app +3. Review webhook logs in GitHub settings +4. Test with public repository first + +## Browser-Specific Issues + +### Chrome +- **File download issues**: Check download settings and blocked downloads +- **Extension conflicts**: Disable ad blockers and privacy extensions temporarily + +### Safari +- **Cookie issues**: Enable cross-site tracking prevention exceptions +- **Local storage**: Ensure not in private browsing mode + +### Firefox +- **Security settings**: Adjust strict enhanced tracking protection +- **Add-on conflicts**: Test in safe mode + +## Server Infrastructure + +### Data Centers +- **Primary**: AWS US-West-2 (Oregon) +- **Secondary**: AWS EU-West-1 (Ireland) +- **CDN**: CloudFlare global network + +### Maintenance Windows +- **Scheduled Maintenance**: Sundays 2:00-4:00 AM PST +- **Emergency Maintenance**: As needed with 30-minute notice +- **Status Updates**: status.teamflow.com and @TeamFlowStatus on Twitter + +## Escalation Criteria + +Escalate to Level 2 Support when: +- Data loss or corruption suspected +- Security breach indicators +- API downtime affecting multiple customers +- Integration partner (Slack, GitHub, etc.) reporting issues +- Customer reports SLA violations +- Enterprise customer experiencing any service disruption + +## Diagnostic Tools + +### Browser Developer Tools +1. **Console Errors**: Check for JavaScript errors (F12 → Console) +2. **Network Tab**: Monitor failed requests and response times +3. **Application Tab**: Check local storage and cookies + +### API Testing +- Use Postman or curl to test API endpoints +- Check response headers for rate limit information +- Verify request format matches API documentation + +### Mobile Debugging +- iOS: Connect device to Xcode for detailed crash logs +- Android: Enable developer options and use ADB logcat \ No newline at end of file diff --git a/dexto/agents/triage-demo/escalation-agent.yml b/dexto/agents/triage-demo/escalation-agent.yml new file mode 100644 index 00000000..07dbc875 --- /dev/null +++ b/dexto/agents/triage-demo/escalation-agent.yml @@ -0,0 +1,82 @@ +# Escalation Agent Configuration +# Handles complex issues requiring human intervention or management approval + +systemPrompt: + contributors: + - id: base-prompt + type: static + priority: 0 + content: | + You are a specialized Escalation Agent for TeamFlow, responsible for handling complex, sensitive, or high-priority issues that require management intervention. + + Your primary responsibilities: + - Manage escalated cases from other support agents + - Handle Enterprise customer issues and urgent requests + - Process complaints, disputes, and sensitive matters + - Coordinate with management and specialized teams + - Document incident reports and follow-up actions + + Your approach: + - Take detailed notes of the issue history and previous resolution attempts + - Gather all relevant context and supporting documentation + - Assess urgency level and potential business impact + - Communicate clearly with both customers and internal teams + - Follow up promptly on all escalated matters + + Types of issues you handle: + - Unresolved technical issues after multiple attempts + - Billing disputes requiring management approval + - Enterprise customer feature requests and contract issues + - Security incidents and data privacy concerns + - Legal or compliance-related inquiries (GDPR, SOC 2, etc.) + - Customer complaints about service quality + + You have access to comprehensive escalation policies, procedures, and contact information for TeamFlow's management team, including security incident protocols and legal compliance procedures. + + Tools available to you: + - Email and communication tools for coordinating with teams + - Filesystem access for documentation and case management + - Web research for regulatory and legal information + + Remember: Maintain professionalism, document everything thoroughly, and ensure proper follow-up on all escalated cases according to TeamFlow's escalation policies. + + - id: company-overview + type: file + priority: 10 + files: + - "${{dexto.agent_dir}}/docs/company-overview.md" + options: + includeFilenames: true + errorHandling: skip + + - id: escalation-policies + type: file + priority: 20 + files: + - "${{dexto.agent_dir}}/docs/escalation-policies.md" + options: + includeFilenames: true + errorHandling: skip + +mcpServers: + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + tavily: + type: stdio + command: npx + args: + - -y + - tavily-mcp@0.1.3 + env: + TAVILY_API_KEY: $TAVILY_API_KEY + connectionMode: lenient + +llm: + provider: openai + model: gpt-5 + apiKey: $OPENAI_API_KEY \ No newline at end of file diff --git a/dexto/agents/triage-demo/product-info-agent.yml b/dexto/agents/triage-demo/product-info-agent.yml new file mode 100644 index 00000000..998b383d --- /dev/null +++ b/dexto/agents/triage-demo/product-info-agent.yml @@ -0,0 +1,85 @@ +# Product Information Agent Configuration +# Specializes in product features, comparisons, and general information + +systemPrompt: + contributors: + - id: base-prompt + type: static + priority: 0 + content: | + You are a specialized Product Information Agent for TeamFlow, with comprehensive knowledge about our project management and team collaboration platform. + + Your primary responsibilities: + - Answer questions about TeamFlow's features, capabilities, and specifications + - Provide product comparisons and recommendations between Basic, Pro, and Enterprise plans + - Explain how to use specific features and functionalities + - Share information about integrations and API capabilities + - Guide users to appropriate documentation and resources + + Your approach: + - Provide accurate, up-to-date product information + - Use clear, non-technical language when explaining complex features + - Offer relevant examples and use cases + - Suggest the best product tier or plan for user needs + - Direct users to detailed documentation or demos when helpful + + Key information to gather: + - User's specific use case or requirements + - Current product tier or plan (if applicable) + - Specific features or functionality they're asking about + - Their technical expertise level + + You have access to comprehensive product documentation covering all TeamFlow features, integrations (Slack, GitHub, Salesforce, etc.), mobile apps, API capabilities, and plan comparisons. + + Tools available to you: + - Web research for latest product information and competitor analysis + - Filesystem access to read product documentation and specs + + Remember: Always provide accurate information about TeamFlow and acknowledge when you need to research or verify details. + + - id: company-overview + type: file + priority: 10 + files: + - "${{dexto.agent_dir}}/docs/company-overview.md" + options: + includeFilenames: true + errorHandling: skip + + - id: product-features + type: file + priority: 20 + files: + - "${{dexto.agent_dir}}/docs/product-features.md" + options: + includeFilenames: true + errorHandling: skip + +mcpServers: + tavily: + type: stdio + command: npx + args: + - -y + - tavily-mcp@0.1.3 + env: + TAVILY_API_KEY: $TAVILY_API_KEY + connectionMode: lenient + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + playwright: + type: stdio + command: npx + args: + - -y + - "@playwright/mcp@latest" + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY \ No newline at end of file diff --git a/dexto/agents/triage-demo/technical-support-agent.yml b/dexto/agents/triage-demo/technical-support-agent.yml new file mode 100644 index 00000000..70cd2409 --- /dev/null +++ b/dexto/agents/triage-demo/technical-support-agent.yml @@ -0,0 +1,70 @@ +# Technical Support Agent Configuration +# Specializes in technical issues, troubleshooting, and bug reports + +systemPrompt: + contributors: + - id: base-prompt + type: static + priority: 0 + content: | + You are a specialized Technical Support Agent for TeamFlow, a cloud-based project management and team collaboration platform. + + Your primary responsibilities: + - Diagnose and resolve technical problems, bugs, and system issues + - Provide step-by-step troubleshooting guidance + - Analyze error logs, system configurations, and performance issues + - Escalate complex technical issues when necessary + - Document common issues and their solutions + + Your approach: + - Always ask for specific details: error messages, system info, steps to reproduce + - Provide clear, step-by-step instructions + - Verify solutions work before considering the issue resolved + - Be patient and explain technical concepts in simple terms + - When unsure, ask clarifying questions or escalate to senior technical support + + Tools available to you: + - Filesystem access for log analysis and configuration checks + - Terminal access for system diagnostics and troubleshooting commands + + You have access to comprehensive technical documentation and troubleshooting guides for TeamFlow's platform, API, mobile apps, and integrations. Use this knowledge to provide accurate, specific solutions. + + Remember: Your goal is to resolve technical issues efficiently while educating the user about TeamFlow's features and capabilities. + + - id: company-overview + type: file + priority: 10 + files: + - "${{dexto.agent_dir}}/docs/company-overview.md" + options: + includeFilenames: true + errorHandling: skip + + - id: technical-docs + type: file + priority: 20 + files: + - "${{dexto.agent_dir}}/docs/technical-documentation.md" + options: + includeFilenames: true + errorHandling: skip + +mcpServers: + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + playwright: + type: stdio + command: npx + args: + - -y + - "@playwright/mcp@latest" + +llm: + provider: openai + model: gpt-5 + apiKey: $OPENAI_API_KEY \ No newline at end of file diff --git a/dexto/agents/triage-demo/test-scenarios.md b/dexto/agents/triage-demo/test-scenarios.md new file mode 100644 index 00000000..076aac87 --- /dev/null +++ b/dexto/agents/triage-demo/test-scenarios.md @@ -0,0 +1,209 @@ +# TeamFlow Triage Agent Test Scenarios + +Use these realistic TeamFlow customer support scenarios to test the triage agent's **complete customer support workflow**. The triage agent will analyze each request, route to the appropriate specialist agent, execute the specialist via MCP tools, and provide complete customer responses. + +## 🔧 Technical Support Scenarios + +### Scenario T1: API Integration Issue +``` +Hi, I'm trying to integrate the TeamFlow API with our system but I keep getting a 401 unauthorized error even though I'm using the correct API key. I've checked the documentation but can't figure out what's wrong. Our rate limit should be 10,000/hour on our Pro plan. Can you help? +``` +**Expected Route**: Technical Support Agent +**Expected Response**: Complete troubleshooting guide including API key validation steps, common 401 causes, rate limit verification, and Pro plan API specifications +**Tool Execution**: `chat_with_agent` → Technical Support provides detailed debugging steps + +### Scenario T2: App Crash +``` +My TeamFlow mobile app crashes every time I try to export project data. It worked fine last week but now it just freezes and closes. I'm using iPhone 15 with iOS 17.2. This is really urgent as I need this data for a client presentation tomorrow. +``` +**Expected Route**: Technical Support Agent +**Expected Response**: Complete crash resolution including device-specific troubleshooting, export alternatives, and immediate workarounds for urgent timeline +**Tool Execution**: `chat_with_agent` → Technical Support provides iOS-specific fixes and emergency data export options + +### Scenario T3: Performance Issue +``` +Your web dashboard has been extremely slow for the past 3 days. Pages take 30+ seconds to load and sometimes timeout completely. My internet connection is fine and other websites work normally. +``` +**Expected Route**: Technical Support Agent +**Expected Response**: Complete performance troubleshooting including browser optimization, cache clearing, system status check, and escalation to infrastructure team if needed +**Tool Execution**: `chat_with_agent` → Technical Support provides systematic performance diagnosis steps + +## 💳 Billing Support Scenarios + +### Scenario B1: Double Charge +``` +I just checked my credit card statement and I was charged twice for this month's subscription - once on the 1st for $49.99 and again on the 3rd for the same amount. I need the duplicate charge refunded immediately. +``` +**Expected Route**: Billing Agent +**Expected Response**: Complete refund process including charge verification, refund timeline (3-5 business days), and account credit options for immediate resolution +**Tool Execution**: `chat_with_agent` → Billing Agent provides specific refund procedures and account investigation steps + +### Scenario B2: Subscription Management +``` +I want to upgrade from the Basic plan to Pro plan but I'm confused about the pricing. Will I be charged the full Pro amount or just the difference? Also, when would the upgrade take effect? +``` +**Expected Route**: Billing Agent +**Expected Response**: Complete upgrade explanation including prorated billing calculation, immediate feature access, next billing cycle details, and upgrade procedure +**Tool Execution**: `chat_with_agent` → Billing Agent provides detailed prorated pricing explanation and upgrade process + +### Scenario B3: Payment Failure +``` +My payment failed this morning and now my account is suspended. I updated my credit card last week so I'm not sure why it didn't work. How can I get my account reactivated quickly? +``` +**Expected Route**: Billing Agent +**Expected Response**: Complete reactivation process including payment method verification, retry options, account status restoration timeline, and prevention steps +**Tool Execution**: `chat_with_agent` → Billing Agent provides immediate account reactivation steps and payment troubleshooting + +## 📖 Product Information Scenarios + +### Scenario P1: Feature Comparison +``` +What's the difference between TeamFlow's Pro and Enterprise plans? I specifically need to know about API rate limits, user management features, and data export capabilities. We're a team of 25 people and currently on the Basic plan. +``` +**Expected Route**: Product Info Agent +**Expected Response**: Complete plan comparison including detailed feature matrix, specific API limits (Pro: 10K/hour, Enterprise: 100K/hour), user management differences, and upgrade recommendation for 25-person team +**Tool Execution**: `chat_with_agent` → Product Info Agent provides comprehensive plan comparison with team-size specific recommendations + +### Scenario P2: How-To Question +``` +How do I set up automated reports to be sent to my team every Monday? I see the reporting feature but can't figure out how to schedule them. Is this available in my current plan? +``` +**Expected Route**: Product Info Agent +**Expected Response**: Complete setup guide including step-by-step report scheduling instructions, plan feature verification, and links to relevant documentation +**Tool Execution**: `chat_with_agent` → Product Info Agent provides detailed automated reporting setup walkthrough + +### Scenario P3: Integration Capabilities +``` +Does TeamFlow integrate with Salesforce and Slack? I need to sync customer project data and get notifications in our Slack channels. What's the setup process like and are there any limitations I should know about? We're on the Pro plan. +``` +**Expected Route**: Product Info Agent +**Expected Response**: Complete integration overview including supported Salesforce/Slack features, Pro plan limitations, setup documentation links, and configuration best practices +**Tool Execution**: `chat_with_agent` → Product Info Agent provides comprehensive integration capabilities and setup guidance + +## 🚨 Escalation Scenarios + +### Scenario E1: Legal Threat +``` +This is my fourth email about data privacy violations. Your service exposed my customer data to unauthorized parties and I'm considering legal action. I need to speak with a manager immediately about this data breach. +``` +**Expected Route**: Escalation Agent +**Expected Response**: Complete escalation including immediate management contact information, legal/compliance team connection, incident escalation procedure, and 2-hour response commitment +**Tool Execution**: `chat_with_agent` → Escalation Agent provides senior management contact and legal compliance escalation process + +### Scenario E2: Business Impact +``` +Your system outage yesterday caused my e-commerce site to be down for 6 hours during Black Friday. This resulted in approximately $50,000 in lost sales. I need compensation for this business interruption and want to discuss SLA violations. +``` +**Expected Route**: Escalation Agent +**Expected Response**: Complete business impact assessment including SLA review, compensation evaluation process, senior account manager contact, and formal incident investigation +**Tool Execution**: `chat_with_agent` → Escalation Agent provides business impact claim process and executive contact information + +### Scenario E3: Service Quality Complaint +``` +I've been a customer for 3 years and the service quality has declined dramatically. Multiple support tickets have been ignored, features are constantly broken, and I'm considering switching to a competitor. I want to speak with someone who can actually resolve these ongoing issues. +``` +**Expected Route**: Escalation Agent +**Expected Response**: Complete retention process including account review, senior support contact, service improvement plan, and customer success manager assignment +**Tool Execution**: `chat_with_agent` → Escalation Agent provides customer retention specialist contact and service quality improvement plan + +## 🤔 Mixed/Complex Scenarios + +### Scenario M1: Technical + Billing +``` +My API requests started failing yesterday with 429 rate limit errors, but I'm on the Pro plan which should have higher limits. Did my plan get downgraded? I'm still being charged the Pro price but getting Basic plan limits. +``` +**Expected Route**: Technical Support Agent (primary) or Billing Agent +**Expected Response**: Complete investigation including API limit verification, account status check, billing verification, and either technical resolution or billing escalation +**Tool Execution**: `chat_with_agent` → Technical Support investigates API limits and coordinates with billing if needed + +### Scenario M2: Product + Escalation +``` +I was promised during the sales call that your Enterprise plan includes custom integrations. However, after upgrading, I'm being told this requires an additional $10,000 implementation fee. This contradicts what I was told by your sales team. +``` +**Expected Route**: Escalation Agent +**Expected Response**: Complete sales promise review including sales team consultation, Enterprise feature verification, implementation fee clarification, and senior sales manager contact +**Tool Execution**: `chat_with_agent` → Escalation Agent provides sales promise investigation and senior management contact + +### Scenario M3: Vague Request +``` +Hi, I'm having trouble with your service. Can you help me? +``` +**Expected Route**: Should ask for clarification before routing +**Expected Response**: Polite clarification request with specific questions to help identify the issue type and appropriate specialist +**Tool Execution**: Triage agent asks clarifying questions without executing specialist tools + +## 🎯 Testing Instructions + +### Interactive Testing + +1. **Start the complete triage system**: + ```bash + npx dexto --agent agents/triage-demo/triage-agent.yml + ``` + +2. **Copy and paste** scenarios from above into the chat + +3. **Observe the complete workflow**: + - **Routing analysis** (which specialist is chosen and why) + - **Tool execution** (`chat_with_agent` tool calls) + - **Complete customer response** (routing confirmation + specialist answer) + - **Response quality** (specificity, completeness, helpfulness) + +### One-Shot Testing + +Test scenarios quickly with command-line execution: + +```bash +# Test Technical Support scenario +npx dexto --agent agents/triage-demo/triage-agent.yml "My TeamFlow mobile app crashes every time I try to export project data. I'm using iPhone 15 with iOS 17.2. This is urgent." + +# Test Billing scenario +npx dexto --agent agents/triage-demo/triage-agent.yml "I want to upgrade from Basic to Pro but confused about pricing. Will I be charged the full amount?" + +# Test Product Info scenario +npx dexto --agent agents/triage-demo/triage-agent.yml "What's the difference between Pro and Enterprise plans? I need API access for 25 people." + +# Test Escalation scenario +npx dexto --agent agents/triage-demo/triage-agent.yml "Your system outage cost my business $50,000 in lost sales. I need compensation and want to discuss SLA violations." +``` + +### Expected Response Quality + +For each test, verify that responses include: + +1. **Brief routing confirmation** (one sentence about which specialist was consulted) +2. **Complete specialist answer** with specific, actionable information +3. **Relevant details** from TeamFlow's business documentation +4. **Appropriate tone** (professional, helpful, empathetic when needed) +5. **Follow-up invitation** (offering additional help if needed) + +## 📊 Expected Results Summary + +| Category | Count | Expected Workflow | +|----------|-------|-------------------| +| Technical | 3 | Route → Execute Technical Support → Complete troubleshooting response | +| Billing | 3 | Route → Execute Billing Agent → Complete billing/payment resolution | +| Product Info | 3 | Route → Execute Product Info Agent → Complete feature/plan information | +| Escalation | 3 | Route → Execute Escalation Agent → Complete escalation with contacts | +| Mixed/Complex | 3 | Route → Execute Primary Agent → Complete investigation/resolution | + +## 🔍 Success Criteria + +The triage system should demonstrate: + +- **95%+ routing accuracy** to appropriate specialist agents +- **100% tool execution** success (no failed `chat_with_agent` calls) +- **Complete responses** that directly address customer needs +- **Professional tone** with empathy for customer situations +- **Specific information** from TeamFlow business context (plans, policies, features) +- **Clear next steps** for customer resolution + +## 🚫 Common Issues to Watch For + +- **Routing without execution**: Agent identifies correct specialist but doesn't call `chat_with_agent` +- **Tool confirmation prompts**: Should auto-approve due to configuration +- **Incomplete responses**: Missing specialist answers or generic routing messages +- **Wrong specialist**: Incorrect routing based on request analysis +- **Multiple tool calls**: Unnecessary repeated calls to specialists + +The complete triage system should provide **seamless, professional customer support** that customers would expect from a real enterprise support team! \ No newline at end of file diff --git a/dexto/agents/triage-demo/triage-agent.yml b/dexto/agents/triage-demo/triage-agent.yml new file mode 100644 index 00000000..191edc4e --- /dev/null +++ b/dexto/agents/triage-demo/triage-agent.yml @@ -0,0 +1,187 @@ +# Customer Support Triage Agent Configuration +# Main coordination agent that routes requests to specialized support agents + +# Optional greeting shown at chat start (UI can consume this) +greeting: "🎯 Hello! I'm your Support Triage Agent. How can I help you today?" + +systemPrompt: + contributors: + - id: base-prompt + type: static + priority: 0 + content: | + You are an intelligent Customer Support Triage Agent for TeamFlow, responsible for analyzing incoming customer requests and routing them to the most appropriate specialized support agent. + + ## Your Core Mission + Quickly analyze each customer inquiry to determine: + 1. The primary issue category + 2. Urgency level + 3. Which specialized agent should handle it + 4. Any immediate information needed for effective routing + + ## Available Specialized Agents + + **Technical Support Agent** - Route here for: + - Bug reports, error messages, system failures + - API integration problems, technical troubleshooting + - Performance problems, crashes, connectivity issues + - Mobile app issues, browser compatibility problems + - Integration issues (Slack, GitHub, Salesforce, etc.) + + **Billing Agent** - Route here for: + - Payment failures, billing questions, invoice disputes + - Plan changes (Basic $9/month, Pro $19/month, Enterprise $39/month) + - Refund requests, pricing inquiries + - Account billing setup and payment method issues + - Subscription cancellations or upgrades + + **Product Info Agent** - Route here for: + - Feature questions, product capabilities, specifications + - How-to questions, usage guidance, best practices + - Plan comparisons and recommendations + - Integration capabilities and setup guidance + - General product information and documentation requests + + **Escalation Agent** - Route here for: + - Unresolved issues after specialist attempts + - Enterprise customer requests and contract issues + - Complaints about service quality or agent performance + - Legal, compliance, or security-related inquiries (GDPR, SOC 2) + - Issues requiring management approval or special handling + + ## Your Triage Process + + 1. **Analyze the Request**: Read the customer's message carefully + 2. **Categorize**: Determine the primary issue type and any secondary concerns + 3. **Assess Urgency**: Is this urgent, normal, or low priority? + 4. **Route Intelligently**: Choose the best-suited specialist agent + 5. **Provide Context**: Give the specialist agent relevant background + + ## Routing Guidelines + + - **Multiple Issues**: Route to the agent handling the most critical/urgent aspect + - **Unclear Requests**: Ask clarifying questions before routing + - **Previous Escalations**: Route directly to Escalation Agent if this is a follow-up + - **Enterprise Customers**: Consider escalation for Enterprise plan customers + - **Time-Sensitive**: Prioritize billing issues near payment dates, urgent technical failures + + ## Your Response Format + + When routing a request, provide: + 1. Brief analysis of the issue + 2. Which agent you're routing to and why + 3. Any additional context the specialist agent should know + 4. Expected resolution timeframe (if applicable) + + You have access to TeamFlow's company information to help with context and routing decisions. + + Remember: You are the first line of intelligent support. Your accurate routing ensures TeamFlow customers get expert help quickly and efficiently. + + ## Tool Usage Instructions + + After you decide which specialist should handle the inquiry, you MUST: + 1. Call the `chat_with_agent` tool with the full customer message as the `message` argument. The runtime will automatically route this call to the correct specialist agent based on the current connection mapping. + 2. Wait for the tool to return the specialist agent’s answer. + 3. Respond back to the customer with a concise helpful answer that includes: + - A short confirmation of the routing you performed (one sentence max). + - The detailed answer from the specialist agent. + + Only use `chat_with_agent` once per customer request unless follow-up questions arise. + + - id: company-overview + type: file + priority: 10 + files: + - "${{dexto.agent_dir}}/docs/company-overview.md" + options: + includeFilenames: true + errorHandling: skip + +# Auto-approve tool executions so the triage agent can seamlessly delegate tasks + +toolConfirmation: + mode: auto-approve + allowedToolsStorage: memory + +mcpServers: + # Filesystem for logging and case management + filesystem: + type: stdio + command: npx + args: + - -y + - "@modelcontextprotocol/server-filesystem" + - . + + # Web research for understanding customer context and company info + tavily: + type: stdio + command: npx + args: + - -y + - tavily-mcp@0.1.3 + env: + TAVILY_API_KEY: $TAVILY_API_KEY + connectionMode: lenient + + # Specialized support agents running as MCP servers via npx + technical_support: + type: stdio + command: npx + args: + - dexto + - --mode + - mcp + - --agent + - "${{dexto.agent_dir}}/technical-support-agent.yml" + connectionMode: lenient + + billing_support: + type: stdio + command: npx + args: + - dexto + - --mode + - mcp + - --agent + - "${{dexto.agent_dir}}/billing-agent.yml" + connectionMode: lenient + + product_info: + type: stdio + command: npx + args: + - dexto + - --mode + - mcp + - --agent + - "${{dexto.agent_dir}}/product-info-agent.yml" + connectionMode: lenient + + escalation: + type: stdio + command: npx + args: + - dexto + - --mode + - mcp + - --agent + - "${{dexto.agent_dir}}/escalation-agent.yml" + connectionMode: lenient + +# Storage configuration +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local # CLI provides storePath automatically + maxBlobSize: 52428800 # 50MB per blob + maxTotalSize: 1073741824 # 1GB total storage + cleanupAfterDays: 30 + +llm: + provider: openai + model: gpt-5 + apiKey: $OPENAI_API_KEY \ No newline at end of file diff --git a/dexto/agents/workflow-builder-agent/workflow-builder-agent.yml b/dexto/agents/workflow-builder-agent/workflow-builder-agent.yml new file mode 100644 index 00000000..44d95f7a --- /dev/null +++ b/dexto/agents/workflow-builder-agent/workflow-builder-agent.yml @@ -0,0 +1,193 @@ +# Workflow Builder Agent +# AI agent for building and managing n8n automation workflows +# Uses the n8n MCP server to interact with n8n instances + +mcpServers: + n8n: + type: stdio + command: npx + args: + - -y + - n8n-mcp + env: + MCP_MODE: stdio + N8N_API_URL: $N8N_MCP_URL + N8N_API_KEY: $N8N_MCP_TOKEN + timeout: 60000 + connectionMode: strict + # Alternative: n8n's built-in MCP (read-only, doesn't support building workflows) + # n8n-builtin: + # type: stdio + # command: npx + # args: + # - -y + # - supergateway + # - --streamableHttp + # - $N8N_MCP_URL + # - --header + # - "authorization:Bearer $N8N_MCP_TOKEN" + +greeting: "Hi! I'm your Workflow Builder Agent. I can help you create, manage, and automate n8n workflows. What would you like to build today?" + +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a Workflow Builder Agent specialized in creating and managing automation workflows using n8n. You have direct access to n8n instances through the MCP server, allowing you to build powerful integrations and automations. + + ## Your Capabilities + + **Workflow Management:** + - Create new workflows from scratch based on user requirements + - List, view, and analyze existing workflows + - Update and modify workflow configurations + - Activate and deactivate workflows + - Delete workflows when requested + + **Execution Management:** + - View workflow execution history + - Analyze execution results and identify failures + - Debug workflow issues by examining execution details + - Clean up old execution records + + **Credential & Authentication:** + - Create new credentials for service integrations + - View credential schemas to understand required fields + - Manage credential lifecycle + + **Organization:** + - Create and manage tags for workflow organization + - Assign tags to workflows for better categorization + - Manage projects (Enterprise feature) + - Create and manage variables + + **Security & Monitoring:** + - Generate security audit reports + - Monitor workflow health and performance + - Identify potential issues in workflow configurations + + ## Workflow Building Guidelines + + When creating workflows, follow these best practices: + 1. **Understand the goal**: Ask clarifying questions to understand what the user wants to automate + 2. **Plan the flow**: Break down the automation into logical steps (trigger -> processing -> output) + 3. **Choose appropriate triggers**: Select the right trigger type (webhook, schedule, event-based) + 4. **Error handling**: Include error handling nodes for robust workflows + 5. **Test incrementally**: Suggest testing each part of the workflow + + ## Common Integration Patterns + + - **Webhook-based**: Receive data from external services + - **Scheduled tasks**: Run workflows on a schedule (cron-based) + - **Event-driven**: React to events from connected services + - **Data transformation**: Process and transform data between systems + - **Multi-step pipelines**: Chain multiple operations together + + ## Interaction Guidelines + + - Before creating workflows, confirm the user's requirements and expected behavior + - Explain what each workflow does in simple terms + - Warn about potential issues (rate limits, authentication, data formats) + - Suggest improvements to existing workflows when appropriate + - Always confirm before deleting or deactivating workflows + + ## Important: Service Credentials + + This agent helps you design and build workflows, but **service credentials must be set up by the user directly in n8n**. The agent cannot create or access actual API keys. + + When a workflow requires connecting to external services (Twitter, Google Sheets, Slack, etc.): + 1. The user must go to their **n8n dashboard → Credentials** (left menu) + 2. Click **Create** and select the service (e.g., "X (Twitter)", "Google Sheets") + 3. Enter the required API keys, OAuth tokens, or authentication details + 4. Save the credential - n8n will test and encrypt it automatically + + **Always remind users:** + - Which credentials are needed for the workflow you're designing + - That credentials are stored securely (encrypted) in n8n, not in the workflow itself + - To use descriptive names for credentials (e.g., "Twitter - Marketing Account") + - To set up separate credentials for development and production environments + + ## Safety Notes + + - Never expose sensitive credentials in responses + - Be cautious with workflows that modify production data + - Recommend testing in a safe environment first + - Alert users to potential security implications of their workflow designs + + - id: date + type: dynamic + priority: 10 + source: date + enabled: true + +storage: + cache: + type: in-memory + database: + type: sqlite + blob: + type: local + maxBlobSize: 52428800 + maxTotalSize: 1073741824 + cleanupAfterDays: 30 + +llm: + provider: openai + model: gpt-5-mini + apiKey: $OPENAI_API_KEY + +toolConfirmation: + mode: manual + allowedToolsStorage: memory + +prompts: + - type: inline + id: social-media-scheduler + title: "Social Media Scheduler" + description: "Auto-post content to Twitter/LinkedIn from a spreadsheet" + prompt: | + Help me build a social media scheduler workflow with these requirements: + 1. Read scheduled posts from Google Sheets (columns: date, time, platform, content, image_url) + 2. Run on a schedule (every hour) to check for posts due + 3. Post to Twitter/X and LinkedIn based on the platform column + 4. Mark posts as "published" in the sheet after posting + 5. Send a Slack notification when posts go live + + Walk me through the n8n nodes needed and how to configure each one. + category: social-media + priority: 10 + showInStarters: true + - type: inline + id: list-workflows + title: "List Workflows" + description: "View all workflows in your n8n instance" + prompt: "List all workflows in my n8n instance and show their status (active/inactive)." + category: workflows + priority: 9 + showInStarters: true + - type: inline + id: create-workflow + title: "Create Workflow" + description: "Build a new automation workflow" + prompt: "I want to create a new automation workflow. Help me design and build it." + category: workflows + priority: 8 + showInStarters: true + - type: inline + id: execution-history + title: "Execution History" + description: "View recent workflow executions" + prompt: "Show me the recent workflow executions and their status." + category: executions + priority: 7 + showInStarters: true + - type: inline + id: debug-workflow + title: "Debug Workflow" + description: "Analyze and debug workflow issues" + prompt: "Help me debug a workflow that's not working correctly." + category: debugging + priority: 6 + showInStarters: true diff --git a/dexto/assets/email_slack_demo.gif b/dexto/assets/email_slack_demo.gif new file mode 100644 index 00000000..46d36225 Binary files /dev/null and b/dexto/assets/email_slack_demo.gif differ diff --git a/dexto/assets/notion_webui_example.gif b/dexto/assets/notion_webui_example.gif new file mode 100644 index 00000000..703c00ec Binary files /dev/null and b/dexto/assets/notion_webui_example.gif differ diff --git a/dexto/assets/website_demo.gif b/dexto/assets/website_demo.gif new file mode 100644 index 00000000..64ecc0c1 Binary files /dev/null and b/dexto/assets/website_demo.gif differ diff --git a/dexto/commands/README.md b/dexto/commands/README.md new file mode 100644 index 00000000..24147ddf --- /dev/null +++ b/dexto/commands/README.md @@ -0,0 +1,319 @@ +# Dexto Commands (File Prompts) + +This directory contains File Prompts — reusable prompt templates that work like Claude Code's custom slash commands. + +## Prompt Types in Dexto + +Dexto supports four types of prompts, each with different capabilities: + +### 1. 📁 File Prompts (Commands) +Location: +- Local: `commands/` +- Global: `~/.dexto/commands` +Format: Markdown files with frontmatter +Arguments: Positional placeholders (`$1`, `$2`, `$ARGUMENTS`) +Best for: Simple, file-based prompts you can version control + +### 2. 🔌 MCP Prompts +Source: Connected MCP servers +Format: Defined by MCP protocol +Arguments: Named arguments (e.g., `report_type: "metrics"`) +Best for: Complex prompts from external services (GitHub, databases, etc.) + +### 3. ⚡ Starter Prompts +Source: Built into Dexto +Format: Hardcoded in code +Arguments: Varies by prompt +Best for: Common operations provided out-of-the-box + +### 4. ✨ Custom Prompts +Source: Created at runtime via API/UI +Format: Stored in database +Arguments: Positional placeholders like File Prompts +Best for: User-created prompts that need persistence + +--- + +**Custom Commands (Create, Use, Manage)** + +Custom commands are prompts you create at runtime. They live in the local database (not on disk) and are available to the active agent across sessions. They support the same placeholder behavior as file prompts: + +- `$1..$9` and `$ARGUMENTS` positional placeholders +- `$$` escapes a literal dollar +- `{{name}}` named placeholders (when you declare `arguments`) +- If no placeholders are used in the template: arguments/context are appended at the end + +Create via Web UI +- Open the “Create Custom Prompt” modal in the web UI +- Provide `name`, optional `title`/`description`, and the prompt `content` +- Use `$1..$9`/`$ARGUMENTS`/`{{name}}` placeholders in `content` +- Optionally attach a resource file; it will be stored and included when the prompt runs + +Create via API +- POST `POST /api/prompts/custom` with JSON: +``` +{ + "name": "research-summary", + "title": "Research Summary", + "description": "Summarize research papers with key findings", + "content": "Summarize in style $1 with length $2.\n\nContent:\n$ARGUMENTS", + "arguments": [ + { "name": "style", "required": true }, + { "name": "length", "required": true } + ], + "resource": { + "base64": "data:application/pdf;base64,...", + "mimeType": "application/pdf", + "filename": "paper.pdf" + } +} +``` +- Declaring `arguments` is optional but recommended; it enables inline argument hints in the slash UI and required-arg validation. +- The `resource` field is optional; attached data is stored in the blob store and sent alongside the prompt when executed. + +Delete via API +- `DELETE /api/prompts/custom/:name` + +Preview resolution (without sending to LLM) +- `GET /api/prompts/:name/resolve?context=...&args={...}` + - `context` becomes `_context` for positional flows + - `args` is a JSON string for structured argument values + +Argument Handling Summary +- Positional tokens typed after `/name` appear in `args._positional` and are expanded into `$1..$9` and `$ARGUMENTS`. +- Named args can be declared in `arguments` and referenced as `{{name}}`. +- If the template contains any placeholders ($1..$9, $ARGUMENTS, or {{name}}), arguments are considered deconstructed and are NOT appended. +- If the template contains no placeholders, providers append at the end: + - `Context: <_context>` if provided, otherwise + - `Arguments: key: value, ...` + +--- + +## Creating File Prompts + +### Basic Structure + +Create a `.md` file in this directory with frontmatter: + +```markdown +--- +description: Short description of what this prompt does +argument-hint: [required-arg] [optional-arg?] +--- + +# Prompt Title + +Your prompt content here using $1, $2, or $ARGUMENTS placeholders. +``` + +### Frontmatter Fields + +| Field | Required | Description | Example | +|-------|----------|-------------|---------| +| `description` | ✅ Yes | Brief description shown in UI | `"Summarize text with style and length"` | +| `argument-hint` | ⚠️ Recommended | Argument names for UI hints | `"[style] [length]"` | +| `name` | ❌ Optional | Override filename as command name | `"quick-summary"` | +| `category` | ❌ Optional | Group prompts by category | `"text-processing"` | +| `id` | ❌ Optional | Unique identifier | `"summarize-v2"` | + +### Argument Placeholders + +File prompts support Claude Code's positional argument system: + +| Placeholder | Expands To | Use Case | +|-------------|------------|----------| +| `$1`, `$2`, ..., `$9` | Individual arguments by position | Structured parameters | +| `$ARGUMENTS` | Remaining arguments after `$1..$9` | Free-form text content | +| `$$` | Literal dollar sign | When you need `$` in output | + +### Examples + +#### Example 1: Structured Arguments Only + +```markdown +--- +description: Translate text between languages +argument-hint: [from-lang] [to-lang] [text] +--- + +Translate from $1 to $2: + +$3 +``` + +Usage: `/translate english spanish "Hello world"` +Expands to: +``` +Translate from english to spanish: + +Hello world +``` + +#### Example 2: Mixed Structured + Free-form + +```markdown +--- +description: Analyze code with focus area +argument-hint: [file] [focus] +--- + +Analyze the code in **$1** focusing on: $2 + +Full input for context: +$ARGUMENTS +``` + +Usage: `/analyze utils.ts performance "function slow() { ... }"` +Expands to: +``` +Analyze the code in **utils.ts** focusing on: performance + +Full input for context: +utils.ts performance function slow() { ... } +``` + +#### Example 3: Free-form Only + +```markdown +--- +description: Improve any text +argument-hint: [text-to-improve] +--- + +Please improve the following text: + +$ARGUMENTS +``` + +Usage: `/improve "This sentence not good"` +Expands to: +``` +Please improve the following text: + +This sentence not good +``` + +--- + +## Usage in the UI + +### Invoking File Prompts + +1. Type `/` in chat — opens slash command autocomplete +2. Select your prompt — shows inline argument hints +3. Provide arguments — positional order matters! + +### Argument Display + +The UI shows: +- `` — Required argument +- `` — Optional argument +- Hover tooltip — Argument description (if provided) + +Example UI display for summarize: +``` +/summarize + + +
+ ${LOGO_HTML} + +

Processing Authentication...

+

Please wait while we complete your Dexto login.

+
+ + + + `); + } else if (req.method === 'POST' && parsedUrl.pathname === '/callback') { + let body = ''; + const MAX_BODY_SIZE = 10 * 1024; // 10KB - plenty for OAuth tokens + req.on('data', (chunk) => { + if (body.length + chunk.length > MAX_BODY_SIZE) { + req.destroy(); + res.writeHead(413); + res.end('Request too large'); + return; + } + body += chunk.toString(); + }); + + req.on('end', async () => { + try { + const data = JSON.parse(body); + + if (data.error) { + res.writeHead(200); + res.end('OK'); + server.close(); + reject(new Error(`OAuth error: ${data.error}`)); + return; + } + + if (data.access_token) { + const userResponse = await fetch(`${config.authUrl}/auth/v1/user`, { + headers: { + Authorization: `Bearer ${data.access_token}`, + apikey: config.clientId, + }, + }); + + const userData = userResponse.ok ? await userResponse.json() : null; + + const result: OAuthResult = { + accessToken: data.access_token, + refreshToken: data.refresh_token, + expiresIn: data.expires_in, + user: userData + ? { + id: userData.id, + email: userData.email, + name: + userData.user_metadata?.full_name || + userData.email, + } + : undefined, + }; + + res.writeHead(200); + res.end('OK'); + server.close(); + resolve(result); + } else { + res.writeHead(400); + res.end('Missing tokens'); + server.close(); + reject(new Error('No access token received')); + } + } catch (_err) { + res.writeHead(400); + res.end('Invalid data'); + server.close(); + reject(new Error('Invalid callback data')); + } + }); + } else { + res.writeHead(404); + res.end('Not Found'); + } + } catch (error) { + logger.error(`Callback server error: ${error}`); + res.writeHead(500); + res.end('Internal Server Error'); + server.close(); + oauthStateStore.delete(port); + reject(error); + } + }); + + const timeoutMs = 5 * 60 * 1000; + const timeoutHandle = setTimeout(() => { + server.close(); + oauthStateStore.delete(port); + reject(new Error('Authentication timed out')); + }, timeoutMs); + + const cleanup = () => { + clearTimeout(timeoutHandle); + oauthStateStore.delete(port); + }; + + server.listen(port, 'localhost', () => { + logger.debug(`OAuth callback server listening on http://localhost:${port}`); + }); + + server.on('close', cleanup); + server.on('error', (error) => { + cleanup(); + reject(new Error(`Failed to start callback server: ${error.message}`)); + }); + }); +} + +/** + * Perform OAuth login flow with Supabase + */ +export async function performOAuthLogin(config: OAuthConfig): Promise { + try { + const port = await ensurePortAvailable(OAUTH_CALLBACK_PORT); + const redirectUri = `http://localhost:${port}`; + + oauthStateStore.set(port, 'active'); + logger.debug(`Registered OAuth callback server on port ${port}`); + + const provider = config.provider || 'google'; + const authParams = querystring.stringify({ + redirect_to: redirectUri, + }); + + const authUrl = `${config.authUrl}/auth/v1/authorize?provider=${provider}&${authParams}`; + + const tokenPromise = startCallbackServer(port, config); + + console.log(chalk.cyan('🌐 Opening browser for authentication...')); + + try { + const { default: open } = await import('open'); + await open(authUrl); + console.log(chalk.green('✅ Browser opened')); + } catch (_error) { + console.log(chalk.yellow(`💡 Please open manually: ${authUrl}`)); + } + + const spinner = p.spinner(); + spinner.start('Waiting for authentication...'); + + try { + const result = await tokenPromise; + spinner.stop('Authentication successful!'); + return result; + } catch (error) { + spinner.stop('Authentication failed'); + throw error; + } + } catch (_error) { + throw new Error( + `OAuth login failed: ${_error instanceof Error ? _error.message : String(_error)}` + ); + } +} + +/** + * Default Supabase OAuth configuration for Dexto CLI + */ +export const DEFAULT_OAUTH_CONFIG: OAuthConfig = { + authUrl: SUPABASE_URL, + clientId: SUPABASE_ANON_KEY, + provider: 'google', + scopes: ['openid', 'email', 'profile'], +}; diff --git a/dexto/packages/cli/src/cli/auth/service.ts b/dexto/packages/cli/src/cli/auth/service.ts new file mode 100644 index 00000000..ae547a5d --- /dev/null +++ b/dexto/packages/cli/src/cli/auth/service.ts @@ -0,0 +1,196 @@ +// packages/cli/src/cli/auth/service.ts +// Auth storage, token management, and refresh logic + +import chalk from 'chalk'; +import { existsSync, promises as fs } from 'fs'; +import { z } from 'zod'; +import { getDextoGlobalPath, logger } from '@dexto/core'; +import { SUPABASE_URL, SUPABASE_ANON_KEY } from './constants.js'; + +const AUTH_CONFIG_FILE = 'auth.json'; + +export interface AuthConfig { + /** Supabase access token from OAuth login (optional if using --api-key) */ + token?: string | undefined; + refreshToken?: string | undefined; + userId?: string | undefined; + email?: string | undefined; + expiresAt?: number | undefined; + createdAt: number; + /** Dexto API key for gateway access (from --api-key or provisioned after OAuth) */ + dextoApiKey?: string | undefined; + dextoKeyId?: string | undefined; +} + +const AuthConfigSchema = z + .object({ + token: z.string().min(1).optional(), + refreshToken: z.string().optional(), + userId: z.string().optional(), + email: z.string().email().optional(), + expiresAt: z.number().optional(), + createdAt: z.number(), + dextoApiKey: z.string().optional(), + dextoKeyId: z.string().optional(), + }) + .refine((data) => data.token || data.dextoApiKey, { + message: 'Either token (from OAuth) or dextoApiKey (from --api-key) is required', + }); + +export async function storeAuth(config: AuthConfig): Promise { + const authPath = getDextoGlobalPath('', AUTH_CONFIG_FILE); + const dextoDir = getDextoGlobalPath('', ''); + + await fs.mkdir(dextoDir, { recursive: true }); + await fs.writeFile(authPath, JSON.stringify(config, null, 2), { mode: 0o600 }); + + logger.debug(`Stored auth config at: ${authPath}`); +} + +export async function loadAuth(): Promise { + const authPath = getDextoGlobalPath('', AUTH_CONFIG_FILE); + + if (!existsSync(authPath)) { + return null; + } + + try { + const content = await fs.readFile(authPath, 'utf-8'); + const config = JSON.parse(content); + + const validated = AuthConfigSchema.parse(config); + + if (validated.expiresAt && validated.expiresAt < Date.now()) { + // Only remove auth if there's no refresh token available + // If refresh token exists, return the expired auth and let getAuthToken() handle refresh + if (!validated.refreshToken) { + await removeAuth(); + return null; + } + } + + return validated; + } catch (error) { + logger.warn(`Invalid auth config, removing: ${error}`); + await removeAuth(); + return null; + } +} + +export async function removeAuth(): Promise { + const authPath = getDextoGlobalPath('', AUTH_CONFIG_FILE); + + if (existsSync(authPath)) { + await fs.unlink(authPath); + logger.debug(`Removed auth config from: ${authPath}`); + } +} + +export async function isAuthenticated(): Promise { + const auth = await loadAuth(); + return auth !== null; +} + +export async function getAuthToken(): Promise { + const auth = await loadAuth(); + + if (!auth) { + return null; + } + + const now = Date.now(); + const fiveMinutes = 5 * 60 * 1000; + const isExpiringSoon = auth.expiresAt && auth.expiresAt < now + fiveMinutes; + + if (!isExpiringSoon) { + return auth.token ?? null; + } + + if (!auth.refreshToken) { + logger.debug('Token expired but no refresh token available'); + await removeAuth(); + return null; + } + + logger.debug('Access token expired or expiring soon, refreshing...'); + console.log(chalk.cyan('🔄 Access token expiring soon, refreshing...')); + + const refreshResult = await refreshAccessToken(auth.refreshToken); + + if (!refreshResult) { + logger.debug('Token refresh failed, removing auth'); + console.log(chalk.red('❌ Token refresh failed. Please login again.')); + await removeAuth(); + return null; + } + + const newExpiresAt = Date.now() + refreshResult.expiresIn * 1000; + await storeAuth({ + ...auth, + token: refreshResult.accessToken, + refreshToken: refreshResult.refreshToken, + expiresAt: newExpiresAt, + }); + + logger.debug('Token refreshed successfully'); + console.log(chalk.green('✅ Access token refreshed successfully')); + return refreshResult.accessToken; +} + +export async function getDextoApiKey(): Promise { + // Explicit env var takes priority (for CI, testing, account override) + if (process.env.DEXTO_API_KEY?.trim()) { + return process.env.DEXTO_API_KEY; + } + // Fall back to auth.json (from `dexto login`) + const auth = await loadAuth(); + return auth?.dextoApiKey || null; +} + +async function refreshAccessToken(refreshToken: string): Promise<{ + accessToken: string; + refreshToken: string; + expiresIn: number; +} | null> { + try { + const response = await fetch(`${SUPABASE_URL}/auth/v1/token?grant_type=refresh_token`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + apikey: SUPABASE_ANON_KEY, + }, + body: JSON.stringify({ + refresh_token: refreshToken, + }), + signal: AbortSignal.timeout(10_000), + }); + + if (!response.ok) { + logger.debug(`Token refresh failed: ${response.status}`); + return null; + } + + const data = await response.json(); + + if (!data.access_token || !data.refresh_token) { + logger.debug('Token refresh response missing required tokens'); + return null; + } + + logger.debug('Successfully refreshed access token'); + return { + accessToken: data.access_token, + refreshToken: data.refresh_token, + expiresIn: data.expires_in || 3600, + }; + } catch (error) { + logger.debug( + `Token refresh error: ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } +} + +export function getAuthFilePath(): string { + return getDextoGlobalPath('', AUTH_CONFIG_FILE); +} diff --git a/dexto/packages/cli/src/cli/cli-subscriber.ts b/dexto/packages/cli/src/cli/cli-subscriber.ts new file mode 100644 index 00000000..4768030e --- /dev/null +++ b/dexto/packages/cli/src/cli/cli-subscriber.ts @@ -0,0 +1,251 @@ +/** + * CLI Event Subscriber for headless mode + * Handles agent events and outputs to stdout/stderr + * + * Simple, composable output suitable for piping and scripting + * No TUI, no boxes - just clean text output + */ + +import { logger, DextoAgent } from '@dexto/core'; +import { EventSubscriber } from '@dexto/server'; +import { AgentEventBus } from '@dexto/core'; +import type { SanitizedToolResult, AgentEventMap } from '@dexto/core'; +import { capture } from '../analytics/index.js'; + +/** + * Event subscriber for CLI headless mode + * Implements the standard EventSubscriber pattern used throughout the codebase + */ +export class CLISubscriber implements EventSubscriber { + private streamingContent: string = ''; + private completionResolve?: () => void; + private completionReject?: (error: Error) => void; + + subscribe(eventBus: AgentEventBus): void { + eventBus.on('llm:thinking', this.onThinking.bind(this)); + eventBus.on('llm:chunk', (payload) => { + if (payload.chunkType === 'text') { + this.onChunk(payload.content); + } + // Ignore reasoning chunks for headless mode + }); + eventBus.on('llm:tool-call', (payload) => this.onToolCall(payload.toolName, payload.args)); + eventBus.on('llm:tool-result', (payload) => { + // Only call onToolResult when we have sanitized result (success case) + if (payload.sanitized) { + this.onToolResult( + payload.toolName, + payload.sanitized, + payload.rawResult, + payload.success + ); + } + // For error case (success=false), the error is handled via llm:error event + }); + eventBus.on('llm:response', (payload) => { + this.onResponse(payload.content); + this.captureTokenUsage(payload); + }); + eventBus.on('llm:error', (payload) => this.onError(payload.error)); + eventBus.on('session:reset', this.onConversationReset.bind(this)); + eventBus.on('context:compacting', this.onContextCompacting.bind(this)); + eventBus.on('context:compacted', this.onContextCompacted.bind(this)); + } + + /** + * Clean up internal state + * Called when the CLI subscriber is being disposed of + */ + cleanup(): void { + this.streamingContent = ''; + + // Reject any pending promises to prevent resource leaks + if (this.completionReject) { + const reject = this.completionReject; + delete this.completionResolve; + delete this.completionReject; + reject(new Error('CLI subscriber cleaned up while operation pending')); + } + + logger.debug('CLI event subscriber cleaned up'); + } + + onThinking(): void { + // Silent in headless mode - no "thinking..." messages + } + + onChunk(text: string): void { + // Stream directly to stdout for real-time output + this.streamingContent += text; + process.stdout.write(text); + } + + onToolCall(toolName: string, _args: any): void { + // Simple tool indicator to stderr (doesn't interfere with stdout) + process.stderr.write(`[Tool: ${toolName}]\n`); + } + + onToolResult( + toolName: string, + sanitized: SanitizedToolResult, + rawResult?: unknown, + success?: boolean + ): void { + // Simple completion indicator to stderr + const status = success ? '✓' : '✗'; + process.stderr.write(`[${status}] ${toolName} complete\n`); + } + + onResponse(text: string): void { + // If we didn't stream anything (no chunks), output the full response now + if (!this.streamingContent) { + process.stdout.write(text); + if (!text.endsWith('\n')) { + process.stdout.write('\n'); + } + } else { + // We already streamed the content, just add newline if needed + if (!this.streamingContent.endsWith('\n')) { + process.stdout.write('\n'); + } + } + + // Clear accumulated state + this.streamingContent = ''; + + // Resolve completion promise if waiting + if (this.completionResolve) { + const resolve = this.completionResolve; + delete this.completionResolve; + delete this.completionReject; + resolve(); + } + } + + onError(error: Error): void { + // Clear any partial response state + this.streamingContent = ''; + + // Show error to stderr for immediate user feedback + console.error(`❌ Error: ${error.message}`); + + // Show recovery guidance if available (for DextoRuntimeError) + if ('recovery' in error && error.recovery) { + const recoveryMessages = Array.isArray(error.recovery) + ? error.recovery + : [error.recovery]; + console.error(''); + recoveryMessages.forEach((msg) => { + console.error(`💡 ${msg}`); + }); + } + + // Show stack for debugging if available + if (error.stack) { + console.error(''); + console.error(error.stack); + } + + // Log details to file + logger.error(`Error: ${error.message}`, { + stack: error.stack, + name: error.name, + cause: error.cause, + recovery: 'recovery' in error ? error.recovery : undefined, + }); + + // Reject completion promise if waiting + if (this.completionReject) { + const reject = this.completionReject; + delete this.completionResolve; + delete this.completionReject; + reject(error); + } + } + + onConversationReset(): void { + // Clear any partial response state + this.streamingContent = ''; + logger.info('🔄 Conversation history cleared.', null, 'blue'); + } + + onContextCompacting(payload: AgentEventMap['context:compacting']): void { + // Output to stderr (doesn't interfere with stdout response stream) + process.stderr.write( + `[📦 Compacting context (~${payload.estimatedTokens.toLocaleString()} tokens)...]\n` + ); + } + + onContextCompacted(payload: AgentEventMap['context:compacted']): void { + const { originalTokens, compactedTokens, originalMessages, compactedMessages, reason } = + payload; + const reductionPercent = + originalTokens > 0 + ? Math.round(((originalTokens - compactedTokens) / originalTokens) * 100) + : 0; + + // Output to stderr (doesn't interfere with stdout response stream) + process.stderr.write( + `[📦 Context compacted (${reason}): ${originalTokens.toLocaleString()} → ~${compactedTokens.toLocaleString()} tokens (${reductionPercent}% reduction), ${originalMessages} → ${compactedMessages} messages]\n` + ); + } + + /** + * Capture LLM token usage analytics + */ + private captureTokenUsage(payload: AgentEventMap['llm:response']): void { + const { tokenUsage, provider, model, sessionId, estimatedInputTokens } = payload; + if (!tokenUsage || (!tokenUsage.inputTokens && !tokenUsage.outputTokens)) { + return; + } + + // Calculate estimate accuracy if both estimate and actual are available + let estimateAccuracyPercent: number | undefined; + if (estimatedInputTokens !== undefined && tokenUsage.inputTokens) { + const diff = estimatedInputTokens - tokenUsage.inputTokens; + estimateAccuracyPercent = Math.round((diff / tokenUsage.inputTokens) * 100); + } + + capture('dexto_llm_tokens_consumed', { + source: 'cli', + sessionId, + provider, + model, + inputTokens: tokenUsage.inputTokens, + outputTokens: tokenUsage.outputTokens, + reasoningTokens: tokenUsage.reasoningTokens, + totalTokens: tokenUsage.totalTokens, + cacheReadTokens: tokenUsage.cacheReadTokens, + cacheWriteTokens: tokenUsage.cacheWriteTokens, + estimatedInputTokens, + estimateAccuracyPercent, + }); + } + + /** + * Run agent in headless mode and wait for completion + * Returns a promise that resolves when the response is complete + */ + async runAndWait(agent: DextoAgent, prompt: string, sessionId: string): Promise { + // Prevent concurrent calls + if (this.completionResolve || this.completionReject) { + throw new Error('Cannot call runAndWait while another operation is pending'); + } + + return new Promise((resolve, reject) => { + this.completionResolve = resolve; + this.completionReject = reject; + + // Execute the prompt + agent.run(prompt, undefined, undefined, sessionId).catch((error) => { + // If agent.run() rejects but we haven't already rejected via event + if (this.completionReject) { + const rejectHandler = this.completionReject; + delete this.completionResolve; + delete this.completionReject; + rejectHandler(error); + } + }); + }); + } +} diff --git a/dexto/packages/cli/src/cli/commands/auth/index.ts b/dexto/packages/cli/src/cli/commands/auth/index.ts new file mode 100644 index 00000000..e547c170 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/auth/index.ts @@ -0,0 +1,5 @@ +// packages/cli/src/cli/commands/auth/index.ts + +export { handleLoginCommand, handleBrowserLogin } from './login.js'; +export { handleLogoutCommand } from './logout.js'; +export { handleStatusCommand } from './status.js'; diff --git a/dexto/packages/cli/src/cli/commands/auth/login.ts b/dexto/packages/cli/src/cli/commands/auth/login.ts new file mode 100644 index 00000000..e4dc177b --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/auth/login.ts @@ -0,0 +1,314 @@ +// packages/cli/src/cli/commands/auth/login.ts + +import chalk from 'chalk'; +import * as p from '@clack/prompts'; +import { + isAuthenticated, + loadAuth, + storeAuth, + getDextoApiClient, + SUPABASE_URL, + SUPABASE_ANON_KEY, +} from '../../auth/index.js'; +import { logger } from '@dexto/core'; + +/** + * Handle login command - multiple methods supported + */ +export async function handleLoginCommand( + options: { + apiKey?: string; + interactive?: boolean; + } = {} +): Promise { + try { + if (await isAuthenticated()) { + const auth = await loadAuth(); + const userInfo = auth?.email || auth?.userId || 'user'; + console.log(chalk.green(`✅ Already logged in as: ${userInfo}`)); + + // In non-interactive mode, already authenticated = success (idempotent) + if (options.interactive === false) { + return; + } + + const shouldContinue = await p.confirm({ + message: 'Do you want to login with a different account?', + initialValue: false, + }); + + if (p.isCancel(shouldContinue) || !shouldContinue) { + return; + } + } + + if (options.apiKey) { + // Validate the Dexto API key before storing + const client = getDextoApiClient(); + const isValid = await client.validateDextoApiKey(options.apiKey); + if (!isValid) { + throw new Error('Invalid API key provided - validation failed'); + } + await storeAuth({ dextoApiKey: options.apiKey, createdAt: Date.now() }); + console.log(chalk.green('✅ Dexto API key saved')); + return; + } + + if (options.interactive === false) { + throw new Error('--api-key is required when --no-interactive is used'); + } + + p.intro(chalk.inverse(' Login to Dexto ')); + console.log(chalk.dim('This will open your browser for authentication.')); + + const shouldUseOAuth = await p.confirm({ + message: 'Continue with browser authentication?', + initialValue: true, + }); + + if (p.isCancel(shouldUseOAuth)) { + p.cancel('Login cancelled'); + return; + } + + if (shouldUseOAuth) { + await handleBrowserLogin(); + } else { + console.log(chalk.dim('\nAlternatively, you can enter a token manually:')); + await handleTokenLogin(); + } + + p.outro(chalk.green('🎉 Login successful!')); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + p.outro(chalk.red(`❌ Login failed: ${errorMessage}`)); + // Re-throw to let CLI wrapper handle exit and analytics tracking + throw error; + } +} + +export async function handleBrowserLogin(): Promise { + const { performOAuthLogin, DEFAULT_OAUTH_CONFIG } = await import('../../auth/oauth.js'); + + try { + const result = await performOAuthLogin(DEFAULT_OAUTH_CONFIG); + const expiresAt = result.expiresIn ? Date.now() + result.expiresIn * 1000 : undefined; + + await storeAuth({ + token: result.accessToken, + refreshToken: result.refreshToken, + userId: result.user?.id, + email: result.user?.email, + createdAt: Date.now(), + expiresAt, + }); + + if (result.user?.email) { + console.log(chalk.dim(`\nWelcome back, ${result.user.email}`)); + } + + await provisionKeys(result.accessToken, result.user?.email); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + if (errorMessage.includes('timed out')) { + throw new Error('Login timed out. Please try again.'); + } else if (errorMessage.includes('user denied')) { + throw new Error('Login was cancelled.'); + } else { + throw new Error(`Login failed: ${errorMessage}`); + } + } +} + +async function handleTokenLogin(): Promise { + const token = await p.password({ + message: 'Enter your API token:', + validate: (value) => { + if (!value) return 'Token is required'; + if (value.length < 10) return 'Token seems too short'; + return undefined; + }, + }); + + if (p.isCancel(token)) { + p.cancel('Token entry cancelled'); + return; + } + + const spinner = p.spinner(); + spinner.start('Verifying token...'); + + try { + const isValid = await verifyToken(token as string); + + if (!isValid) { + spinner.stop('Invalid token'); + throw new Error('Token verification failed'); + } + + spinner.stop('Token verified!'); + + await storeAuth({ + token: token as string, + createdAt: Date.now(), + }); + + // Provision Dexto API key for gateway access + await provisionKeys(token as string); + } catch (error) { + spinner.stop('Verification failed'); + throw error; + } +} + +async function verifyToken(token: string): Promise { + try { + const response = await fetch(`${SUPABASE_URL}/auth/v1/user`, { + headers: { + Authorization: `Bearer ${token}`, + apikey: SUPABASE_ANON_KEY, + 'User-Agent': 'dexto-cli/1.0.0', + }, + signal: AbortSignal.timeout(10_000), + }); + + if (response.ok) { + const userData = await response.json(); + return !!userData.id; + } + + return false; + } catch (error) { + logger.debug( + `Token verification failed: ${error instanceof Error ? error.message : String(error)}` + ); + return false; + } +} + +/** + * Helper to save Dexto API key to ~/.dexto/.env + * This ensures the key is available for the layered env loading at startup. + */ +async function saveDextoApiKey(apiKey: string): Promise { + const { getDextoEnvPath, ensureDextoGlobalDirectory } = await import('@dexto/core'); + const path = await import('path'); + const fs = await import('fs/promises'); + + const envVar = 'DEXTO_API_KEY'; + const targetEnvPath = getDextoEnvPath(); + + // Ensure directory exists + await ensureDextoGlobalDirectory(); + await fs.mkdir(path.dirname(targetEnvPath), { recursive: true }); + + // Read existing .env or create empty + let envContent = ''; + try { + envContent = await fs.readFile(targetEnvPath, 'utf-8'); + } catch { + // File doesn't exist, start fresh + } + + // Update or add the key + const lines = envContent.split('\n'); + const keyPattern = new RegExp(`^${envVar}=`); + const keyIndex = lines.findIndex((line) => keyPattern.test(line)); + + if (keyIndex >= 0) { + lines[keyIndex] = `${envVar}=${apiKey}`; + } else { + lines.push(`${envVar}=${apiKey}`); + } + + // Write back + await fs.writeFile(targetEnvPath, lines.filter(Boolean).join('\n') + '\n', 'utf-8'); + + // Make available in current process immediately + process.env[envVar] = apiKey; +} + +async function provisionKeys(authToken: string, _userEmail?: string): Promise { + try { + const apiClient = await getDextoApiClient(); + const auth = await loadAuth(); + + // 1. Check if we already have a local key + if (auth?.dextoApiKey) { + console.log(chalk.cyan('🔍 Validating existing API key...')); + + try { + const isValid = await apiClient.validateDextoApiKey(auth.dextoApiKey); + + if (isValid) { + console.log(chalk.green('✅ Existing key is valid')); + // Ensure .env is in sync + await saveDextoApiKey(auth.dextoApiKey); + return; // All good, we're done + } + + // Key is invalid - need new one + console.log(chalk.yellow('⚠️ Existing key is invalid, provisioning new one...')); + const provisionResult = await apiClient.provisionDextoApiKey(authToken); + + if (!provisionResult.dextoApiKey) { + throw new Error('Failed to get new API key'); + } + + await storeAuth({ + ...auth, + dextoApiKey: provisionResult.dextoApiKey, + dextoKeyId: provisionResult.keyId, + }); + await saveDextoApiKey(provisionResult.dextoApiKey); + + console.log(chalk.green('✅ New key provisioned')); + console.log(chalk.dim(` Key ID: ${provisionResult.keyId}`)); + return; + } catch (error) { + // Validation or rotation failed - this is a critical error + logger.warn(`Key validation/rotation failed: ${error}`); + throw new Error( + `Failed to validate or rotate key: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + // 2. No local key - provision one + console.log(chalk.cyan('🔑 Provisioning Dexto API key...')); + let provisionResult = await apiClient.provisionDextoApiKey(authToken); + + if (!auth) { + throw new Error('Authentication state not found'); + } + + // If key already exists server-side but we don't have it locally, regenerate it + if (!provisionResult.isNewKey) { + console.log( + chalk.yellow('⚠️ CLI key exists on server but not locally, regenerating...') + ); + provisionResult = await apiClient.provisionDextoApiKey( + authToken, + 'Dexto CLI Key', + true + ); + } + + await storeAuth({ + ...auth, + dextoApiKey: provisionResult.dextoApiKey, + dextoKeyId: provisionResult.keyId, + }); + await saveDextoApiKey(provisionResult.dextoApiKey); + + console.log(chalk.green('✅ Dexto API key provisioned!')); + console.log(chalk.dim(` Key ID: ${provisionResult.keyId}`)); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.log(chalk.red(`❌ Failed to provision Dexto API key: ${errorMessage}`)); + console.log(chalk.dim(' You can still use Dexto with your own API keys')); + logger.warn(`Provisioning failed: ${errorMessage}`); + // Don't throw - login should still succeed even if key provisioning fails + } +} diff --git a/dexto/packages/cli/src/cli/commands/auth/logout.ts b/dexto/packages/cli/src/cli/commands/auth/logout.ts new file mode 100644 index 00000000..5c777aad --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/auth/logout.ts @@ -0,0 +1,73 @@ +// packages/cli/src/cli/commands/auth/logout.ts + +import chalk from 'chalk'; +import * as p from '@clack/prompts'; +import { isAuthenticated, removeAuth } from '../../auth/index.js'; +import { isUsingDextoCredits } from '../../../config/effective-llm.js'; +import { logger } from '@dexto/core'; + +export async function handleLogoutCommand( + options: { + force?: boolean; + interactive?: boolean; + } = {} +): Promise { + try { + if (!(await isAuthenticated())) { + console.log(chalk.yellow('ℹ️ Not currently logged in')); + return; + } + + // Check if user is configured to use Dexto credits + // Uses getEffectiveLLMConfig() to check all config layers + const usingDextoCredits = await isUsingDextoCredits(); + + if (options.interactive !== false && !options.force) { + p.intro(chalk.inverse(' Logout ')); + + // Warn if using Dexto credits + if (usingDextoCredits) { + console.log( + chalk.yellow( + '\n⚠️ You are currently configured to use Dexto credits (provider: dexto)' + ) + ); + console.log( + chalk.dim(' After logout, you will need to run `dexto setup` to configure') + ); + console.log( + chalk.dim(' a different provider, or `dexto login` to log back in.\n') + ); + } + + const shouldLogout = await p.confirm({ + message: usingDextoCredits + ? 'Logout will disable Dexto credits. Continue?' + : 'Are you sure you want to logout?', + initialValue: false, + }); + + if (p.isCancel(shouldLogout) || !shouldLogout) { + p.cancel('Logout cancelled'); + return; + } + } + + await removeAuth(); + console.log(chalk.green('✅ Successfully logged out')); + + if (usingDextoCredits) { + console.log(); + console.log(chalk.cyan('Next steps:')); + console.log(chalk.dim(' • Run `dexto login` to log back in')); + console.log(chalk.dim(' • Or run `dexto setup` to configure a different provider')); + } + + logger.info('User logged out'); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.error(chalk.red(`❌ Logout failed: ${errorMessage}`)); + // Re-throw to let CLI wrapper handle exit and analytics tracking + throw error; + } +} diff --git a/dexto/packages/cli/src/cli/commands/auth/status.ts b/dexto/packages/cli/src/cli/commands/auth/status.ts new file mode 100644 index 00000000..17017d06 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/auth/status.ts @@ -0,0 +1,29 @@ +// packages/cli/src/cli/commands/auth/status.ts + +import chalk from 'chalk'; +import { loadAuth } from '../../auth/index.js'; + +export async function handleStatusCommand(): Promise { + const auth = await loadAuth(); + + if (!auth) { + console.log(chalk.yellow('❌ Not logged in')); + console.log(chalk.dim('Run `dexto login` to authenticate')); + return; + } + + console.log(chalk.green('✅ Logged in')); + + if (auth.email) { + console.log(chalk.dim(`Email: ${auth.email}`)); + } + + if (auth.userId) { + console.log(chalk.dim(`User ID: ${auth.userId}`)); + } + + if (auth.expiresAt) { + const expiresDate = new Date(auth.expiresAt); + console.log(chalk.dim(`Expires: ${expiresDate.toLocaleDateString()}`)); + } +} diff --git a/dexto/packages/cli/src/cli/commands/billing/index.ts b/dexto/packages/cli/src/cli/commands/billing/index.ts new file mode 100644 index 00000000..24372d93 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/billing/index.ts @@ -0,0 +1,3 @@ +// packages/cli/src/cli/commands/billing/index.ts + +export { handleBillingStatusCommand } from './status.js'; diff --git a/dexto/packages/cli/src/cli/commands/billing/status.ts b/dexto/packages/cli/src/cli/commands/billing/status.ts new file mode 100644 index 00000000..f5437017 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/billing/status.ts @@ -0,0 +1,75 @@ +// packages/cli/src/cli/commands/billing/status.ts + +import chalk from 'chalk'; +import { loadAuth, getDextoApiClient } from '../../auth/index.js'; + +/** + * Handle the `dexto billing` command. + * Shows Dexto account billing information including balance and usage. + */ +export async function handleBillingStatusCommand(): Promise { + const auth = await loadAuth(); + + if (!auth) { + console.log(chalk.yellow('❌ Not logged in to Dexto')); + console.log(chalk.dim('Run `dexto login` to authenticate')); + return; + } + + if (!auth.dextoApiKey) { + console.log(chalk.yellow('❌ No Dexto API key found')); + console.log(chalk.dim('Run `dexto login` to provision an API key')); + return; + } + + console.log(chalk.green('✅ Logged in to Dexto')); + + if (auth.email) { + console.log(chalk.dim(`Account: ${auth.email}`)); + } + + console.log(); + + try { + const apiClient = getDextoApiClient(); + const usage = await apiClient.getUsageSummary(auth.dextoApiKey); + + // Display balance + console.log(chalk.cyan('💰 Balance')); + console.log(` ${chalk.bold('$' + usage.credits_usd.toFixed(2))} remaining`); + console.log(); + + // Display month-to-date usage + console.log(chalk.cyan('📊 This Month')); + console.log(` Spent: ${chalk.yellow('$' + usage.mtd_usage.total_cost_usd.toFixed(4))}`); + console.log(` Requests: ${chalk.yellow(usage.mtd_usage.total_requests.toString())}`); + + // Show usage by model if there's any + const modelEntries = Object.entries(usage.mtd_usage.by_model); + if (modelEntries.length > 0) { + console.log(); + console.log(chalk.cyan('📈 Usage by Model')); + for (const [model, stats] of modelEntries) { + console.log( + ` ${chalk.dim(model)}: $${stats.cost_usd.toFixed(4)} (${stats.requests} requests)` + ); + } + } + + // Show recent usage if any + if (usage.recent.length > 0) { + console.log(); + console.log(chalk.cyan('🕐 Recent Activity')); + for (const entry of usage.recent.slice(0, 5)) { + const date = new Date(entry.timestamp).toLocaleString(); + console.log( + ` ${chalk.dim(date)} - ${entry.model}: $${entry.cost_usd.toFixed(4)}` + ); + } + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.log(chalk.red(`❌ Failed to fetch billing info: ${errorMessage}`)); + console.log(chalk.dim('Your API key may be invalid. Try `dexto login` to refresh.')); + } +} diff --git a/dexto/packages/cli/src/cli/commands/create-app.ts b/dexto/packages/cli/src/cli/commands/create-app.ts new file mode 100644 index 00000000..b5640c02 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/create-app.ts @@ -0,0 +1,780 @@ +import fs from 'fs-extra'; +import path from 'path'; +import chalk from 'chalk'; +import * as p from '@clack/prompts'; +import { logger } from '@dexto/core'; +import { selectOrExit, textOrExit } from '../utils/prompt-helpers.js'; +import { + promptForProjectName, + createProjectDirectory, + setupGitRepo, + createGitignore, + initPackageJson, + createTsconfigForApp, + installDependencies, + createEnvExample, + ensureDirectory, +} from '../utils/scaffolding-utils.js'; +import { + generateIndexForImage, + generateWebServerIndex, + generateWebAppHTML, + generateWebAppJS, + generateWebAppCSS, + generateAppReadme, + generateExampleTool, + generateDiscoveryScript, +} from '../utils/template-engine.js'; +import { getExecutionContext } from '@dexto/agent-management'; + +type AppMode = 'from-image' | 'from-core'; +type AppType = 'script' | 'webapp'; + +export interface CreateAppOptions { + fromImage?: string; + fromCore?: boolean; + type?: AppType; +} + +/** + * Creates a Dexto application with two possible modes: + * - from-image: Use existing image (recommended) + * - from-core: Build from @dexto/core with custom providers (advanced) + * + * Note: To create a new image that extends another image, use `dexto create-image` instead. + * + * @param name - Optional name of the app project + * @param options - Optional flags to specify mode and base image + * @returns The absolute path to the created project directory + */ +export async function createDextoProject( + name?: string, + options?: CreateAppOptions +): Promise { + console.log(chalk.blue('🚀 Creating a Dexto application\n')); + + // Step 1: Get project name + const projectName = name + ? name + : await promptForProjectName('my-dexto-app', 'What do you want to name your app?'); + + // Step 2: Determine app type + let appType: AppType = options?.type || 'script'; + + if (!options?.type) { + appType = await selectOrExit( + { + message: 'What type of app?', + options: [ + { + value: 'script', + label: 'Script', + hint: 'Simple script (default)', + }, + { + value: 'webapp', + label: 'Web App', + hint: 'REST API server with web frontend', + }, + ], + }, + 'App creation cancelled' + ); + } + + // Step 3: Determine app mode (from flags or prompt) + let mode: AppMode; + let baseImage: string | undefined; + + if (options?.fromCore) { + mode = 'from-core'; + } else if (options?.fromImage) { + mode = 'from-image'; + baseImage = options.fromImage; + } else { + // No flags provided, use interactive prompt + mode = await selectOrExit( + { + message: 'How do you want to start?', + options: [ + { + value: 'from-image', + label: 'Use existing image (recommended)', + hint: 'Pre-built image with providers', + }, + { + value: 'from-core', + label: 'Build from core (advanced)', + hint: 'Custom standalone app with your own providers', + }, + ], + }, + 'App creation cancelled' + ); + } + + const spinner = p.spinner(); + let projectPath: string; + const originalCwd = process.cwd(); + + try { + // Create project directory + projectPath = await createProjectDirectory(projectName, spinner); + + // Change to project directory + process.chdir(projectPath); + + if (mode === 'from-core') { + // Mode C: Build from core - custom image with bundler + await scaffoldFromCore(projectPath, projectName, spinner); + + spinner.stop(chalk.green(`✓ Successfully created app: ${projectName}`)); + + console.log(`\n${chalk.cyan('Next steps:')}`); + console.log(` ${chalk.gray('$')} cd ${projectName}`); + console.log( + ` ${chalk.gray('$')} pnpm start ${chalk.gray('(discovers providers, builds, and runs)')}` + ); + console.log(`\n${chalk.gray('Learn more:')} https://docs.dexto.ai\n`); + + return projectPath; + } + + // For from-image mode, select the image (if not already provided via flag) + if (!baseImage) { + const imageChoice = await selectOrExit( + { + message: 'Which image?', + options: [ + { + value: '@dexto/image-local', + label: '@dexto/image-local (recommended)', + hint: 'Local dev - SQLite, filesystem', + }, + { value: 'custom', label: 'Custom npm package...' }, + ], + }, + 'App creation cancelled' + ); + + if (imageChoice === 'custom') { + const customImage = await textOrExit( + { + message: 'Enter the npm package name:', + placeholder: '@myorg/image-custom', + validate: (value) => { + if (!value || value.trim() === '') { + return 'Package name is required'; + } + return undefined; + }, + }, + 'App creation cancelled' + ); + + baseImage = customImage; + } else { + baseImage = imageChoice; + } + } + + // Scaffold from existing image + await scaffoldFromImage(projectPath, projectName, baseImage, appType, originalCwd, spinner); + + spinner.stop(chalk.green(`✓ Successfully created app: ${projectName}`)); + + console.log(`\n${chalk.cyan('Next steps:')}`); + console.log(` ${chalk.gray('$')} cd ${projectName}`); + console.log(` ${chalk.gray('$')} pnpm start`); + console.log(`\n${chalk.gray('Learn more:')} https://docs.dexto.ai\n`); + + return projectPath; + } catch (error) { + // Restore original directory on error + if (originalCwd) { + try { + process.chdir(originalCwd); + } catch { + // Ignore if we can't restore - likely a more serious issue + } + } + + if (spinner) { + spinner.stop(chalk.red('✗ Failed to create app')); + } + throw error; + } +} + +/** + * Mode A: Scaffold app using existing image + */ +async function scaffoldFromImage( + projectPath: string, + projectName: string, + imageName: string, + appType: AppType, + originalCwd: string, + spinner: ReturnType +): Promise { + spinner.start('Setting up app structure...'); + + // Resolve package name for local images (needed for import statements) + let packageNameForImport = imageName; + if (imageName.startsWith('.')) { + const fullPath = path.resolve(originalCwd, imageName); + let packageDir = fullPath; + + // If path ends with /dist/index.js, resolve to package root (parent of dist) + if (fullPath.endsWith('/dist/index.js') || fullPath.endsWith('\\dist\\index.js')) { + packageDir = path.dirname(path.dirname(fullPath)); + } else if (fullPath.endsWith('.js')) { + packageDir = path.dirname(fullPath); + } + + // Read package.json to get the actual package name for imports + try { + const pkgJsonPath = path.join(packageDir, 'package.json'); + const pkgJson = JSON.parse(await fs.readFile(pkgJsonPath, 'utf8')); + packageNameForImport = pkgJson.name; + } catch (_error) { + logger.warn(`Could not read package.json from ${packageDir}, using path as import`); + } + } + + // Create folders + await ensureDirectory('src'); + await ensureDirectory('agents'); + + // Create src/index.ts based on app type + let indexContent: string; + if (appType === 'webapp') { + indexContent = generateWebServerIndex({ + projectName, + packageName: projectName, + description: 'Dexto web server application', + imageName: packageNameForImport, + }); + + // Create web app directory and files + await ensureDirectory('app'); + await ensureDirectory('app/assets'); + await fs.writeFile('app/index.html', generateWebAppHTML(projectName)); + await fs.writeFile('app/assets/main.js', generateWebAppJS()); + await fs.writeFile('app/assets/style.css', generateWebAppCSS()); + } else { + indexContent = generateIndexForImage({ + projectName, + packageName: projectName, + description: 'Dexto application', + imageName: packageNameForImport, + }); + } + await fs.writeFile('src/index.ts', indexContent); + + // Create default agent config + const agentConfig = `# Default Agent Configuration + +# Image: Specifies the provider bundle for this agent +image: '${imageName}' + +# System prompt +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a helpful AI assistant. + +# LLM configuration +llm: + provider: openai + model: gpt-4o + apiKey: $OPENAI_API_KEY + +# Storage +storage: + cache: + type: in-memory + database: + type: sqlite + path: ./data/agent.db + blob: + type: local + storePath: ./data/blobs + +# Custom tools - uncomment to enable filesystem and process tools +# customTools: +# - type: filesystem-tools +# allowedPaths: ['.'] +# blockedPaths: ['.git', 'node_modules'] +# - type: process-tools +# securityLevel: moderate + +# MCP servers - add external tools here +# mcpServers: +# filesystem: +# type: stdio +# command: npx +# args: +# - -y +# - "@modelcontextprotocol/server-filesystem" +# - . +`; + await fs.writeFile('agents/default.yml', agentConfig); + + spinner.message('Creating configuration files...'); + + // Create package.json + await initPackageJson(projectPath, projectName, 'app'); + + // Add scripts + const packageJsonPath = path.join(projectPath, 'package.json'); + const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8')); + + packageJson.scripts = { + start: 'tsx src/index.ts', + build: 'tsc', + ...packageJson.scripts, + }; + await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); + + // Create tsconfig.json + await createTsconfigForApp(projectPath, 'src'); + + // Create README + const readmeContent = generateAppReadme({ + projectName, + packageName: projectName, + description: 'Dexto application using official image', + imageName, + }); + await fs.writeFile('README.md', readmeContent); + + // Create .env.example + await createEnvExample(projectPath, { + OPENAI_API_KEY: 'sk-...', + ANTHROPIC_API_KEY: 'sk-ant-...', + }); + + // Create .gitignore + await createGitignore(projectPath); + + // Initialize git + spinner.message('Initializing git repository...'); + await setupGitRepo(projectPath); + + spinner.message('Installing dependencies...'); + + // Detect if we're in dexto source - use workspace protocol for local development + const executionContext = getExecutionContext(); + const isDextoSource = executionContext === 'dexto-source'; + + const agentMgmtVersion = isDextoSource ? 'workspace:*' : '^1.3.0'; + + // Resolve relative paths to absolute for local images + // (npm/pnpm need absolute paths to package directories when installing from file system) + let resolvedImageName = imageName; + if (imageName.startsWith('.')) { + const fullPath = path.resolve(originalCwd, imageName); + // If path ends with /dist/index.js, resolve to package root (parent of dist) + if (fullPath.endsWith('/dist/index.js') || fullPath.endsWith('\\dist\\index.js')) { + resolvedImageName = path.dirname(path.dirname(fullPath)); + } else if (fullPath.endsWith('.js')) { + // If it's a .js file but not the standard structure, use the directory + resolvedImageName = path.dirname(fullPath); + } else { + // It's already a directory + resolvedImageName = fullPath; + } + } else if (isDextoSource && imageName.startsWith('@dexto/image-')) { + // In dexto source, resolve official images to local workspace packages + // e.g., @dexto/image-local -> packages/image-local + const imagePkgName = imageName.replace('@dexto/', ''); + const imagePkgPath = path.resolve(originalCwd, 'packages', imagePkgName); + if (await fs.pathExists(imagePkgPath)) { + resolvedImageName = imagePkgPath; + } + } + + // Install dependencies (use pnpm in dexto source for workspace protocol support) + // Image is loaded as "environment" - we import from core packages directly + const coreVersion = isDextoSource ? 'workspace:*' : '^1.3.0'; + const serverVersion = isDextoSource ? 'workspace:*' : '^1.3.0'; + + const dependencies = [ + resolvedImageName, // Image provides the environment/providers + `@dexto/core@${coreVersion}`, // Import DextoAgent from here + `@dexto/agent-management@${agentMgmtVersion}`, // Import loadAgentConfig from here + 'tsx', + ]; + + // Add @dexto/server dependency for webapp type + if (appType === 'webapp') { + dependencies.push(`@dexto/server@${serverVersion}`); + } + + await installDependencies( + projectPath, + { + dependencies, + devDependencies: ['typescript@^5.0.0', '@types/node@^20.0.0'], + }, + isDextoSource ? 'pnpm' : undefined + ); +} + +/** + * Mode B: Scaffold standalone app built from @dexto/core + * Supports both dev (runtime discovery) and production (build-time discovery) workflows + */ +async function scaffoldFromCore( + projectPath: string, + projectName: string, + spinner: ReturnType +): Promise { + spinner.start('Setting up app structure...'); + + // Always include example tool for from-core mode + const includeExample = true; + + // Create convention-based folders + await ensureDirectory('src'); + await ensureDirectory('scripts'); + await ensureDirectory('tools'); + await ensureDirectory('blob-store'); + await ensureDirectory('compression'); + await ensureDirectory('plugins'); + await ensureDirectory('agents'); + + // Create .gitkeep files for empty directories + await fs.writeFile('blob-store/.gitkeep', ''); + await fs.writeFile('compression/.gitkeep', ''); + await fs.writeFile('plugins/.gitkeep', ''); + + // Create example tool if requested + if (includeExample) { + await ensureDirectory('tools/example-tool'); + const exampleToolCode = generateExampleTool('example-tool'); + await fs.writeFile('tools/example-tool/index.ts', exampleToolCode); + } else { + await fs.writeFile('tools/.gitkeep', ''); + } + + spinner.message('Generating app files...'); + + // Create dexto.config.ts for provider discovery configuration + const dextoConfigContent = `import { defineConfig } from '@dexto/core'; + +/** + * Dexto Configuration + * + * Provider Discovery Modes: + * - Development (pnpm dev): Runtime discovery - fast iteration, no rebuild needed + * - Production (pnpm build): Build-time discovery - validates and optimizes everything + * + * This mirrors Next.js approach: + * - next dev: Runtime compilation + * - next build + next start: Pre-built production bundle + */ +export default defineConfig({ + providers: { + // Auto-discover providers from convention-based folders + autoDiscover: true, + folders: ['tools', 'blob-store', 'compression', 'plugins'], + }, +}); +`; + await fs.writeFile('dexto.config.ts', dextoConfigContent); + + // Create build-time discovery script + const discoveryScript = generateDiscoveryScript(); + await fs.writeFile('scripts/discover-providers.ts', discoveryScript); + + // Create app entry point - completely clean, no provider registration code + const appIndexContent = `// Standalone Dexto app +// Development: Providers auto-discovered at runtime (pnpm dev) +// Production: Providers bundled at build time (pnpm build + pnpm start) + +import { DextoAgent } from '@dexto/core'; +import { loadAgentConfig } from '@dexto/agent-management'; + +async function main() { + console.log('🚀 Starting ${projectName}\\n'); + + // Load agent configuration + // In dev mode: providers discovered at runtime from dexto.config.ts + // In production: providers pre-registered at build time + const config = await loadAgentConfig('./agents/default.yml'); + + // Create agent + const agent = new DextoAgent(config, './agents/default.yml'); + + await agent.start(); + console.log('✅ Agent started\\n'); + + // Create a session + const session = await agent.createSession(); + + // Example interaction + const response = await agent.run( + 'Hello! Can you help me understand what custom tools are available?', + undefined, // imageDataInput + undefined, // fileDataInput + session.id // sessionId + ); + + console.log('Agent response:', response); + + // Cleanup + await agent.stop(); +} + +main().catch((error) => { + console.error('Error:', error); + process.exit(1); +}); +`; + await fs.writeFile('src/index.ts', appIndexContent); + + // Create default agent config + const agentConfig = `# Default Agent Configuration + +# System prompt +systemPrompt: + contributors: + - id: primary + type: static + priority: 0 + content: | + You are a helpful AI assistant with custom tools. + +# LLM configuration +llm: + provider: openai + model: gpt-4o + apiKey: $OPENAI_API_KEY + +# Storage +storage: + cache: + type: in-memory + database: + type: sqlite + path: ./data/agent.db + blob: + type: local + storePath: ./data/blobs + +# Custom tools are auto-discovered at runtime from tools/ folder +# See dexto.config.ts for provider discovery configuration + +# MCP servers - add external tools here +# mcpServers: +# filesystem: +# type: stdio +# command: npx +# args: +# - -y +# - "@modelcontextprotocol/server-filesystem" +# - . +`; + await fs.writeFile('agents/default.yml', agentConfig); + + spinner.message('Creating configuration files...'); + + // Create package.json for standalone app + await initPackageJson(projectPath, projectName, 'app'); + + // Add scripts for both development and production workflows + const packageJsonPath = path.join(projectPath, 'package.json'); + const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8')); + packageJson.scripts = { + // Development: runtime discovery (fast iteration) + dev: 'tsx src/index.ts', + // Production: build-time discovery + bundling + build: 'tsx scripts/discover-providers.ts && tsup', + start: 'node dist/_entry.js', + typecheck: 'tsc --noEmit', + discover: 'tsx scripts/discover-providers.ts', + ...packageJson.scripts, + }; + await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); + + // Create tsconfig.json + const tsconfigContent = { + compilerOptions: { + target: 'ES2022', + module: 'ESNext', + moduleResolution: 'bundler', + lib: ['ES2022'], + outDir: './dist', + rootDir: './src', + strict: true, + esModuleInterop: true, + skipLibCheck: true, + forceConsistentCasingInFileNames: true, + resolveJsonModule: true, + declaration: true, + declarationMap: true, + sourceMap: true, + }, + include: ['src/**/*', 'tools/**/*', 'blob-store/**/*', 'compression/**/*', 'plugins/**/*'], + exclude: ['node_modules', 'dist'], + }; + await fs.writeFile('tsconfig.json', JSON.stringify(tsconfigContent, null, 2)); + + // Create tsup.config.ts - builds from generated _entry.ts for production + const tsupConfig = `import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: ['src/_entry.ts'], // Generated by scripts/discover-providers.ts + format: ['esm'], + dts: false, // Skip DTS for build artifacts + sourcemap: true, + clean: true, + external: ['@dexto/core', '@dexto/agent-management'], + noExternal: [], +}); +`; + await fs.writeFile('tsup.config.ts', tsupConfig); + + // Create .gitignore - ignore generated build artifacts + await createGitignore(projectPath, [ + '*.tsbuildinfo', + 'dist/', + 'src/_entry.ts', + 'src/_providers.ts', + ]); + + // Create .env.example + await createEnvExample(projectPath, { + OPENAI_API_KEY: 'sk-...', + ANTHROPIC_API_KEY: 'sk-ant-...', + }); + + // Create README + const readmeContent = `# ${projectName} + +Standalone Dexto app with convention-based auto-discovery. + +## Getting Started + +\`\`\`bash +# Install dependencies +pnpm install + +# Add your API key +cp .env.example .env +# Edit .env and add your OPENAI_API_KEY + +# Development (runtime discovery - fast iteration) +pnpm dev + +# Production (build-time discovery + optimized bundle) +pnpm build +pnpm start +\`\`\` + +That's it! Custom providers are discovered automatically: +- **Dev mode** (\`pnpm dev\`): Runtime discovery - add/modify providers without rebuilding +- **Production** (\`pnpm build\`): Build-time discovery - validates and bundles everything + +## Project Structure + +\`\`\` +${projectName}/ +├── src/ +│ ├── index.ts # Your app code (clean, no boilerplate!) +│ ├── _entry.ts # Auto-generated (build only, gitignored) +│ └── _providers.ts # Auto-generated (build only, gitignored) +├── scripts/ +│ └── discover-providers.ts # Build-time discovery script +├── dexto.config.ts # Provider discovery configuration +├── tools/ # Add custom tool providers here +├── blob-store/ # Add custom blob storage providers here +├── compression/ # Add custom compression providers here +├── plugins/ # Add custom plugins here +└── agents/ + └── default.yml # Agent configuration +\`\`\` + +## Adding Custom Providers + +1. Create a provider in the appropriate folder (tools/, blob-store/, compression/, plugins/) +2. Export it with the naming convention: \`Provider\` +3. Run \`pnpm dev\` (instant) or \`pnpm build\` (validated) - everything is auto-discovered! + +**Example** - Adding a custom tool: +\`\`\`typescript +// tools/my-tool/index.ts +import { z } from 'zod'; + +export const myToolProvider = { + type: 'my-tool', + configSchema: z.object({ type: z.literal('my-tool') }), + tools: [ + { + name: 'do_something', + description: 'Does something useful', + parameters: z.object({ input: z.string() }), + execute: async ({ input }) => { + return \`Processed: \${input}\`; + }, + }, + ], +}; +\`\`\` + +That's it! No imports, no registration code needed. + +## Scripts + +- \`pnpm start\` - Build and run (auto-discovers providers) +- \`pnpm run dev\` - Development mode with hot reload +- \`pnpm run build\` - Build only +- \`pnpm run discover\` - Manually run provider discovery +- \`pnpm run typecheck\` - Type check + +## How It Works + +1. **Discovery**: Scans conventional folders for providers +2. **Generation**: Creates \`src/_providers.ts\` (registrations) and \`src/_entry.ts\` (wiring) +3. **Build**: Bundles everything into \`dist/_entry.js\` +4. **Run**: Your clean app code runs with all providers pre-registered + +## Learn More + +- [Dexto Documentation](https://docs.dexto.ai) +- [Custom Tools Guide](https://docs.dexto.ai/docs/guides/custom-tools) +`; + await fs.writeFile('README.md', readmeContent); + + // Initialize git + spinner.message('Initializing git repository...'); + await setupGitRepo(projectPath); + + spinner.message('Installing dependencies...'); + + // Detect if we're in dexto source - use workspace protocol for local development + const executionContext = getExecutionContext(); + const isDextoSource = executionContext === 'dexto-source'; + + const coreVersion = isDextoSource ? 'workspace:*' : '^1.3.0'; + const agentMgmtVersion = isDextoSource ? 'workspace:*' : '^1.3.0'; + + // Install dependencies (use pnpm in dexto source for workspace protocol support) + await installDependencies( + projectPath, + { + dependencies: [ + `@dexto/core@${coreVersion}`, + 'zod', + `@dexto/agent-management@${agentMgmtVersion}`, + ], + devDependencies: ['typescript@^5.0.0', '@types/node@^20.0.0', 'tsx', 'tsup'], + }, + isDextoSource ? 'pnpm' : undefined + ); +} diff --git a/dexto/packages/cli/src/cli/commands/create-image.ts b/dexto/packages/cli/src/cli/commands/create-image.ts new file mode 100644 index 00000000..bb84c8fa --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/create-image.ts @@ -0,0 +1,270 @@ +import path from 'path'; +import * as p from '@clack/prompts'; +import chalk from 'chalk'; +import { selectOrExit, textOrExit, confirmOrExit } from '../utils/prompt-helpers.js'; +import { + promptForProjectName, + createProjectDirectory, + setupGitRepo, + createGitignore, + initPackageJson, + createTsconfigForImage, + installDependencies, + ensureDirectory, +} from '../utils/scaffolding-utils.js'; +import { + generateDextoImageFile, + generateImageReadme, + generateExampleTool, +} from '../utils/template-engine.js'; +import fs from 'fs-extra'; +import { getExecutionContext } from '@dexto/agent-management'; + +/** + * Creates a Dexto image project - a distributable agent harness package + * @param name - Optional name of the image project + * @returns The absolute path to the created project directory + */ +export async function createImage(name?: string): Promise { + console.log(chalk.blue('🎨 Creating a Dexto image - a distributable agent harness package\n')); + + // Step 1: Get project name + const projectName = name + ? name + : await promptForProjectName('my-dexto-image', 'What do you want to name your image?'); + + // Step 2: Get description + const description = await textOrExit( + { + message: 'Describe your image:', + placeholder: 'Custom agent harness for my organization', + defaultValue: 'Custom agent harness for my organization', + }, + 'Image creation cancelled' + ); + + // Step 3: Starting point - new base or extend existing + const startingPoint = await selectOrExit<'base' | 'extend'>( + { + message: 'Starting point:', + options: [ + { value: 'base', label: 'New base image (build from scratch)' }, + { value: 'extend', label: 'Extend existing image (add providers to base)' }, + ], + }, + 'Image creation cancelled' + ); + + let baseImage: string | undefined; + if (startingPoint === 'extend') { + // Step 4: Which image to extend? + const baseImageChoice = await selectOrExit( + { + message: 'Which image to extend?', + options: [ + { + value: '@dexto/image-local', + label: '@dexto/image-local (local development)', + }, + { value: '@dexto/image-cloud', label: '@dexto/image-cloud (cloud production)' }, + { value: '@dexto/image-edge', label: '@dexto/image-edge (edge/serverless)' }, + { value: 'custom', label: 'Custom npm package...' }, + ], + }, + 'Image creation cancelled' + ); + + if (baseImageChoice === 'custom') { + const customBase = await textOrExit( + { + message: 'Enter the npm package name:', + placeholder: '@myorg/image-base', + validate: (value) => { + if (!value || value.trim() === '') { + return 'Package name is required'; + } + return undefined; + }, + }, + 'Image creation cancelled' + ); + + baseImage = customBase; + } else { + baseImage = baseImageChoice; + } + } + + // Step 5: Target environment + const target = await selectOrExit( + { + message: 'Target environment:', + options: [ + { value: 'local-development', label: 'Local development' }, + { value: 'cloud-production', label: 'Cloud production' }, + { value: 'edge-serverless', label: 'Edge/serverless' }, + { value: 'custom', label: 'Custom' }, + ], + }, + 'Image creation cancelled' + ); + + // Step 6: Include example providers? + const includeExamples = await confirmOrExit( + { + message: 'Include example tool provider?', + initialValue: true, + }, + 'Image creation cancelled' + ); + + // Start scaffolding + const spinner = p.spinner(); + let projectPath: string | undefined; + + try { + // Save original cwd before changing directories (for resolving relative paths) + const originalCwd = process.cwd(); + + // Create project directory + projectPath = await createProjectDirectory(projectName, spinner); + + // Change to project directory + process.chdir(projectPath); + + spinner.start('Setting up project structure...'); + + // Create convention-based folders + await ensureDirectory('tools'); + await ensureDirectory('blob-store'); + await ensureDirectory('compression'); + await ensureDirectory('plugins'); + + // Create .gitkeep files for empty directories + await fs.writeFile('blob-store/.gitkeep', ''); + await fs.writeFile('compression/.gitkeep', ''); + await fs.writeFile('plugins/.gitkeep', ''); + + // Create example tool if requested + if (includeExamples) { + await ensureDirectory('tools/example-tool'); + const exampleToolCode = generateExampleTool('example-tool'); + await fs.writeFile('tools/example-tool/index.ts', exampleToolCode); + } else { + await fs.writeFile('tools/.gitkeep', ''); + } + + spinner.message('Generating configuration files...'); + + // Create dexto.image.ts + const dextoImageContent = generateDextoImageFile({ + projectName, + packageName: projectName, + description, + imageName: projectName, + ...(baseImage ? { baseImage } : {}), + target, + }); + await fs.writeFile('dexto.image.ts', dextoImageContent); + + // Create package.json + await initPackageJson(projectPath, projectName, 'image'); + + // Update package.json with build script + const packageJsonPath = path.join(projectPath, 'package.json'); + const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8')); + packageJson.scripts = { + build: 'dexto-bundle build', + typecheck: 'tsc --noEmit', + ...packageJson.scripts, + }; + await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); + + // Create tsconfig.json + await createTsconfigForImage(projectPath); + + // Create README + const readmeContent = generateImageReadme({ + projectName, + packageName: projectName, + description, + imageName: projectName, + ...(baseImage ? { baseImage } : {}), + }); + await fs.writeFile('README.md', readmeContent); + + // Create .gitignore + await createGitignore(projectPath, ['*.tsbuildinfo']); + + // Initialize git + spinner.message('Initializing git repository...'); + await setupGitRepo(projectPath); + + spinner.message('Installing dependencies...'); + + // Detect if we're in dexto source - use workspace protocol for local development + const executionContext = getExecutionContext(); + const isDextoSource = executionContext === 'dexto-source'; + + const coreVersion = isDextoSource ? 'workspace:*' : '^1.3.0'; + const bundlerVersion = isDextoSource ? 'workspace:*' : '^1.3.0'; + + // Determine dependencies based on whether extending + const dependencies: string[] = [`@dexto/core@${coreVersion}`, 'zod']; + const devDependencies = [ + 'typescript@^5.0.0', + '@types/node@^20.0.0', + `@dexto/image-bundler@${bundlerVersion}`, + ]; + + if (baseImage) { + // Resolve base image path if we're in dexto source + let resolvedBaseImage = baseImage; + if (isDextoSource && baseImage.startsWith('@dexto/image-')) { + // In dexto source, resolve official images to local workspace packages + // e.g., @dexto/image-local -> packages/image-local + const imagePkgName = baseImage.replace('@dexto/', ''); + const imagePkgPath = path.resolve(originalCwd, 'packages', imagePkgName); + if (await fs.pathExists(imagePkgPath)) { + resolvedBaseImage = imagePkgPath; + } + } + dependencies.push(resolvedBaseImage); + } + + // Install dependencies (use pnpm in dexto source for workspace protocol support) + await installDependencies( + projectPath, + { + dependencies, + devDependencies, + }, + isDextoSource ? 'pnpm' : undefined + ); + + spinner.stop(chalk.green(`✓ Successfully created image: ${projectName}`)); + + console.log(`\n${chalk.cyan('Next steps:')}`); + console.log(` ${chalk.gray('$')} cd ${projectName}`); + console.log(` ${chalk.gray('$')} pnpm run build`); + console.log( + `\n${chalk.gray('Add your custom providers to the convention-based folders:')}` + ); + console.log(` ${chalk.gray('tools/')} - Custom tool providers`); + console.log(` ${chalk.gray('blob-store/')} - Blob storage providers`); + console.log(` ${chalk.gray('compression/')} - Compression strategies`); + console.log(` ${chalk.gray('plugins/')} - Plugin providers`); + console.log(`\n${chalk.gray('Learn more:')} https://docs.dexto.ai/docs/guides/images\n`); + } catch (error) { + if (spinner) { + spinner.stop(chalk.red('✗ Failed to create image')); + } + throw error; + } + + if (!projectPath) { + throw new Error('Failed to create project directory'); + } + + return projectPath; +} diff --git a/dexto/packages/cli/src/cli/commands/helpers/formatters.ts b/dexto/packages/cli/src/cli/commands/helpers/formatters.ts new file mode 100644 index 00000000..0e4def5f --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/helpers/formatters.ts @@ -0,0 +1,107 @@ +/** + * Session Formatting Utilities + * + * This module contains formatting functions for session-related CLI output. + * Shared between interactive and non-interactive session commands. + */ + +import chalk from 'chalk'; +import type { SessionMetadata, InternalMessage, ToolCall } from '@dexto/core'; +import { isAssistantMessage } from '@dexto/core'; + +/** + * Helper to format session information consistently + */ +export function formatSessionInfo( + sessionId: string, + metadata?: SessionMetadata, + isCurrent: boolean = false +): string { + const prefix = isCurrent ? chalk.green('→') : ' '; + const name = isCurrent ? chalk.green.bold(sessionId) : chalk.cyan(sessionId); + + let info = `${prefix} ${name}`; + + if (metadata) { + const messages = metadata.messageCount || 0; + const activity = + metadata.lastActivity && metadata.lastActivity > 0 + ? new Date(metadata.lastActivity).toLocaleString() + : 'Never'; + + info += chalk.gray(` (${messages} messages, last: ${activity})`); + + if (isCurrent) { + info += chalk.rgb(255, 165, 0)(' [ACTIVE]'); + } + } + + return info; +} + +/** + * Helper to format conversation history + */ +export function formatHistoryMessage(message: InternalMessage, index: number): string { + const timestamp = message.timestamp + ? new Date(message.timestamp).toLocaleTimeString() + : `#${index + 1}`; + + let roleColor = chalk.gray; + let displayLabel: string = message.role; + + switch (message.role) { + case 'user': + roleColor = chalk.blue; + displayLabel = 'You'; + break; + case 'assistant': + roleColor = chalk.green; + displayLabel = 'Assistant'; + break; + case 'system': + roleColor = chalk.rgb(255, 165, 0); + displayLabel = 'System'; + break; + case 'tool': + roleColor = chalk.green; + displayLabel = 'Tool'; + break; + } + + // Handle content formatting + let content = ''; + if (typeof message.content === 'string') { + content = message.content; + } else if (message.content === null) { + content = '[No content]'; + } else if (Array.isArray(message.content)) { + // Handle multimodal content + content = message.content + .map((part) => { + if (part.type === 'text') return part.text; + if (part.type === 'image') return '[Image]'; + if (part.type === 'file') return `[File: ${part.filename || 'unknown'}]`; + return '[Unknown content]'; + }) + .join(' '); + } else { + content = '[No content]'; + } + + // Truncate very long messages + if (content.length > 200) { + content = content.substring(0, 200) + '...'; + } + + // Format tool calls if present + let toolInfo = ''; + if (isAssistantMessage(message) && message.toolCalls && message.toolCalls.length > 0) { + const toolNames = message.toolCalls + .map((tc: ToolCall) => tc.function?.name || 'unknown') + .join(', '); + toolInfo = chalk.gray(` [Tools: ${toolNames}]`); + } + + return ` ${chalk.gray(timestamp)} ${roleColor.bold(displayLabel)}: ${content}${toolInfo}`; +} diff --git a/dexto/packages/cli/src/cli/commands/index.ts b/dexto/packages/cli/src/cli/commands/index.ts new file mode 100644 index 00000000..fd6a6cf3 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/index.ts @@ -0,0 +1,60 @@ +// packages/cli/src/cli/commands/index.ts + +// Project setup commands +export { createDextoProject, type CreateAppOptions } from './create-app.js'; + +export { createImage } from './create-image.js'; + +export { getUserInputToInitDextoApp, initDexto, postInitDexto } from './init-app.js'; + +export { handleSetupCommand, type CLISetupOptions, type CLISetupOptionsInput } from './setup.js'; +export { handleInstallCommand, type InstallCommandOptions } from './install.js'; +export { handleUninstallCommand, type UninstallCommandOptions } from './uninstall.js'; +export { + handleListAgentsCommand, + type ListAgentsCommandOptions, + type ListAgentsCommandOptionsInput, +} from './list-agents.js'; +export { handleWhichCommand, type WhichCommandOptions } from './which.js'; +export { + handleSyncAgentsCommand, + shouldPromptForSync, + markSyncDismissed, + clearSyncDismissed, + type SyncAgentsCommandOptions, +} from './sync-agents.js'; + +// Auth commands +export { handleLoginCommand, handleLogoutCommand, handleStatusCommand } from './auth/index.js'; + +// Billing commands +export { handleBillingStatusCommand } from './billing/index.js'; + +// Plugin commands +export { + handlePluginListCommand, + handlePluginInstallCommand, + handlePluginUninstallCommand, + handlePluginValidateCommand, + // Marketplace handlers + handleMarketplaceAddCommand, + handleMarketplaceRemoveCommand, + handleMarketplaceUpdateCommand, + handleMarketplaceListCommand, + handleMarketplacePluginsCommand, + handleMarketplaceInstallCommand, + type PluginListCommandOptions, + type PluginListCommandOptionsInput, + type PluginInstallCommandOptions, + type PluginInstallCommandOptionsInput, + type PluginUninstallCommandOptions, + type PluginUninstallCommandOptionsInput, + type PluginValidateCommandOptions, + type PluginValidateCommandOptionsInput, + // Marketplace types + type MarketplaceAddCommandOptionsInput, + type MarketplaceRemoveCommandOptionsInput, + type MarketplaceUpdateCommandOptionsInput, + type MarketplaceListCommandOptionsInput, + type MarketplaceInstallCommandOptionsInput, +} from './plugin.js'; diff --git a/dexto/packages/cli/src/cli/commands/init-app.test.ts b/dexto/packages/cli/src/cli/commands/init-app.test.ts new file mode 100644 index 00000000..678e3162 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/init-app.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import fs from 'fs/promises'; +import path from 'path'; +import os from 'os'; +import { createDextoDirectories, createDextoExampleFile, postInitDexto } from './init-app.js'; + +describe('Init Module', () => { + let tempDir: string; + + beforeEach(async () => { + // Create a temporary directory for testing + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-init-test-')); + }); + + afterEach(async () => { + // Cleanup the temporary directory + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + describe('createDextoDirectories', () => { + it('creates dexto and agents directories when they do not exist', async () => { + const result = await createDextoDirectories(tempDir); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.dirPath).toBe(path.join(tempDir, 'dexto')); + } + + // Verify directories exist + const dextoDir = path.join(tempDir, 'dexto'); + const agentsDir = path.join(tempDir, 'dexto', 'agents'); + + expect( + await fs + .access(dextoDir) + .then(() => true) + .catch(() => false) + ).toBe(true); + expect( + await fs + .access(agentsDir) + .then(() => true) + .catch(() => false) + ).toBe(true); + }); + + it('returns false when dexto directory already exists', async () => { + // Create the dexto directory first + const dextoDir = path.join(tempDir, 'dexto'); + await fs.mkdir(dextoDir, { recursive: true }); + + const result = await createDextoDirectories(tempDir); + + expect(result.ok).toBe(false); + }); + }); + + describe('createDextoExampleFile', () => { + it('creates example file with correct content', async () => { + // Change to temp directory to simulate real usage where paths are relative to cwd + const originalCwd = process.cwd(); + process.chdir(tempDir); + + try { + const dextoDir = path.join('src', 'dexto'); // Relative path like real usage + await fs.mkdir(dextoDir, { recursive: true }); + + const examplePath = await createDextoExampleFile(dextoDir); + + expect(examplePath).toBe(path.join(dextoDir, 'dexto-example.ts')); + + // Verify file exists + expect( + await fs + .access(examplePath) + .then(() => true) + .catch(() => false) + ).toBe(true); + + // Verify content contains expected elements + const content = await fs.readFile(examplePath, 'utf8'); + expect(content).toContain( + "import { DextoAgent, loadAgentConfig } from '@dexto/core'" + ); + expect(content).toContain("console.log('🚀 Starting Dexto Basic Example"); + expect(content).toContain('./src/dexto/agents/coding-agent.yml'); // Correct relative path + expect(content).toContain('const agent = new DextoAgent(config)'); + } finally { + process.chdir(originalCwd); + } + }); + + it('generates correct config path for different directory structures', async () => { + const originalCwd = process.cwd(); + process.chdir(tempDir); + + try { + const dextoDir = path.join('custom', 'dexto'); // Relative path + await fs.mkdir(dextoDir, { recursive: true }); + + const examplePath = await createDextoExampleFile(dextoDir); + const content = await fs.readFile(examplePath, 'utf8'); + + expect(content).toContain('./custom/dexto/agents/coding-agent.yml'); + } finally { + process.chdir(originalCwd); + } + }); + }); + + describe('postInitDexto', () => { + it('runs without throwing errors', async () => { + // This function just prints output, so we mainly test it doesn't crash + expect(() => postInitDexto('src')).not.toThrow(); + }); + }); +}); diff --git a/dexto/packages/cli/src/cli/commands/init-app.ts b/dexto/packages/cli/src/cli/commands/init-app.ts new file mode 100644 index 00000000..bf6c2eb2 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/init-app.ts @@ -0,0 +1,401 @@ +import * as p from '@clack/prompts'; +import chalk from 'chalk'; +import fs from 'node:fs/promises'; +import fsExtra from 'fs-extra'; +import path from 'node:path'; +import { getPackageManager, getPackageManagerInstallCommand } from '../utils/package-mgmt.js'; +import { executeWithTimeout } from '../utils/execute.js'; +import { createRequire } from 'module'; +import { type LLMProvider, logger } from '@dexto/core'; +import { updateDextoConfigFile } from '../utils/project-utils.js'; +import { saveProviderApiKey } from '@dexto/agent-management'; +import { + getProviderDisplayName, + isValidApiKeyFormat, + PROVIDER_OPTIONS, +} from '../utils/provider-setup.js'; + +const require = createRequire(import.meta.url); + +/** + * Get user preferences needed to initialize a Dexto app + * @returns The user preferences + */ +export async function getUserInputToInitDextoApp(): Promise<{ + llmProvider: LLMProvider; + llmApiKey: string; + directory: string; + createExampleFile: boolean; +}> { + const answers = await p.group( + { + llmProvider: () => + p.select({ + message: 'Choose your AI provider', + options: PROVIDER_OPTIONS, + }), + llmApiKey: async ({ results }) => { + const llmProvider = results.llmProvider as LLMProvider; + const selection = await p.select({ + message: `Enter your API key for ${getProviderDisplayName(llmProvider)}?`, + options: [ + { value: 'enter', label: 'Enter', hint: 'recommended' }, + { value: 'skip', label: 'Skip', hint: '' }, + ], + initialValue: 'enter', + }); + + if (p.isCancel(selection)) { + p.cancel('Dexto initialization cancelled'); + process.exit(0); + } + + if (selection === 'enter') { + const apiKey = await p.password({ + message: `Enter your ${getProviderDisplayName(llmProvider)} API key`, + mask: '*', + validate: (value) => { + if (!value || value.trim().length === 0) { + return 'API key is required'; + } + if (!isValidApiKeyFormat(value.trim(), llmProvider)) { + return `Invalid ${getProviderDisplayName(llmProvider)} API key format`; + } + return undefined; + }, + }); + + if (p.isCancel(apiKey)) { + p.cancel('Dexto initialization cancelled'); + process.exit(0); + } + + return apiKey; + } + return ''; + }, + directory: () => + p.text({ + message: 'Enter the directory to add the dexto files in', + placeholder: 'src/', + defaultValue: 'src/', + }), + createExampleFile: () => + p.confirm({ + message: 'Create a dexto example file? [Recommended]', + initialValue: true, + }), + }, + { + onCancel: () => { + p.cancel('Dexto initialization cancelled'); + process.exit(0); + }, + } + ); + + // Type assertion to bypass the possible 'Symbol' type returned by p.group which is handled in onCancel + return answers as { + llmProvider: LLMProvider; + directory: string; + llmApiKey: string; + createExampleFile: boolean; + }; +} + +/** + * Initializes an existing project with Dexto in the given directory. + * @param directory - The directory to initialize the Dexto project in + * @param llmProvider - The LLM provider to use + * @param llmApiKey - The API key for the LLM provider + * @returns The path to the created Dexto project + */ +export async function initDexto( + directory: string, + createExampleFile = true, + llmProvider?: LLMProvider, + llmApiKey?: string +): Promise { + const spinner = p.spinner(); + + try { + // install dexto + const packageManager = getPackageManager(); + const installCommand = getPackageManagerInstallCommand(packageManager); + spinner.start('Installing Dexto...'); + const label = 'latest'; + logger.debug( + `Installing Dexto using ${packageManager} with install command: ${installCommand} and label: ${label}` + ); + try { + await executeWithTimeout(packageManager, [installCommand, `@dexto/core@${label}`], { + cwd: process.cwd(), + }); + } catch (installError) { + // Handle pnpm workspace root add error specifically + console.error( + `Install error: ${ + installError instanceof Error ? installError.message : String(installError) + }` + ); + if ( + packageManager === 'pnpm' && + installError instanceof Error && + /\bERR_PNPM_ADDING_TO_ROOT\b/.test(installError.message) + ) { + spinner.stop(chalk.red('Error: Cannot install in pnpm workspace root')); + p.note( + 'You are initializing dexto in a pnpm workspace root. Go to a specific workspace package and run "pnpm add @dexto/core" there.', + chalk.rgb(255, 165, 0)('Workspace Error') + ); + process.exit(1); + } + throw installError; // Re-throw other errors + } + + spinner.stop('Dexto installed successfully!'); + + spinner.start('Creating Dexto files...'); + // create dexto directories (dexto, dexto/agents) + const result = await createDextoDirectories(directory); + + if (!result.ok) { + spinner.stop( + chalk.inverse( + `Dexto already initialized in ${path.join(directory, 'dexto')}. Would you like to overwrite it?` + ) + ); + const overwrite = await p.confirm({ + message: 'Overwrite Dexto?', + initialValue: false, + }); + + if (p.isCancel(overwrite) || !overwrite) { + p.cancel('Dexto initialization cancelled'); + process.exit(1); + } + } + + // create dexto config file + logger.debug('Creating dexto config file...'); + const dextoDir = path.join(directory, 'dexto'); + const agentsDir = path.join(dextoDir, 'agents'); + + let configPath: string; + try { + configPath = await createDextoConfigFile(agentsDir); + logger.debug(`Dexto config file created at ${configPath}`); + } catch (configError) { + spinner.stop(chalk.red('Failed to create agent config file')); + logger.error(`Config creation error: ${configError}`); + throw new Error( + `Failed to create coding-agent.yml: ${configError instanceof Error ? configError.message : String(configError)}` + ); + } + + // update dexto config file based on llmProvider + if (llmProvider) { + logger.debug(`Updating dexto config file based on llmProvider: ${llmProvider}`); + await updateDextoConfigFile(configPath, llmProvider); + logger.debug(`Dexto config file updated with llmProvider: ${llmProvider}`); + } + // create dexto example file if requested + if (createExampleFile) { + logger.debug('Creating dexto example file...'); + await createDextoExampleFile(dextoDir); + logger.debug('Dexto example file created successfully!'); + } + + // add/update .env file (only if user provided a key) + spinner.start('Saving API key to .env file...'); + logger.debug( + `Saving API key: provider=${llmProvider ?? 'none'}, hasApiKey=${Boolean(llmApiKey)}` + ); + if (llmProvider && llmApiKey) { + await saveProviderApiKey(llmProvider, llmApiKey, process.cwd()); + } + spinner.stop('Saved .env updates'); + } catch (err) { + spinner.stop(chalk.inverse(`An error occurred initializing Dexto project - ${err}`)); + logger.debug(`Error: ${err}`); + process.exit(1); + } +} + +/** Adds notes for users to get started with their new initialized Dexto project */ +export async function postInitDexto(directory: string) { + const nextSteps = [ + `1. Run the example: ${chalk.cyan(`node --loader ts-node/esm ${path.join(directory, 'dexto', 'dexto-example.ts')}`)}`, + `2. Add/update your API key(s) in ${chalk.cyan('.env')}`, + `3. Check out the agent configuration file ${chalk.cyan(path.join(directory, 'dexto', 'agents', 'coding-agent.yml'))}`, + `4. Try out different LLMs and MCP servers in the coding-agent.yml file`, + `5. Read more about Dexto: ${chalk.cyan('https://github.com/truffle-ai/dexto')}`, + ].join('\n'); + p.note(nextSteps, chalk.rgb(255, 165, 0)('Next steps:')); +} +/** + * Creates the dexto directories (dexto, dexto/agents) in the given directory. + * @param directory - The directory to create the dexto directories in + * @returns The path to the created dexto directory + */ +export async function createDextoDirectories( + directory: string +): Promise<{ ok: true; dirPath: string } | { ok: false }> { + const dirPath = path.join(directory, 'dexto'); + const agentsPath = path.join(directory, 'dexto', 'agents'); + + try { + await fs.access(dirPath); + return { ok: false }; + } catch { + // fsExtra.ensureDir creates directories recursively if they don't exist + await fsExtra.ensureDir(dirPath); + await fsExtra.ensureDir(agentsPath); + return { ok: true, dirPath }; + } +} + +/** + * Creates a dexto config file in the given directory. Pulls the config file from the installed Dexto package. + * @param directory - The directory to create the config file in + * @returns The path to the created config file + */ +export async function createDextoConfigFile(directory: string): Promise { + // Ensure the directory exists + await fsExtra.ensureDir(directory); + + try { + // Locate the Dexto package installation directory + const pkgJsonPath = require.resolve('dexto/package.json'); + const pkgDir = path.dirname(pkgJsonPath); + logger.debug(`Package directory: ${pkgDir}`); + + // Build path to the configuration template for create-app (with auto-approve toolConfirmation) + const templateConfigSrc = path.join(pkgDir, 'dist', 'agents', 'agent-template.yml'); + logger.debug(`Looking for template at: ${templateConfigSrc}`); + + // Check if template exists - fail if not found + const templateExists = await fsExtra.pathExists(templateConfigSrc); + if (!templateExists) { + throw new Error( + `Template file not found at: ${templateConfigSrc}. This indicates a build issue - the template should be included in the package.` + ); + } + + // Path to the destination config file + const destConfigPath = path.join(directory, 'coding-agent.yml'); + logger.debug(`Copying template to: ${destConfigPath}`); + + // Copy the config file from the Dexto package + await fsExtra.copy(templateConfigSrc, destConfigPath); + logger.debug(`Successfully created config file at: ${destConfigPath}`); + + return destConfigPath; + } catch (error) { + logger.error(`Failed to create Dexto config file: ${error}`); + throw error; + } +} + +/** + * Creates an example file in the given directory showing how to use Dexto in code. This file has example code to get you started. + * @param directory - The directory to create the example index file in + * @returns The path to the created example index file + */ +export async function createDextoExampleFile(directory: string): Promise { + // Extract the base directory from the given path (e.g., "src" from "src/dexto") + const baseDir = path.dirname(directory); + + const configPath = `./${path.posix.join(baseDir, 'dexto/agents/coding-agent.yml')}`; + + const indexTsLines = [ + "import 'dotenv/config';", + "import { DextoAgent, loadAgentConfig } from '@dexto/core';", + '', + "console.log('🚀 Starting Dexto Basic Example\\n');", + '', + 'try {', + ' // Load the agent configuration', + ` const config = await loadAgentConfig('${configPath}');`, + '', + ' // Create a new DextoAgent instance', + ' const agent = new DextoAgent(config);', + '', + ' // Start the agent (connects to MCP servers)', + " console.log('🔗 Connecting to MCP servers...');", + ' await agent.start();', + " console.log('✅ Agent started successfully!\\n');", + '', + ' // Create a session for this conversation', + ' const session = await agent.createSession();', + " console.log('📝 Session created:', session.id, '\\n');", + '', + ' // Example 1: Simple task', + " console.log('📋 Example 1: Simple information request');", + " const request1 = 'What tools do you have available?';", + " console.log('Request:', request1);", + ' const response1 = await agent.run(request1, undefined, undefined, session.id);', + " console.log('Response:', response1);", + " console.log('\\n——————\\n');", + '', + ' // Example 2: File operation', + " console.log('📄 Example 2: File creation');", + ' const request2 = \'Create a file called test-output.txt with the content "Hello from Dexto!"\';', + " console.log('Request:', request2);", + ' const response2 = await agent.run(request2, undefined, undefined, session.id);', + " console.log('Response:', response2);", + " console.log('\\n——————\\n');", + '', + ' // Example 3: Multi-step conversation', + " console.log('🗣️ Example 3: Multi-step conversation');", + ' const request3a = \'Create a simple HTML file called demo.html with a heading that says "Dexto Demo"\';', + " console.log('Request 3a:', request3a);", + ' const response3a = await agent.run(request3a, undefined, undefined, session.id);', + " console.log('Response:', response3a);", + " console.log('\\n\\n');", + " const request3b = 'Now add a paragraph to that HTML file explaining what Dexto is';", + " console.log('Request 3b:', request3b);", + ' const response3b = await agent.run(request3b, undefined, undefined, session.id);', + " console.log('Response:', response3b);", + " console.log('\\n——————\\n');", + '', + ' // Reset conversation (clear context)', + " console.log('🔄 Resetting conversation context...');", + ' await agent.resetConversation(session.id);', + " console.log('🔄 Conversation context reset');", + " console.log('\\n——————\\n');", + '', + ' // Example 4: Complex task', + " console.log('🏗️ Example 4: Complex multi-tool task');", + ' const request4 = ', + " 'Create a simple webpage about AI agents with HTML, CSS, and JavaScript. ' +", + " 'The page should have a title, some content about what AI agents are, ' +", + " 'and a button that shows an alert when clicked.';", + " console.log('Request:', request4);", + ' const response4 = await agent.run(request4, undefined, undefined, session.id);', + " console.log('Response:', response4);", + " console.log('\\n——————\\n');", + '', + ' // Stop the agent (disconnect from MCP servers)', + " console.log('\\n🛑 Stopping agent...');", + ' await agent.stop();', + " console.log('✅ Agent stopped successfully!');", + '', + '} catch (error) {', + " console.error('❌ Error:', error);", + '}', + '', + "console.log('\\n📖 Read Dexto documentation to understand more about using Dexto: https://docs.dexto.ai');", + ]; + const indexTsContent = indexTsLines.join('\n'); + const outputPath = path.join(directory, 'dexto-example.ts'); + + // Log the generated file content and paths for debugging + logger.debug(`Creating example file with config path: ${configPath}`); + logger.debug(`Base directory: ${baseDir}, Output path: ${outputPath}`); + logger.debug(`Generated file content:\n${indexTsContent}`); + + // Ensure the directory exists before writing the file + await fs.writeFile(outputPath, indexTsContent); + return outputPath; +} diff --git a/dexto/packages/cli/src/cli/commands/install.test.ts b/dexto/packages/cli/src/cli/commands/install.test.ts new file mode 100644 index 00000000..c684a987 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/install.test.ts @@ -0,0 +1,283 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as fs from 'fs'; + +// Mock @dexto/core partially: preserve real exports and override specific functions +vi.mock('@dexto/core', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getDextoGlobalPath: vi.fn(), + }; +}); + +// Mock @dexto/agent-management +vi.mock('@dexto/agent-management', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + loadBundledRegistryAgents: vi.fn(), + }; +}); + +// Mock agent-helpers +vi.mock('../../utils/agent-helpers.js', () => ({ + installBundledAgent: vi.fn(), + installCustomAgent: vi.fn(), + listInstalledAgents: vi.fn(), +})); + +// Mock fs +vi.mock('fs', () => ({ + existsSync: vi.fn(), + statSync: vi.fn(), +})); + +// Mock @clack/prompts +vi.mock('@clack/prompts', () => ({ + intro: vi.fn(), + text: vi.fn(), + outro: vi.fn(), + cancel: vi.fn(), + isCancel: vi.fn(), +})); + +// Mock analytics +vi.mock('../../analytics/index.js', () => ({ + capture: vi.fn(), +})); + +// Import SUT after mocks +import { handleInstallCommand } from './install.js'; +import { + installBundledAgent, + installCustomAgent, + listInstalledAgents, +} from '../../utils/agent-helpers.js'; +import { loadBundledRegistryAgents } from '@dexto/agent-management'; + +describe('Install Command', () => { + let consoleSpy: any; + const mockBundledRegistry = { + 'test-agent': { + id: 'test-agent', + name: 'Test Agent', + description: 'Test agent', + author: 'Test', + tags: ['test'], + source: 'test.yml', + type: 'builtin' as const, + }, + 'other-agent': { + id: 'other-agent', + name: 'Other Agent', + description: 'Other agent', + author: 'Test', + tags: ['test'], + source: 'other.yml', + type: 'builtin' as const, + }, + }; + + beforeEach(() => { + vi.clearAllMocks(); + + // Mock loadBundledRegistryAgents to return the mock registry directly + vi.mocked(loadBundledRegistryAgents).mockReturnValue(mockBundledRegistry); + + // Mock agent helper functions + vi.mocked(installBundledAgent).mockResolvedValue('/mock/path/agent.yml'); + vi.mocked(installCustomAgent).mockResolvedValue('/mock/path/custom-agent.yml'); + vi.mocked(listInstalledAgents).mockResolvedValue([]); + + // Mock existsSync to return false by default (agent not installed) + vi.mocked(fs.existsSync).mockReturnValue(false); + + // Mock statSync to return file stats (default: file, not directory) + vi.mocked(fs.statSync).mockReturnValue({ + isDirectory: () => false, + } as any); + + // Mock console + consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + consoleSpy.mockRestore(); + }); + + describe('Validation', () => { + it('throws error when no agents specified and all flag is false', async () => { + await expect(handleInstallCommand([], {})).rejects.toThrow(); + }); + + it('throws error for unknown agents', async () => { + await expect(handleInstallCommand(['test-agent', 'unknown-agent'], {})).rejects.toThrow( + /Unknown agents.*unknown-agent/ + ); + }); + + it('accepts valid agents', async () => { + // Should not throw + await handleInstallCommand(['test-agent'], {}); + + expect(installBundledAgent).toHaveBeenCalledWith('test-agent'); + }); + }); + + describe('Single agent installation', () => { + it('installs single agent', async () => { + await handleInstallCommand(['test-agent'], {}); + + expect(installBundledAgent).toHaveBeenCalledWith('test-agent'); + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('test-agent')); + }); + + it('respects force flag', async () => { + await handleInstallCommand(['test-agent'], { force: true }); + + expect(installBundledAgent).toHaveBeenCalledWith('test-agent'); + }); + + it('skips already installed agents when force is false', async () => { + // Mock agent as already installed + vi.mocked(fs.existsSync).mockReturnValue(true); + + await handleInstallCommand(['test-agent'], { force: false }); + + expect(installBundledAgent).not.toHaveBeenCalled(); + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('already installed')); + }); + + it('reinstalls already installed agents when force is true', async () => { + // Mock agent as already installed + vi.mocked(fs.existsSync).mockReturnValue(true); + + await handleInstallCommand(['test-agent'], { force: true }); + + // Should still install despite existing + expect(installBundledAgent).toHaveBeenCalledWith('test-agent'); + }); + }); + + describe('Bulk installation (--all flag)', () => { + it('installs all available agents when --all flag is used', async () => { + await handleInstallCommand([], { all: true }); + + // Should install both agents from mockBundledRegistry + expect(installBundledAgent).toHaveBeenCalledWith('test-agent'); + expect(installBundledAgent).toHaveBeenCalledWith('other-agent'); + expect(installBundledAgent).toHaveBeenCalledTimes(2); + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('Installing all 2 available agents') + ); + }); + + it('ignores agent list when --all flag is used', async () => { + await handleInstallCommand(['should-be-ignored'], { all: true }); + + // Should install bundled agents, not the specified one + expect(installBundledAgent).toHaveBeenCalledWith('test-agent'); + expect(installBundledAgent).toHaveBeenCalledWith('other-agent'); + expect(installBundledAgent).not.toHaveBeenCalledWith('should-be-ignored'); + }); + }); + + describe('Error handling', () => { + it('continues installing other agents when one fails', async () => { + vi.mocked(installBundledAgent).mockImplementation(async (agentId: string) => { + if (agentId === 'other-agent') { + throw new Error('Installation failed'); + } + return '/path/to/agent.yml'; + }); + + // Should not throw - partial success is acceptable + await handleInstallCommand(['test-agent', 'other-agent'], {}); + + expect(installBundledAgent).toHaveBeenCalledTimes(2); + expect(installBundledAgent).toHaveBeenCalledWith('test-agent'); + expect(installBundledAgent).toHaveBeenCalledWith('other-agent'); + }); + + it('throws when single agent installation fails', async () => { + vi.mocked(installBundledAgent).mockRejectedValue(new Error('Installation failed')); + + // Single agent failure should propagate the error directly + await expect(handleInstallCommand(['test-agent'], {})).rejects.toThrow(); + }); + }); + + describe('Custom agent installation from file paths', () => { + let mockPrompts: any; + + beforeEach(async () => { + const prompts = await import('@clack/prompts'); + mockPrompts = { + intro: vi.mocked(prompts.intro), + text: vi.mocked(prompts.text), + outro: vi.mocked(prompts.outro), + isCancel: vi.mocked(prompts.isCancel), + }; + + // Default prompt responses + mockPrompts.text.mockImplementation(async (opts: any) => { + if (opts.message.includes('Agent name')) return 'my-custom-agent'; + if (opts.message.includes('Description')) return 'Test description'; + if (opts.message.includes('Author')) return 'Test Author'; + if (opts.message.includes('Tags')) return 'custom, test'; + return ''; + }); + mockPrompts.isCancel.mockReturnValue(false); + }); + + it('detects file paths and installs custom agent', async () => { + // Mock existsSync: source file exists, installed path does not + vi.mocked(fs.existsSync).mockImplementation((path: any) => { + if (path.toString().includes('my-agent.yml')) return true; + return false; // Not installed yet + }); + + await handleInstallCommand(['./my-agent.yml'], {}); + + expect(installCustomAgent).toHaveBeenCalledWith( + 'my-custom-agent', + expect.stringContaining('my-agent.yml'), + { + name: 'my-custom-agent', + description: 'Test description', + author: 'Test Author', + tags: ['custom', 'test'], + } + ); + }); + + it('detects paths with forward slashes', async () => { + vi.mocked(fs.existsSync).mockImplementation((path: any) => { + if (path.toString().includes('custom.yml')) return true; + return false; + }); + + await handleInstallCommand(['./agents/custom.yml'], {}); + + expect(installCustomAgent).toHaveBeenCalled(); + }); + + it('validates file exists before installation', async () => { + vi.mocked(fs.existsSync).mockReturnValue(false); + + await expect(handleInstallCommand(['./nonexistent.yml'], {})).rejects.toThrow( + /File not found/ + ); + + expect(installCustomAgent).not.toHaveBeenCalled(); + }); + + it('treats non-path strings as registry names', async () => { + await handleInstallCommand(['test-agent'], {}); + + // Should use bundled agent installation, not custom + expect(installBundledAgent).toHaveBeenCalledWith('test-agent'); + expect(installCustomAgent).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/dexto/packages/cli/src/cli/commands/install.ts b/dexto/packages/cli/src/cli/commands/install.ts new file mode 100644 index 00000000..f10cdc7c --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/install.ts @@ -0,0 +1,359 @@ +// packages/cli/src/cli/commands/install.ts + +import { existsSync, statSync } from 'fs'; +import path from 'path'; +import { z } from 'zod'; +import * as p from '@clack/prompts'; +import { getDextoGlobalPath, loadBundledRegistryAgents } from '@dexto/agent-management'; +import { textOrExit } from '../utils/prompt-helpers.js'; +import { installBundledAgent, installCustomAgent } from '../../utils/agent-helpers.js'; +import { capture } from '../../analytics/index.js'; + +// Zod schema for install command validation +const InstallCommandSchema = z + .object({ + agents: z.array(z.string().min(1, 'Agent name cannot be empty')), + all: z.boolean().default(false), + force: z.boolean().default(false), + }) + .strict(); + +export type InstallCommandOptions = z.output; + +/** + * Check if a string is a file path (contains path separators or ends with .yml) + */ +function isFilePath(input: string): boolean { + return ( + input.includes('/') || + input.includes('\\') || + input.endsWith('.yml') || + input.endsWith('.yaml') + ); +} + +/** + * Extract agent name from file path and sanitize for validity + * Agent names must be lowercase alphanumeric with hyphens only. + * Examples: + * './my-agent.yml' -> 'my-agent' + * './my_agent.yml' -> 'my-agent' (underscore converted) + * './agents/foo/agent.yml' -> 'agent' + * './MyAgent.yml' -> 'myagent' + */ +function extractAgentNameFromPath(filePath: string): string { + const basename = path.basename(filePath); + + // If it's a file, remove the extension + let name = basename; + if (basename.endsWith('.yml') || basename.endsWith('.yaml')) { + name = basename.replace(/\.(yml|yaml)$/, ''); + } + + // Sanitize: lowercase, replace underscores and invalid chars with hyphens + name = name + .toLowerCase() + .replace(/[_\s]+/g, '-') // Replace underscores and spaces with hyphens + .replace(/[^a-z0-9-]/g, '') // Remove any other invalid characters + .replace(/-+/g, '-') // Collapse multiple hyphens + .replace(/^-|-$/g, ''); // Remove leading/trailing hyphens + + return name; +} + +/** + * Prompt user for custom agent metadata + */ +async function promptForMetadata(suggestedName: string): Promise<{ + agentName: string; + description: string; + author: string; + tags: string[]; +}> { + p.intro('📝 Custom Agent Installation'); + + const agentName = await textOrExit( + { + message: 'Agent name:', + placeholder: suggestedName, + defaultValue: suggestedName, + validate: (value) => { + if (!value || value.trim().length === 0) { + return 'Agent name is required'; + } + if (!/^[a-z0-9-]+$/.test(value)) { + return 'Agent name must contain only lowercase letters, numbers, and hyphens'; + } + return undefined; + }, + }, + 'Installation cancelled' + ); + + const description = await textOrExit( + { + message: 'Description:', + placeholder: 'A custom agent for...', + validate: (value) => { + if (!value || value.trim().length === 0) { + return 'Description is required'; + } + return undefined; + }, + }, + 'Installation cancelled' + ); + + const author = await textOrExit( + { + message: 'Author:', + placeholder: 'Your Name', + validate: (value) => { + if (!value || value.trim().length === 0) { + return 'Author is required'; + } + return undefined; + }, + }, + 'Installation cancelled' + ); + + const tagsInput = await textOrExit( + { + message: 'Tags (comma-separated):', + placeholder: 'custom, coding, productivity', + defaultValue: 'custom', + }, + 'Installation cancelled' + ); + + const tags = tagsInput + .split(',') + .map((tag) => tag.trim()) + .filter((tag) => tag.length > 0); + + // Ask about main config file for directory-based agents + // We'll determine if it's a directory later in the flow + + return { agentName, description, author, tags }; +} + +/** + * Validate install command arguments with registry-aware validation + */ +function validateInstallCommand( + agents: string[], + options: Partial +): InstallCommandOptions { + // Basic structure validation + const validated = InstallCommandSchema.parse({ + ...options, + agents, + }); + + // Business logic validation + const availableAgents = loadBundledRegistryAgents(); + if (!validated.all && validated.agents.length === 0) { + throw new Error( + `No agents specified. Use agent names or --all flag. Available agents: ${Object.keys(availableAgents).join(', ')}` + ); + } + + if (!validated.all) { + // Separate file paths from registry names + const filePaths = validated.agents.filter(isFilePath); + const registryNames = validated.agents.filter((agent) => !isFilePath(agent)); + + // Validate registry names exist in registry + const invalidAgents = registryNames.filter((agent) => !(agent in availableAgents)); + if (invalidAgents.length > 0) { + throw new Error( + `Unknown agents: ${invalidAgents.join(', ')}. ` + + `Available agents: ${Object.keys(availableAgents).join(', ')}` + ); + } + + // Validate file paths exist + for (const filePath of filePaths) { + const resolved = path.resolve(filePath); + if (!existsSync(resolved)) { + throw new Error(`File not found: ${filePath}`); + } + } + } + + return validated; +} + +// TODO: move registry code into CLI and move dexto_install_agent metric into registry +export async function handleInstallCommand( + agents: string[], + options: Partial +): Promise { + // Validate command with Zod + const validated = validateInstallCommand(agents, options); + + // Determine which agents to install + let agentsToInstall: string[]; + if (validated.all) { + // --all flag only works with registry agents, not file paths + const availableAgents = loadBundledRegistryAgents(); + agentsToInstall = Object.keys(availableAgents); + console.log(`📋 Installing all ${agentsToInstall.length} available agents...`); + } else { + agentsToInstall = validated.agents; + } + + console.log(`🚀 Installing ${agentsToInstall.length} agents...`); + + let successCount = 0; + let errorCount = 0; + const errors: string[] = []; + const installed: string[] = []; + const skipped: string[] = []; + const failed: string[] = []; + + // Install each agent + for (const agentInput of agentsToInstall) { + try { + // Check if this is a file path or registry name + if (isFilePath(agentInput)) { + // Custom agent installation from file path + console.log(`\n📦 Installing custom agent from ${agentInput}...`); + + const resolvedPath = path.resolve(agentInput); + + // Detect if source is directory or file + const stats = statSync(resolvedPath); + const isDirectory = stats.isDirectory(); + + // Extract suggested name based on whether it's a directory or file + const suggestedName = isDirectory + ? path.basename(resolvedPath) + : extractAgentNameFromPath(resolvedPath); + + // Prompt for metadata + const metadata = await promptForMetadata(suggestedName); + + // Check if already installed (unless --force) + const globalAgentsDir = getDextoGlobalPath('agents'); + const installedPath = path.join(globalAgentsDir, metadata.agentName); + if (existsSync(installedPath) && !validated.force) { + console.log( + `⏭️ ${metadata.agentName} already installed (use --force to reinstall)` + ); + skipped.push(metadata.agentName); + capture('dexto_install_agent', { + agent: metadata.agentName, + status: 'skipped', + reason: 'already_installed', + force: validated.force, + }); + continue; + } + + // Install custom agent + await installCustomAgent(metadata.agentName, resolvedPath, { + name: metadata.agentName, + description: metadata.description, + author: metadata.author, + tags: metadata.tags, + }); + + successCount++; + console.log(`✅ ${metadata.agentName} installed successfully`); + installed.push(metadata.agentName); + + p.outro('🎉 Custom agent installed successfully!'); + + capture('dexto_install_agent', { + agent: metadata.agentName, + status: 'installed', + force: validated.force, + }); + } else { + // Bundled agent installation from registry + console.log(`\n📦 Installing ${agentInput}...`); + + // Check if already installed (unless --force) + const globalAgentsDir = getDextoGlobalPath('agents'); + const installedPath = path.join(globalAgentsDir, agentInput); + if (existsSync(installedPath) && !validated.force) { + console.log(`⏭️ ${agentInput} already installed (use --force to reinstall)`); + skipped.push(agentInput); + capture('dexto_install_agent', { + agent: agentInput, + status: 'skipped', + reason: 'already_installed', + force: validated.force, + }); + continue; + } + + await installBundledAgent(agentInput); + successCount++; + console.log(`✅ ${agentInput} installed successfully`); + installed.push(agentInput); + + capture('dexto_install_agent', { + agent: agentInput, + status: 'installed', + force: validated.force, + }); + } + } catch (error) { + errorCount++; + const errorMsg = `Failed to install ${agentInput}: ${error instanceof Error ? error.message : String(error)}`; + errors.push(errorMsg); + failed.push(agentInput); + console.error(`❌ ${errorMsg}`); + + // Sanitize agent identifier for analytics (avoid sending full local paths) + const safeAgentId = isFilePath(agentInput) ? path.basename(agentInput) : agentInput; + capture('dexto_install_agent', { + agent: safeAgentId, + status: 'failed', + error_message: error instanceof Error ? error.message : String(error), + force: validated.force, + }); + } + } + + // Emit analytics for both single- and multi-agent cases + try { + capture('dexto_install', { + requested: agentsToInstall, + installed, + skipped, + failed, + successCount, + errorCount, + }); + } catch { + // Analytics failures should not block CLI execution. + } + + // For single agent operations, throw error if it failed (after emitting analytics) + if (agentsToInstall.length === 1) { + if (errorCount > 0) { + throw new Error(errors[0]); + } + return; + } + + // Show summary if more than 1 agent installed + console.log(`\n📊 Installation Summary:`); + console.log(`✅ Successfully installed: ${successCount}`); + if (errorCount > 0) { + console.log(`❌ Failed to install: ${errorCount}`); + errors.forEach((error) => console.log(` • ${error}`)); + } + + if (errorCount > 0 && successCount === 0) { + throw new Error('All installations failed'); + } else if (errorCount > 0) { + console.log(`⚠️ Some installations failed, but ${successCount} succeeded.`); + } else { + console.log(`🎉 All agents installed successfully!`); + } +} diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/auth/index.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/auth/index.ts new file mode 100644 index 00000000..0c12e4eb --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/auth/index.ts @@ -0,0 +1,23 @@ +/** + * Auth Commands Module + * + * Authentication commands for interactive CLI. + */ + +import type { CommandDefinition } from '../command-parser.js'; +import { handleLoginCommand } from '../../auth/login.js'; + +/** + * Login command - triggers OAuth flow for Dexto authentication + * Only available when DEXTO_FEATURE_AUTH=true + */ +export const loginCommand: CommandDefinition = { + name: 'login', + description: 'Login to Dexto', + usage: '/login', + category: 'General', + handler: async () => { + await handleLoginCommand({ interactive: true }); + return true; + }, +}; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/command-parser.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/command-parser.ts new file mode 100644 index 00000000..5f676c5b --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/command-parser.ts @@ -0,0 +1,237 @@ +import chalk from 'chalk'; +import type { DextoAgent } from '@dexto/core'; +import type { StyledOutput, SendMessageMarker } from '../../ink-cli/services/CommandService.js'; + +export interface CommandResult { + type: 'command' | 'prompt'; + command?: string; + args?: string[]; + rawInput: string; +} + +/** + * Command handler return type: + * - boolean: Command handled (true) or not found (false) + * - string: Output text to display + * - StyledOutput: Styled output with structured data for rich rendering + * - SendMessageMarker: Send text through normal streaming flow (for prompt commands) + */ +export type CommandHandlerResult = boolean | string | StyledOutput | SendMessageMarker; + +/** + * Context passed to command handlers + */ +export interface CommandContext { + /** Current session ID, or null if no active session */ + sessionId: string | null; +} + +export interface CommandDefinition { + name: string; + description: string; + usage: string; + category?: string; + aliases?: string[]; + subcommands?: CommandDefinition[]; + handler: ( + args: string[], + agent: DextoAgent, + ctx: CommandContext + ) => Promise; +} + +/** + * No-op handler for overlay-only commands. + * Used by commands in ALWAYS_OVERLAY that are handled entirely by the overlay system. + */ +export const overlayOnlyHandler = async (): Promise => true; + +/** + * Parse arguments respecting quotes and escape sequences + */ +function parseQuotedArguments(input: string): string[] { + const args: string[] = []; + let current = ''; + let inQuotes = false; + let quoteChar = ''; + let i = 0; + + while (i < input.length) { + const char = input[i]; + const nextChar = input[i + 1]; + + if (!inQuotes && (char === '"' || char === "'")) { + inQuotes = true; + quoteChar = char; + } else if (inQuotes && char === quoteChar) { + inQuotes = false; + quoteChar = ''; + } else if (!inQuotes && char === ' ') { + if (current) { + args.push(current); + current = ''; + } + } else if (char === '\\' && nextChar) { + // Handle escape sequences + current += nextChar; + i++; // Skip next character + } else { + current += char; + } + i++; + } + + if (current) { + args.push(current); + } + + return args.filter((arg) => arg.length > 0); +} + +/** + * Parses user input to determine if it's a slash command, shell command, or regular prompt + */ +export function parseInput(input: string): CommandResult { + const trimmed = input.trim(); + + // Check if it's a shell command (! prefix) + if (trimmed.startsWith('!')) { + const shellCommand = trimmed.slice(1).trim(); + return { + type: 'command', + command: 'shell', + args: [shellCommand], + rawInput: trimmed, + }; + } + + // Check if it's a slash command + if (trimmed.startsWith('/')) { + const args = parseQuotedArguments(trimmed.slice(1)); + const command = args[0] || ''; + const commandArgs = args.slice(1); + + return { + type: 'command', + command, + args: commandArgs, + rawInput: trimmed, + }; + } + + // Regular user prompt + return { + type: 'prompt', + rawInput: input, + }; +} + +/** + * Finds command suggestions based on partial input + */ +export function getCommandSuggestions(partial: string, commands: CommandDefinition[]): string[] { + const suggestions: string[] = []; + + for (const cmd of commands) { + // Check main command name + if (cmd.name.startsWith(partial)) { + suggestions.push(cmd.name); + } + + // Check aliases + if (cmd.aliases) { + for (const alias of cmd.aliases) { + if (alias.startsWith(partial)) { + suggestions.push(alias); + } + } + } + } + + return suggestions.sort(); +} + +/** + * Formats help text for a command + */ +export function formatCommandHelp(cmd: CommandDefinition, detailed: boolean = false): string { + let help = chalk.cyan(`/${cmd.name}`) + ' - ' + cmd.description; + + if (detailed) { + help += '\n' + chalk.gray(`Usage: ${cmd.usage}`); + + if (cmd.aliases && cmd.aliases.length > 0) { + help += '\n' + chalk.gray(`Aliases: ${cmd.aliases.map((a) => `/${a}`).join(', ')}`); + } + + if (cmd.subcommands && cmd.subcommands.length > 0) { + help += '\n' + chalk.gray('Subcommands:'); + for (const sub of cmd.subcommands) { + help += '\n ' + chalk.cyan(`/${cmd.name} ${sub.name}`) + ' - ' + sub.description; + } + } + } + + return help; +} + +/** + * Displays a formatted list of all available commands + */ +export function displayAllCommands(commands: CommandDefinition[]): void { + console.log(chalk.bold.green('\n📋 Available Commands:\n')); + + // Define category order for consistent display + const categoryOrder = [ + 'General', + 'Session Management', + 'Model Management', + 'MCP Management', + 'Plugin Management', + 'Tool Management', + 'Prompt Management', + 'System', + ]; + + const categories: { [key: string]: CommandDefinition[] } = {}; + + // Initialize categories + for (const category of categoryOrder) { + categories[category] = []; + } + + // Categorize commands using metadata + for (const cmd of commands) { + const category = cmd.category || 'General'; + if (!categories[category]) { + categories[category] = []; + } + categories[category]!.push(cmd); + } + + // Display by category in order + for (const category of categoryOrder) { + const cmds = categories[category]; + if (cmds && cmds.length > 0) { + console.log(chalk.bold.rgb(255, 165, 0)(`${category}:`)); + for (const cmd of cmds) { + console.log(' ' + formatCommandHelp(cmd)); + } + console.log(); + } + } + + // Display any uncategorized commands (fallback) + for (const [category, cmds] of Object.entries(categories)) { + if (!categoryOrder.includes(category) && cmds.length > 0) { + console.log(chalk.bold.rgb(255, 165, 0)(`${category}:`)); + for (const cmd of cmds) { + console.log(' ' + formatCommandHelp(cmd)); + } + console.log(); + } + } + + console.log(chalk.gray('💡 Tip: Use /help for detailed help on any command')); + console.log(chalk.gray('💡 Tip: Type your message normally (without /) to chat with the AI\n')); +} diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/commands.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/commands.ts new file mode 100644 index 00000000..c7a81e34 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/commands.ts @@ -0,0 +1,176 @@ +/** + * CLI Commands Module (Modular Version) + * + * This module aggregates all CLI commands from extracted modular components. + * It maintains the same external interface as the original monolithic commands.ts + * while using the new modular structure internally. + * + * The commands are organized into logical modules: + * - General Commands: Basic CLI functionality (help, exit, clear) + * - Conversation Commands: Session management, history, and search + * - Model Commands: Model switching and configuration + * - MCP Commands: MCP server management + * - Plugin Commands: Claude Code plugin management + * - System Commands: Configuration, logging, and statistics + * - Tool Commands: Tool listing and management + * - Prompt Commands: System prompt management + * - Documentation Commands: Help and documentation access + * + * This file serves as the integration layer that combines all modular commands + * into a single CLI_COMMANDS array for the command execution system. + */ + +import type { DextoAgent } from '@dexto/core'; +import type { CommandDefinition, CommandHandlerResult } from './command-parser.js'; +import { isDextoAuthEnabled } from '@dexto/agent-management'; + +// Import modular command definitions +import { generalCommands, createHelpCommand } from './general-commands.js'; +import { searchCommand, resumeCommand, renameCommand } from './session/index.js'; +import { exportCommand } from './export/index.js'; +import { modelCommands } from './model/index.js'; +import { mcpCommands } from './mcp/index.js'; +import { pluginCommands } from './plugin/index.js'; +import { systemCommands } from './system/index.js'; +import { toolCommands } from './tool-commands.js'; +import { promptCommands } from './prompt-commands.js'; +import { documentationCommands } from './documentation-commands.js'; +import { loginCommand } from './auth/index.js'; + +/** + * Complete list of all available CLI commands. + * This array combines commands from all extracted modules to maintain + * the same interface as the original monolithic implementation. + * + * Commands are organized by category: + * - General: help, exit, clear + * - Session Management: session, history, search + * - Model Management: model + * - MCP Management: mcp + * - Tool Management: tools + * - Prompt Management: prompt + * - System: log, config, stats + * - Documentation: docs + */ +export const CLI_COMMANDS: CommandDefinition[] = []; + +// Build the commands array with proper help command that can access all commands +// All commands here use interactive overlays - no text-based subcommands +const baseCommands: CommandDefinition[] = [ + // General commands (without help) + ...generalCommands, + + // Session management + searchCommand, // /search - opens search overlay + resumeCommand, // /resume - opens session selector overlay + renameCommand, // /rename - rename current session + exportCommand, // /export - opens export wizard overlay + + // Model management + modelCommands, // /model - opens model selector overlay + + // MCP server management + mcpCommands, // /mcp - opens MCP server list overlay + + // Plugin management + pluginCommands, // /plugin - manage Claude Code compatible plugins + + // Tool management commands + ...toolCommands, + + // Prompt management commands + ...promptCommands, + + // System commands + ...systemCommands, + + // Documentation commands + ...documentationCommands, + + // Auth commands (feature-flagged) + ...(isDextoAuthEnabled() ? [loginCommand] : []), +]; + +// Add help command that can see all commands +CLI_COMMANDS.push(createHelpCommand(() => CLI_COMMANDS)); + +// Add all other commands +CLI_COMMANDS.push(...baseCommands); + +/** + * Execute a slash command + * + * @param sessionId - Session ID to use for agent.run() calls + * @returns CommandHandlerResult - boolean, string, or StyledOutput + */ +export async function executeCommand( + command: string, + args: string[], + agent: DextoAgent, + sessionId?: string +): Promise<CommandHandlerResult> { + // Create command context with sessionId + const ctx = { sessionId: sessionId ?? null }; + + // Find the command (including aliases) + const cmd = CLI_COMMANDS.find( + (c) => c.name === command || (c.aliases && c.aliases.includes(command)) + ); + + if (cmd) { + try { + // Execute the handler with context + const result = await cmd.handler(args, agent, ctx); + // If handler returns a string, it's formatted output for ink-cli + // If it returns boolean, it's the old behavior (handled or not) + return result; + } catch (error) { + const errorMsg = `❌ Error executing command /${command}:\n${error instanceof Error ? error.message : String(error)}`; + agent.logger.error( + `Error executing command /${command}: ${error instanceof Error ? error.message : String(error)}` + ); + return errorMsg; // Return for ink-cli + } + } + + // Command not found in static commands - check if it's a dynamic prompt command + // Dynamic commands use displayName (e.g., "quick-start" instead of "config:quick-start") + try { + // Import prompt command creation dynamically to avoid circular dependencies + const { getDynamicPromptCommands } = await import('./prompt-commands.js'); + const dynamicCommands = await getDynamicPromptCommands(agent); + // Commands are registered by displayName, so search by command name directly + const promptCmd = dynamicCommands.find((c) => c.name === command); + + if (promptCmd) { + try { + const result = await promptCmd.handler(args, agent, ctx); + // Return the result directly - can be string, boolean, StyledOutput, or SendMessageMarker + return result; + } catch (error) { + const errorMsg = `❌ Error executing prompt /${command}:\n${error instanceof Error ? error.message : String(error)}`; + agent.logger.error( + `Error executing prompt /${command}: ${error instanceof Error ? error.message : String(error)}` + ); + return errorMsg; + } + } + } catch (error) { + // If loading dynamic commands fails, continue to unknown command error + agent.logger.debug( + `Failed to check dynamic commands for ${command}: ${error instanceof Error ? error.message : String(error)}` + ); + } + + // Command not found and not a prompt + const errorMsg = `❌ Unknown command: /${command}\nType / to see available commands, /prompts to add new ones`; + return errorMsg; // Return for ink-cli +} + +/** + * Get all available command definitions + * This is used by external systems that need to inspect available commands + */ +export function getAllCommands(): CommandDefinition[] { + return CLI_COMMANDS; +} diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/documentation-commands.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/documentation-commands.ts new file mode 100644 index 00000000..a6d8a3f3 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/documentation-commands.ts @@ -0,0 +1,52 @@ +/** + * Documentation Commands Module + * + * This module defines documentation-related slash commands for the Dexto CLI interface. + * These commands provide functionality for accessing documentation and help resources. + * + * Available Documentation Commands: + * - /docs, /doc - Open Dexto documentation in browser + */ + +import type { DextoAgent } from '@dexto/core'; +import type { CommandDefinition, CommandContext } from './command-parser.js'; +import { CommandOutputHelper } from './utils/command-output.js'; + +/** + * Documentation commands + */ +export const documentationCommands: CommandDefinition[] = [ + { + name: 'docs', + description: 'Open Dexto documentation in browser', + usage: '/docs', + category: 'Documentation', + aliases: ['doc'], + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean | string> => { + const docsUrl = 'https://docs.dexto.ai/docs/category/getting-started'; + try { + const { spawn } = await import('child_process'); + + // Cross-platform browser opening + const command = + process.platform === 'darwin' + ? 'open' + : process.platform === 'win32' + ? 'start' + : 'xdg-open'; + + spawn(command, [docsUrl], { detached: true, stdio: 'ignore' }); + return true; // Silent success + } catch { + return CommandOutputHelper.error( + new Error(`Could not open browser. Visit: ${docsUrl}`), + 'Failed to open documentation' + ); + } + }, + }, +]; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/export/index.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/export/index.ts new file mode 100644 index 00000000..30bcb170 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/export/index.ts @@ -0,0 +1,29 @@ +/** + * Export Command Module + * + * Provides the /export command for exporting conversation history. + * Always shows the interactive export wizard overlay. + */ + +import type { CommandDefinition, CommandContext, CommandHandlerResult } from '../command-parser.js'; +import type { DextoAgent } from '@dexto/core'; + +/** + * Export command definition + * Always shows the interactive export wizard overlay (handled by ALWAYS_OVERLAY) + */ +export const exportCommand: CommandDefinition = { + name: 'export', + description: 'Export conversation to markdown or JSON', + usage: '/export', + category: 'Session', + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<CommandHandlerResult> => { + // This handler is never called - export is in ALWAYS_OVERLAY + // which intercepts and shows the export wizard overlay instead + return true; + }, +}; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/general-commands.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/general-commands.ts new file mode 100644 index 00000000..21ad984d --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/general-commands.ts @@ -0,0 +1,562 @@ +/** + * General Commands Module + * + * This module defines general-purpose slash commands for the Dexto CLI interface. + * These are basic commands that don't fit into specific categories. + * + * Available General Commands: + * - /help [command] - Show help information + * - /exit, /quit, /q - Exit the CLI application + * - /clear, /reset - Clear conversation history + */ + +import chalk from 'chalk'; +import { spawn } from 'child_process'; +import type { DextoAgent } from '@dexto/core'; +import type { CommandDefinition, CommandHandlerResult, CommandContext } from './command-parser.js'; +import { formatForInkCli } from './utils/format-output.js'; +import { CommandOutputHelper } from './utils/command-output.js'; +import type { HelpStyledData, ShortcutsStyledData } from '../../ink-cli/state/types.js'; +import { writeToClipboard } from '../../ink-cli/utils/clipboardUtils.js'; + +/** + * Get the shell rc file path for the given shell + */ +function getShellRcFile(shell: string): string | null { + const home = process.env.HOME; + if (!home) return null; + + if (shell.includes('zsh')) { + return `${home}/.zshrc`; + } else if (shell.includes('bash')) { + return `${home}/.bashrc`; + } + return null; +} + +/** + * Wrap command to source shell rc file for alias support + * This mimics how Claude Code handles shell aliases - by sourcing the rc file + * before executing the command, we get aliases without using -i flag. + * We use eval to force the command to be re-parsed after sourcing, since + * aliases are expanded at parse time, not execution time. + */ +function wrapCommandWithRcSource(command: string, shell: string): string { + const rcFile = getShellRcFile(shell); + if (!rcFile) { + return command; + } + + // Escape single quotes in the command for safe eval + const escapedCommand = command.replace(/'/g, "'\\''"); + + // Source the rc file (suppressing errors if it doesn't exist), then eval the command + // eval forces re-parsing after sourcing, allowing aliases to expand + // For bash, we also need to enable expand_aliases + if (shell.includes('bash')) { + return `source "${rcFile}" 2>/dev/null; shopt -s expand_aliases 2>/dev/null; eval '${escapedCommand}'`; + } + return `source "${rcFile}" 2>/dev/null; eval '${escapedCommand}'`; +} + +async function executeShellCommand( + command: string, + cwd: string, + timeoutMs: number = 30000 +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + return new Promise((resolve) => { + // Use user's default shell from SHELL env var, fallback to /bin/sh + // Avoid -i (interactive) as it sets up job control which causes SIGTTIN + // when the parent process tries to read stdin while shell runs. + // Instead, source the shell's rc file to get aliases (similar to Claude Code). + // Use detached: true to create a new process group, preventing the child + // from interfering with the parent's terminal control. + const userShell = process.env.SHELL || '/bin/sh'; + const wrappedCommand = wrapCommandWithRcSource(command, userShell); + + const child = spawn(userShell, ['-c', wrappedCommand], { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env }, + detached: true, + }); + + let stdout = ''; + let stderr = ''; + + const timer = setTimeout(() => { + child.kill(); + resolve({ stdout, stderr: `Command timed out after ${timeoutMs}ms`, exitCode: -1 }); + }, timeoutMs); + + child.stdout.on('data', (data: Buffer) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data: Buffer) => { + stderr += data.toString(); + }); + + child.on('error', (error: Error) => { + clearTimeout(timer); + resolve({ stdout, stderr: error.message, exitCode: -1 }); + }); + + child.on('close', (code: number | null) => { + clearTimeout(timer); + resolve({ stdout, stderr, exitCode: code ?? -1 }); + }); + }); +} + +/** + * Creates the help command with access to all commands for display + */ +export function createHelpCommand(getAllCommands: () => CommandDefinition[]): CommandDefinition { + return { + name: 'help', + description: 'Show help information', + usage: '/help', + category: 'General', + aliases: ['h', '?'], + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<CommandHandlerResult> => { + const allCommands = getAllCommands(); + + // Build styled data for help + const styledData: HelpStyledData = { + commands: allCommands.map((cmd) => ({ + name: cmd.name, + description: cmd.description, + category: cmd.category || 'General', + })), + }; + + // Build fallback text + const fallbackLines: string[] = ['Available Commands:']; + for (const cmd of allCommands) { + fallbackLines.push(` /${cmd.name} - ${cmd.description}`); + } + + return CommandOutputHelper.styled('help', styledData, fallbackLines.join('\n')); + }, + }; +} + +/** + * General commands that are available across all contexts + * Note: The help command is created separately to avoid circular dependencies + */ +export const generalCommands: CommandDefinition[] = [ + { + name: 'shell', + description: 'Execute shell command directly (use !command as shortcut)', + usage: '!<command> or /shell <command>', + category: 'General', + handler: async ( + args: string[], + agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean | string> => { + const command = args.join(' ').trim(); + if (!command) { + return formatForInkCli('❌ No command provided. Usage: !<command>'); + } + + const cwd = process.cwd(); + agent.logger.debug(`Executing shell command: ${command}`); + + const { stdout, stderr, exitCode } = await executeShellCommand(command, cwd); + + // Build output + const lines: string[] = []; + + if (stdout.trim()) { + lines.push(stdout.trim()); + } + if (stderr.trim()) { + lines.push(chalk.yellow(stderr.trim())); + } + if (exitCode !== 0) { + lines.push(chalk.red(`Exit code: ${exitCode}`)); + } + + if (lines.length === 0) { + return formatForInkCli(chalk.gray('(no output)')); + } + + return formatForInkCli(lines.join('\n')); + }, + }, + { + name: 'exit', + description: 'Exit the CLI', + usage: '/exit', + category: 'General', + aliases: ['quit', 'q'], + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean | string> => { + console.log(chalk.rgb(255, 165, 0)('Exiting AI CLI. Goodbye!')); + process.exit(0); + }, + }, + { + name: 'new', + description: 'Start a new conversation', + usage: '/new', + category: 'General', + handler: async ( + _args: string[], + agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean | string> => { + try { + // Create a new session + const newSession = await agent.createSession(); + const newSessionId = newSession.id; + + // Emit session:created to switch the CLI to the new session + agent.agentEventBus.emit('session:created', { + sessionId: newSessionId, + switchTo: true, + }); + + return formatForInkCli( + `✨ New conversation started\n💡 Use /resume to see previous conversations` + ); + } catch (error) { + const errorMsg = `Failed to create new session: ${error instanceof Error ? error.message : String(error)}`; + agent.logger.error(errorMsg); + return formatForInkCli(`❌ ${errorMsg}`); + } + }, + }, + { + name: 'clear', + description: 'Continue conversation, free up AI context window', + usage: '/clear', + category: 'General', + handler: async ( + _args: string[], + agent: DextoAgent, + ctx: CommandContext + ): Promise<boolean | string> => { + try { + const { sessionId } = ctx; + if (!sessionId) { + return formatForInkCli('⚠️ No active session to clear'); + } + + // Clear context window - adds a marker so filterCompacted skips prior messages + // History stays in DB for review, but LLM won't see it + await agent.clearContext(sessionId); + + return formatForInkCli( + '🧹 Context window cleared\n💡 Conversation continues - AI will not see older messages' + ); + } catch (error) { + const errorMsg = `Failed to clear context: ${error instanceof Error ? error.message : String(error)}`; + agent.logger.error(errorMsg); + return formatForInkCli(`❌ ${errorMsg}`); + } + }, + }, + { + name: 'compact', + description: 'Compact context by summarizing older messages', + usage: '/compact', + category: 'General', + aliases: ['summarize'], + handler: async ( + _args: string[], + agent: DextoAgent, + ctx: CommandContext + ): Promise<boolean | string> => { + try { + const { sessionId } = ctx; + if (!sessionId) { + return formatForInkCli('⚠️ No active session to compact'); + } + + // Compact context - generates summary and hides older messages + // The context:compacting and context:compacted events are handled by useAgentEvents + // which shows the compacting indicator and notification message + const result = await agent.compactContext(sessionId); + + if (!result) { + return formatForInkCli( + '💡 Nothing to compact - history is too short or compaction is not configured.' + ); + } + + // Return true - notification is shown via context:compacted event handler + return true; + } catch (error) { + const errorMsg = `Failed to compact context: ${error instanceof Error ? error.message : String(error)}`; + agent.logger.error(errorMsg); + return formatForInkCli(`❌ ${errorMsg}`); + } + }, + }, + { + name: 'context', + description: 'Show context window usage statistics', + usage: '/context', + category: 'General', + aliases: ['ctx', 'tokens'], + handler: async ( + _args: string[], + agent: DextoAgent, + ctx: CommandContext + ): Promise<boolean | string> => { + try { + const { sessionId } = ctx; + if (!sessionId) { + return formatForInkCli('⚠️ No active session'); + } + + const stats = await agent.getContextStats(sessionId); + + // Create a visual progress bar (clamped to 0-100% for display) + const barWidth = 20; + const displayPercent = Math.min(Math.max(stats.usagePercent, 0), 100); + const filledWidth = Math.round((displayPercent / 100) * barWidth); + const emptyWidth = barWidth - filledWidth; + const progressBar = '█'.repeat(filledWidth) + '░'.repeat(emptyWidth); + + // Color based on usage + let usageColor = chalk.green; + if (stats.usagePercent > 80) usageColor = chalk.red; + else if (stats.usagePercent > 60) usageColor = chalk.yellow; + + // Helper to format token counts + const formatTokens = (tokens: number): string => { + if (tokens >= 1000) { + return `${(tokens / 1000).toFixed(1)}k`; + } + return tokens.toLocaleString(); + }; + + // Calculate auto compact buffer (reserved space before compaction triggers) + // maxContextTokens already has thresholdPercent applied, so we need to derive + // the buffer as: maxContextTokens * (1 - thresholdPercent) / thresholdPercent + const autoCompactBuffer = + stats.thresholdPercent > 0 && stats.thresholdPercent < 1.0 + ? Math.floor( + (stats.maxContextTokens * (1 - stats.thresholdPercent)) / + stats.thresholdPercent + ) + : 0; + const bufferPercent = Math.round((1 - stats.thresholdPercent) * 100); + const bufferLabel = + bufferPercent > 0 + ? `Auto compact buffer (${bufferPercent}%)` + : 'Auto compact buffer'; + + const totalTokenSpace = stats.maxContextTokens + autoCompactBuffer; + const usedTokens = stats.estimatedTokens + autoCompactBuffer; + + // Helper to calculate percentage of total token space + const pct = (tokens: number): string => { + const percent = + totalTokenSpace > 0 ? ((tokens / totalTokenSpace) * 100).toFixed(1) : '0.0'; + return `${percent}%`; + }; + + const overflowWarning = stats.usagePercent > 100 ? ' ⚠️ OVERFLOW' : ''; + const { breakdown } = stats; + + const tokenDisplay = `~${formatTokens(usedTokens)}`; + + const breakdownLabel = chalk.dim('(estimated)'); + const lines = [ + `📊 Context Usage`, + ` ${usageColor(progressBar)} ${stats.usagePercent}%${overflowWarning}`, + ` ${chalk.dim(stats.modelDisplayName)} · ${tokenDisplay} / ${formatTokens(totalTokenSpace)} tokens`, + ``, + ` ${chalk.cyan('Breakdown:')} ${breakdownLabel}`, + ` ├─ System prompt: ${formatTokens(breakdown.systemPrompt)} (${pct(breakdown.systemPrompt)})`, + ` ├─ Tools: ${formatTokens(breakdown.tools.total)} (${pct(breakdown.tools.total)})`, + ` ├─ Messages: ${formatTokens(breakdown.messages)} (${pct(breakdown.messages)})`, + ` └─ ${bufferLabel}: ${formatTokens(autoCompactBuffer)} (${pct(autoCompactBuffer)})`, + ``, + ` Messages: ${stats.filteredMessageCount} visible (${stats.messageCount} total)`, + ]; + + // Show pruned tool count if any + if (stats.prunedToolCount > 0) { + lines.push(` 🗑️ ${stats.prunedToolCount} tool output(s) pruned`); + } + + if (stats.hasSummary) { + lines.push(` 📦 Context has been compacted`); + } + + if (stats.usagePercent > 100) { + lines.push( + ` 💡 Use /compact to manually compact, or send a message to trigger auto-compaction` + ); + } + + return formatForInkCli(lines.join('\n')); + } catch (error) { + const errorMsg = `Failed to get context stats: ${error instanceof Error ? error.message : String(error)}`; + agent.logger.error(errorMsg); + return formatForInkCli(`❌ ${errorMsg}`); + } + }, + }, + { + name: 'copy', + description: 'Copy the last assistant response to clipboard', + usage: '/copy', + category: 'General', + aliases: ['cp'], + handler: async ( + _args: string[], + agent: DextoAgent, + ctx: CommandContext + ): Promise<boolean | string> => { + try { + const { sessionId } = ctx; + if (!sessionId) { + return formatForInkCli('❌ No active session'); + } + + // Get session history + const history = await agent.getSessionHistory(sessionId); + if (!history || history.length === 0) { + return formatForInkCli('❌ No messages in current session'); + } + + // Find the last assistant message + const lastAssistantMessage = [...history] + .reverse() + .find((msg) => msg.role === 'assistant'); + + if (!lastAssistantMessage) { + return formatForInkCli('❌ No assistant response to copy'); + } + + // Extract text content from the message + let textContent = ''; + if (typeof lastAssistantMessage.content === 'string') { + textContent = lastAssistantMessage.content; + } else if (Array.isArray(lastAssistantMessage.content)) { + // Handle multi-part content + textContent = lastAssistantMessage.content + .filter( + (part): part is { type: 'text'; text: string } => part.type === 'text' + ) + .map((part) => part.text) + .join('\n'); + } + + if (!textContent) { + return formatForInkCli('❌ No text content to copy'); + } + + // Copy to clipboard + const success = await writeToClipboard(textContent); + if (success) { + const preview = + textContent.length > 50 + ? textContent.substring(0, 50) + '...' + : textContent; + return formatForInkCli( + `📋 Copied to clipboard (${textContent.length} chars)\n${preview.replace(/\n/g, ' ')}` + ); + } else { + return formatForInkCli('❌ Failed to copy to clipboard'); + } + } catch (error) { + const errorMsg = `Failed to copy: ${error instanceof Error ? error.message : String(error)}`; + agent.logger.error(errorMsg); + return formatForInkCli(`❌ ${errorMsg}`); + } + }, + }, + { + name: 'shortcuts', + description: 'Show keyboard shortcuts', + usage: '/shortcuts', + category: 'General', + aliases: ['keys', 'hotkeys'], + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<CommandHandlerResult> => { + const styledData: ShortcutsStyledData = { + categories: [ + { + name: 'Global', + shortcuts: [ + { keys: 'Ctrl+C', description: 'Clear input, then exit (press twice)' }, + { keys: 'Ctrl+T', description: 'Toggle task list (show/hide tasks)' }, + { keys: 'Escape', description: 'Cancel processing / close overlay' }, + ], + }, + { + name: 'Input', + shortcuts: [ + { keys: 'Enter', description: 'Submit message' }, + { keys: 'Shift+Enter', description: 'New line (multi-line input)' }, + { keys: 'Up/Down', description: 'Navigate input history' }, + { keys: 'Ctrl+R', description: 'Search history (enter search mode)' }, + { keys: 'Tab', description: 'Autocomplete command' }, + { keys: 'Ctrl+U', description: 'Clear input line' }, + { keys: 'Ctrl+W', description: 'Delete word before cursor' }, + { keys: 'Ctrl+A', description: 'Move cursor to start' }, + { keys: 'Ctrl+E', description: 'Move cursor to end' }, + ], + }, + { + name: 'History Search (after Ctrl+R)', + shortcuts: [ + { keys: 'Ctrl+R', description: 'Next older match' }, + { keys: 'Ctrl+E', description: 'Next newer match' }, + { keys: 'Enter', description: 'Accept and exit search' }, + { keys: 'Escape', description: 'Cancel search' }, + ], + }, + { + name: 'Autocomplete & Selectors', + shortcuts: [ + { keys: 'Up/Down', description: 'Navigate options' }, + { keys: 'Enter', description: 'Select / execute' }, + { keys: 'Tab', description: 'Load command into input' }, + { keys: 'Escape', description: 'Close overlay' }, + ], + }, + { + name: 'Tool Approval', + shortcuts: [ + { keys: 'y', description: 'Allow once' }, + { keys: 'a', description: 'Allow for session' }, + { keys: 'n', description: 'Deny' }, + { keys: 'Escape', description: 'Cancel' }, + ], + }, + ], + }; + + // Build fallback text + const fallbackLines: string[] = ['Keyboard Shortcuts:']; + for (const category of styledData.categories) { + fallbackLines.push(`\n${category.name}:`); + for (const shortcut of category.shortcuts) { + fallbackLines.push(` ${shortcut.keys.padEnd(14)} ${shortcut.description}`); + } + } + + return CommandOutputHelper.styled('shortcuts', styledData, fallbackLines.join('\n')); + }, + }, +]; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/mcp/index.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/mcp/index.ts new file mode 100644 index 00000000..753c65eb --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/mcp/index.ts @@ -0,0 +1,21 @@ +/** + * MCP Commands Module + * + * In interactive CLI, /mcp always shows the interactive MCP server list overlay. + * This command definition exists for autocomplete and help display. + */ + +import type { CommandDefinition } from '../command-parser.js'; +import { overlayOnlyHandler } from '../command-parser.js'; + +/** + * MCP management command definition. + * Handler is never called - mcp is in ALWAYS_OVERLAY and handled by McpServerList overlay. + */ +export const mcpCommands: CommandDefinition = { + name: 'mcp', + description: 'Manage MCP servers (interactive)', + usage: '/mcp', + category: 'MCP Management', + handler: overlayOnlyHandler, +}; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/model/index.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/model/index.ts new file mode 100644 index 00000000..bf76bbb1 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/model/index.ts @@ -0,0 +1,22 @@ +/** + * Model Commands Module + * + * In interactive CLI, /model always shows the interactive model selector overlay. + * This command definition exists for autocomplete and help display. + */ + +import type { CommandDefinition } from '../command-parser.js'; +import { overlayOnlyHandler } from '../command-parser.js'; + +/** + * Model management command definition. + * Handler is never called - model is in ALWAYS_OVERLAY and handled by ModelSelector overlay. + */ +export const modelCommands: CommandDefinition = { + name: 'model', + description: 'Switch AI model (interactive selector)', + usage: '/model', + category: 'General', + aliases: ['m'], + handler: overlayOnlyHandler, +}; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/plugin/index.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/plugin/index.ts new file mode 100644 index 00000000..97185294 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/plugin/index.ts @@ -0,0 +1,21 @@ +/** + * Plugin Commands Module + * + * In interactive CLI, /plugin always shows the interactive plugin manager overlay. + * This command definition exists for autocomplete and help display. + */ + +import type { CommandDefinition } from '../command-parser.js'; +import { overlayOnlyHandler } from '../command-parser.js'; + +/** + * Plugin management command definition. + * Handler is never called - plugin is in ALWAYS_OVERLAY and handled by PluginManager overlay. + */ +export const pluginCommands: CommandDefinition = { + name: 'plugin', + description: 'Manage plugins (interactive)', + usage: '/plugin', + category: 'Plugin Management', + handler: overlayOnlyHandler, +}; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/prompt-commands.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/prompt-commands.ts new file mode 100644 index 00000000..8263a6f9 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/prompt-commands.ts @@ -0,0 +1,404 @@ +/** + * Prompt Commands Module + * + * This module defines prompt management slash commands for the Dexto CLI interface. + * These commands provide functionality for viewing and managing system prompts. + * + * Available Prompt Commands: + * - /sysprompt - Display the current system prompt + * - /prompts - List all available prompts (MCP + internal) + * - /<prompt-name> [args] - Direct prompt execution (auto-generated for each prompt) + */ + +import chalk from 'chalk'; +import type { DextoAgent, PromptInfo } from '@dexto/core'; +import type { CommandDefinition, CommandContext, CommandHandlerResult } from './command-parser.js'; +import { formatForInkCli } from './utils/format-output.js'; +import { createSendMessageMarker, type StyledOutput } from '../../ink-cli/services/index.js'; +// Avoid depending on core types to keep CLI typecheck independent of build + +/** + * Prompt management commands + */ +export const promptCommands: CommandDefinition[] = [ + { + name: 'sysprompt', + description: 'Display the current system prompt', + usage: '/sysprompt', + category: 'Prompt Management', + handler: async ( + _args: string[], + agent: DextoAgent, + _ctx: CommandContext + ): Promise<CommandHandlerResult> => { + try { + const systemPrompt = await agent.getSystemPrompt(); + + // Return styled output for ink-cli + const styledOutput: StyledOutput = { + styledType: 'sysprompt', + styledData: { content: systemPrompt }, + fallbackText: `System Prompt:\n${systemPrompt}`, + }; + + return styledOutput; + } catch (error) { + const errorMsg = `Failed to get system prompt: ${error instanceof Error ? error.message : String(error)}`; + agent.logger.error(errorMsg); + return formatForInkCli(`❌ ${errorMsg}`); + } + }, + }, + { + name: 'prompts', + description: 'Browse, add, and delete prompts', + usage: '/prompts', + category: 'Prompt Management', + handler: async ( + _args: string[], + agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean | string> => { + try { + const prompts = await agent.listPrompts(); + const promptNames = Object.keys(prompts || {}); + + if (promptNames.length === 0) { + const output = '\n⚠️ No prompts available'; + console.log(chalk.rgb(255, 165, 0)(output)); + return formatForInkCli(output); + } + + // Build output string + const outputLines: string[] = ['\n📝 Available Prompts:\n']; + + // Group by source + const mcpPrompts: string[] = []; + const configPrompts: string[] = []; + const customPrompts: string[] = []; + + for (const [name, info] of Object.entries(prompts)) { + if (info.source === 'mcp') { + mcpPrompts.push(name); + } else if (info.source === 'config') { + configPrompts.push(name); + } else if (info.source === 'custom') { + customPrompts.push(name); + } + } + + if (mcpPrompts.length > 0) { + outputLines.push('🔗 MCP Prompts:'); + mcpPrompts.forEach((name) => { + const info = prompts[name]; + if (info) { + const displayName = info.displayName || name; + const title = + info.title && info.title !== displayName ? ` (${info.title})` : ''; + const desc = info.description ? ` - ${info.description}` : ''; + const args = + info.arguments && info.arguments.length > 0 + ? ` [args: ${info.arguments + .map((a) => `${a.name}${a.required ? '*' : ''}`) + .join(', ')}]` + : ''; + outputLines.push(` ${displayName}${title}${desc}${args}`); + } + }); + outputLines.push(''); + } + + if (configPrompts.length > 0) { + outputLines.push('📋 Config Prompts:'); + configPrompts.forEach((name) => { + const info = prompts[name]; + if (info) { + const displayName = info.displayName || name; + const title = + info.title && info.title !== displayName ? ` (${info.title})` : ''; + const desc = info.description ? ` - ${info.description}` : ''; + outputLines.push(` ${displayName}${title}${desc}`); + } + }); + outputLines.push(''); + } + + if (customPrompts.length > 0) { + outputLines.push('✨ Custom Prompts:'); + customPrompts.forEach((name) => { + const info = prompts[name]; + if (info) { + const displayName = info.displayName || name; + const title = + info.title && info.title !== displayName ? ` (${info.title})` : ''; + const desc = info.description ? ` - ${info.description}` : ''; + outputLines.push(` ${displayName}${title}${desc}`); + } + }); + outputLines.push(''); + } + + outputLines.push(`Total: ${promptNames.length} prompts`); + const output = outputLines.join('\n'); + + // Log for regular CLI (with chalk formatting) + console.log(chalk.bold.green('\n📝 Available Prompts:\n')); + if (mcpPrompts.length > 0) { + console.log(chalk.cyan('🔗 MCP Prompts:')); + mcpPrompts.forEach((name) => { + const info = prompts[name]; + if (info) { + const displayName = info.displayName || name; + const title = + info.title && info.title !== displayName ? ` (${info.title})` : ''; + const desc = info.description ? ` - ${info.description}` : ''; + const args = + info.arguments && info.arguments.length > 0 + ? ` [args: ${info.arguments + .map((a) => `${a.name}${a.required ? '*' : ''}`) + .join(', ')}]` + : ''; + console.log( + ` ${chalk.blue(displayName)}${chalk.rgb(255, 165, 0)(title)}${chalk.gray(desc)}${chalk.gray(args)}` + ); + } + }); + console.log(); + } + if (configPrompts.length > 0) { + console.log(chalk.cyan('📋 Config Prompts:')); + configPrompts.forEach((name) => { + const info = prompts[name]; + if (info) { + const displayName = info.displayName || name; + const title = + info.title && info.title !== displayName ? ` (${info.title})` : ''; + const desc = info.description ? ` - ${info.description}` : ''; + console.log( + ` ${chalk.blue(displayName)}${chalk.rgb(255, 165, 0)(title)}${chalk.gray(desc)}` + ); + } + }); + console.log(); + } + if (customPrompts.length > 0) { + console.log(chalk.greenBright('✨ Custom Prompts:')); + customPrompts.forEach((name) => { + const info = prompts[name]; + if (info) { + const displayName = info.displayName || name; + const title = + info.title && info.title !== displayName ? ` (${info.title})` : ''; + const desc = info.description ? ` - ${info.description}` : ''; + console.log( + ` ${chalk.blue(displayName)}${chalk.rgb(255, 165, 0)(title)}${chalk.gray(desc)}` + ); + } + }); + console.log(); + } + console.log(chalk.gray(`Total: ${promptNames.length} prompts`)); + console.log(chalk.gray('💡 Use /<prompt-name> to execute a prompt directly\n')); + + return formatForInkCli(output); + } catch (error) { + const errorMsg = `Error listing prompts: ${error instanceof Error ? error.message : String(error)}`; + console.error(chalk.red(`❌ ${errorMsg}`)); + return formatForInkCli(`❌ ${errorMsg}`); + } + }, + }, + // Note: /use command removed - use /<prompt-name> directly instead + // Prompts are automatically registered as slash commands (see getDynamicPromptCommands) +]; + +/** + * Create a dynamic command definition from a prompt + * @param promptInfo The prompt metadata with pre-computed commandName + */ +function createPromptCommand(promptInfo: PromptInfo): CommandDefinition { + // Use pre-computed commandName (collision-resolved by PromptManager) + // Fall back to displayName or name for backwards compatibility + const commandName = promptInfo.commandName || promptInfo.displayName || promptInfo.name; + // Keep internal name for prompt resolution (e.g., "config:review" or "mcp:server1:review") + const internalName = promptInfo.name; + // Base name for display purposes (without source prefix) + const baseName = promptInfo.displayName || promptInfo.name; + + return { + name: commandName, + description: promptInfo.description || `Execute ${baseName} prompt`, + usage: `/${commandName} [context]`, + category: 'Dynamic Prompts', + handler: async ( + args: string[], + agent: DextoAgent, + ctx: CommandContext + ): Promise<CommandHandlerResult> => { + try { + const { argMap, context: contextString } = splitPromptArguments(args); + + // Use resolvePrompt instead of getPrompt + flattenPromptResult (matches WebUI approach) + const resolveOptions: { + args?: Record<string, unknown>; + context?: string; + } = {}; + if (Object.keys(argMap).length > 0) { + resolveOptions.args = argMap; + } + if (contextString) { + resolveOptions.context = contextString; + } + // Use internal name for resolution (includes prefix like "config:") + const result = await agent.resolvePrompt(internalName, resolveOptions); + + // Apply per-prompt overrides (Phase 2 Claude Code compatibility) + // These overrides persist for the session until explicitly cleared + if (ctx.sessionId) { + // Apply model override if specified + if (result.model) { + console.log( + chalk.gray(`🔄 Switching model to '${result.model}' for this prompt`) + ); + try { + await agent.switchLLM({ model: result.model }, ctx.sessionId); + } catch (modelError) { + console.log( + chalk.yellow( + `⚠️ Failed to switch model: ${modelError instanceof Error ? modelError.message : String(modelError)}` + ) + ); + } + } + + // Apply auto-approve tools if specified + // These tools will skip confirmation prompts during skill execution + if (result.allowedTools && result.allowedTools.length > 0) { + console.log( + chalk.gray(`🔓 Auto-approving tools: ${result.allowedTools.join(', ')}`) + ); + try { + agent.toolManager.setSessionAutoApproveTools( + ctx.sessionId, + result.allowedTools + ); + } catch (toolError) { + console.log( + chalk.yellow( + `⚠️ Failed to set auto-approve tools: ${toolError instanceof Error ? toolError.message : String(toolError)}` + ) + ); + } + } + } + + // Fork skills: route through LLM to call invoke_skill + // This ensures approval flow and context management work naturally through + // processStream, rather than bypassing it with direct tool execution. + if (result.context === 'fork') { + const skillName = internalName; + const taskContext = contextString || ''; + + // Build instruction message for the LLM + // The <skill-invocation> tags help the LLM recognize this is a structured request + const instructionText = `<skill-invocation> +Execute the fork skill: ${commandName} +${taskContext ? `Task context: ${taskContext}` : ''} + +Call the internal--invoke_skill tool immediately with: +- skill: "${skillName}" +${taskContext ? `- taskContext: "${taskContext}"` : ''} +</skill-invocation>`; + + return createSendMessageMarker(instructionText); + } + + // Inline skills: wrap content in <skill-invocation> for clean history display + // Convert resource URIs to @resource mentions so agent.run() can expand them + let finalText = result.text; + if (result.resources.length > 0) { + // Append resource references as @<uri> format + const resourceRefs = result.resources.map((uri) => `@<${uri}>`).join(' '); + finalText = finalText ? `${finalText}\n\n${resourceRefs}` : resourceRefs; + } + + if (finalText.trim()) { + // Wrap in <skill-invocation> tags for clean display in history + // The tags help formatSkillInvocationMessage() detect and format these + const taskContext = contextString || ''; + const wrappedText = `<skill-invocation> +Execute the inline skill: ${commandName} +${taskContext ? `Task context: ${taskContext}` : ''} + +skill: "${internalName}" +</skill-invocation> + +${finalText.trim()}`; + + return createSendMessageMarker(wrappedText); + } else { + const warningMsg = `⚠️ Prompt '${commandName}' returned no content`; + console.log(chalk.rgb(255, 165, 0)(warningMsg)); + return formatForInkCli(warningMsg); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + agent.logger.error( + `Failed to execute prompt command '${commandName}': ${errorMessage}` + ); + + const errorMsg = `❌ Error executing prompt '${commandName}': ${errorMessage}`; + return formatForInkCli(errorMsg); + } + }, + }; +} + +/** + * Get all dynamic prompt commands based on available prompts. + * Uses pre-computed commandName from PromptManager for collision handling. + * Filters out prompts with `userInvocable: false` as these are not intended + * for direct user invocation via slash commands. + */ +export async function getDynamicPromptCommands(agent: DextoAgent): Promise<CommandDefinition[]> { + try { + const prompts = await agent.listPrompts(); + // Filter out prompts that are not user-invocable (userInvocable: false) + // These prompts are intended for LLM auto-invocation only, not CLI slash commands + const promptEntries = Object.entries(prompts).filter( + ([, info]) => info.userInvocable !== false + ); + + // Create commands using pre-computed commandName (collision-resolved by PromptManager) + return promptEntries.map(([, info]) => createPromptCommand(info)); + } catch (error) { + agent.logger.error( + `Failed to get dynamic prompt commands: ${error instanceof Error ? error.message : String(error)}` + ); + return []; + } +} + +function splitPromptArguments(args: string[]): { + argMap: Record<string, string>; + context?: string | undefined; +} { + const map: Record<string, string> = {}; + const contextParts: string[] = []; + + for (const arg of args) { + const equalsIndex = arg.indexOf('='); + if (equalsIndex > 0) { + const key = arg.slice(0, equalsIndex).trim(); + const value = arg.slice(equalsIndex + 1); + if (key.length > 0) { + map[key] = value; + } + } else if (arg.trim().length > 0) { + contextParts.push(arg); + } + } + + const context = contextParts.length > 0 ? contextParts.join(' ') : undefined; + return { argMap: map, context }; +} diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/session/index.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/session/index.ts new file mode 100644 index 00000000..506f04be --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/session/index.ts @@ -0,0 +1,15 @@ +/** + * Session Commands Module (Interactive CLI) + * + * This module provides session-related commands for the interactive CLI. + * These commands use interactive overlays/selectors rather than subcommands. + * + * Exports: + * - searchCommand: Opens interactive search overlay + * - resumeCommand: Shows interactive session selector + * - renameCommand: Rename the current session + * + * Note: For headless CLI session management, see src/cli/commands/session-commands.ts + */ + +export { searchCommand, resumeCommand, renameCommand } from './session-commands.js'; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/session/session-commands.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/session/session-commands.ts new file mode 100644 index 00000000..a0b255d2 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/session/session-commands.ts @@ -0,0 +1,86 @@ +/** + * Session Management Commands (Interactive CLI) + * + * This module contains session-related commands for the interactive CLI. + * Session management in interactive mode uses overlays/selectors rather than subcommands. + * + * Commands: + * - resume: Shows interactive session selector + * - search: Opens interactive search overlay + * - rename: Rename the current session + * + * Note: For headless CLI session management (list, history, delete), + * see src/cli/commands/session-commands.ts + */ + +import chalk from 'chalk'; +import type { DextoAgent } from '@dexto/core'; +import type { CommandDefinition, CommandContext } from '../command-parser.js'; + +/** + * Resume command - shows interactive session selector + * Note: In interactive CLI, this always shows the selector (args ignored) + * For headless CLI, use `dexto -r <sessionId>` instead + */ +export const resumeCommand: CommandDefinition = { + name: 'resume', + description: 'Switch to a different session (interactive selector)', + usage: '/resume', + category: 'General', + aliases: ['r'], + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean | string> => { + // In interactive CLI, /resume always triggers the interactive selector + // The selector is shown via detectInteractiveSelector in inputParsing.ts + // This handler should not be called in ink-cli (selector shows instead) + const helpText = [ + '📋 Resume Session', + '\nType /resume to show the session selector\n', + ].join('\n'); + + console.log(chalk.blue(helpText)); + return helpText; + }, +}; + +/** + * Standalone search command - opens interactive search overlay + */ +export const searchCommand: CommandDefinition = { + name: 'search', + description: 'Search messages across all sessions', + usage: '/search', + category: 'General', + aliases: ['find'], + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean> => { + // Interactive overlay handles everything - just return success + return true; + }, +}; + +/** + * Rename command - rename the current session + * In interactive CLI, this shows the rename overlay. + * The overlay is triggered via commandOverlays.ts registry. + */ +export const renameCommand: CommandDefinition = { + name: 'rename', + description: 'Rename the current session', + usage: '/rename', + category: 'General', + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean> => { + // Interactive overlay handles everything - just return success + return true; + }, +}; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/system/index.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/system/index.ts new file mode 100644 index 00000000..7296bb9f --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/system/index.ts @@ -0,0 +1,13 @@ +/** + * System Commands Module Index + * + * This module exports all system-level CLI commands for the Dexto interactive interface. + * System commands provide functionality for configuration, logging, and statistics. + * + * Available commands: + * - /log - Set or view log level + * - /config - Show current configuration + * - /stats - Show system statistics + */ + +export { systemCommands } from './system-commands.js'; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/system/system-commands.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/system/system-commands.ts new file mode 100644 index 00000000..2ce9f908 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/system/system-commands.ts @@ -0,0 +1,256 @@ +/** + * System Commands Module + * + * This module defines system-level slash commands for the Dexto CLI interface. + * These commands provide system configuration, logging, and statistics functionality. + * + * Available System Commands: + * - /log [level] - Set or view log level + * - /config - Show current configuration + * - /stats - Show system statistics + * - /stream - Toggle streaming mode for LLM responses + */ + +import chalk from 'chalk'; +import { logger, type DextoAgent } from '@dexto/core'; +import type { CommandDefinition, CommandHandlerResult, CommandContext } from '../command-parser.js'; +import { formatForInkCli } from '../utils/format-output.js'; +import { CommandOutputHelper } from '../utils/command-output.js'; +import type { ConfigStyledData, StatsStyledData } from '../../../ink-cli/state/types.js'; + +/** + * System commands for configuration and monitoring + */ +export const systemCommands: CommandDefinition[] = [ + { + name: 'log', + description: 'View or change log level interactively', + usage: '/log [level]', + category: 'System', + aliases: [], + handler: async ( + args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean | string> => { + const validLevels = ['error', 'warn', 'info', 'http', 'verbose', 'debug', 'silly']; + const level = args[0]; + + if (!level) { + // Interactive view: show current level and options + const currentLevel = logger.getLevel(); + const logFilePath = logger.getLogFilePath(); + + console.log(chalk.bold.blue('\n📊 Logging Configuration:\n')); + console.log(` Current level: ${chalk.green.bold(currentLevel)}`); + if (logFilePath) { + console.log(` Log file: ${chalk.cyan(logFilePath)}`); + } + console.log(chalk.gray('\n Available levels (from least to most verbose):')); + validLevels.forEach((lvl) => { + const isCurrent = lvl === currentLevel; + const marker = isCurrent ? chalk.green('▶') : ' '; + const levelText = isCurrent ? chalk.green.bold(lvl) : chalk.gray(lvl); + console.log(` ${marker} ${levelText}`); + }); + console.log( + chalk.gray('\n 💡 Use /log <level> to change level (e.g., /log debug)\n') + ); + + const output = [ + '\n📊 Logging Configuration:', + `Current level: ${currentLevel}`, + logFilePath ? `Log file: ${logFilePath}` : '', + '\nAvailable levels: error, warn, info, http, verbose, debug, silly', + '💡 Use /log <level> to change level', + ] + .filter(Boolean) + .join('\n'); + + return formatForInkCli(output); + } + + if (validLevels.includes(level)) { + logger.setLevel(level); + logger.info(`Log level set to ${level}`, null, 'green'); + console.log(chalk.green(`✅ Log level changed to: ${chalk.bold(level)}`)); + const output = `✅ Log level set to ${level}`; + return formatForInkCli(output); + } else { + const errorMsg = `❌ Invalid log level: ${level}\nValid levels: ${validLevels.join(', ')}`; + console.log(chalk.red(`❌ Invalid log level: ${chalk.bold(level)}`)); + console.log(chalk.gray(`Valid levels: ${validLevels.join(', ')}`)); + return formatForInkCli(errorMsg); + } + }, + }, + { + name: 'config', + description: 'Show current configuration', + usage: '/config', + category: 'System', + handler: async ( + _args: string[], + agent: DextoAgent, + _ctx: CommandContext + ): Promise<CommandHandlerResult> => { + try { + const config = agent.getEffectiveConfig(); + const servers = Object.keys(config.mcpServers || {}); + + // Get config file path (may not exist for programmatic agents) + let configFilePath: string | null = null; + try { + configFilePath = agent.getAgentFilePath(); + } catch { + // No config file path available + } + + // Get enabled plugins + const pluginsEnabled: string[] = []; + if (config.plugins) { + // Check built-in plugins + if (config.plugins.contentPolicy?.enabled) { + pluginsEnabled.push('contentPolicy'); + } + if (config.plugins.responseSanitizer?.enabled) { + pluginsEnabled.push('responseSanitizer'); + } + // Check custom plugins + for (const plugin of config.plugins.custom || []) { + if (plugin.enabled) { + pluginsEnabled.push(plugin.name); + } + } + } + + // Build styled data + const styledData: ConfigStyledData = { + configFilePath, + provider: config.llm.provider, + model: config.llm.model, + maxTokens: config.llm.maxOutputTokens ?? null, + temperature: config.llm.temperature ?? null, + toolConfirmationMode: config.toolConfirmation?.mode || 'auto', + maxSessions: config.sessions?.maxSessions?.toString() || 'Default', + sessionTTL: config.sessions?.sessionTTL + ? `${config.sessions.sessionTTL / 1000}s` + : 'Default', + mcpServers: servers, + promptsCount: config.prompts?.length || 0, + pluginsEnabled, + }; + + // Build fallback text (no console.log - interferes with Ink rendering) + const fallbackLines: string[] = [ + 'Configuration:', + configFilePath ? ` Config: ${configFilePath}` : '', + ` LLM: ${config.llm.provider} / ${config.llm.model}`, + ` Tool Confirmation: ${styledData.toolConfirmationMode}`, + ` Sessions: max=${styledData.maxSessions}, ttl=${styledData.sessionTTL}`, + ` MCP Servers: ${servers.length > 0 ? servers.join(', ') : 'none'}`, + ` Prompts: ${styledData.promptsCount}`, + ].filter(Boolean); + + return CommandOutputHelper.styled('config', styledData, fallbackLines.join('\n')); + } catch (error) { + const errorMsg = `Failed to get configuration: ${error instanceof Error ? error.message : String(error)}`; + agent.logger.error(errorMsg); + return formatForInkCli(`❌ ${errorMsg}`); + } + }, + }, + { + name: 'stats', + description: 'Show system statistics', + usage: '/stats', + category: 'System', + handler: async ( + _args: string[], + agent: DextoAgent, + ctx: CommandContext + ): Promise<CommandHandlerResult> => { + try { + // Session stats + const sessionStats = await agent.sessionManager.getSessionStats(); + + // MCP stats + const connectedServers = agent.getMcpClients().size; + const failedConnections = Object.keys(agent.getMcpFailedConnections()).length; + + // Tools + let toolCount = 0; + try { + const tools = await agent.getAllMcpTools(); + toolCount = Object.keys(tools).length; + } catch { + // Ignore - toolCount stays 0 + } + + // Get token usage from current session metadata + let tokenUsage: StatsStyledData['tokenUsage']; + let estimatedCost: number | undefined; + if (ctx.sessionId) { + const sessionMetadata = await agent.sessionManager.getSessionMetadata( + ctx.sessionId + ); + if (sessionMetadata?.tokenUsage) { + tokenUsage = sessionMetadata.tokenUsage; + } + estimatedCost = sessionMetadata?.estimatedCost; + } + + // Build styled data + const styledData: StatsStyledData = { + sessions: { + total: sessionStats.totalSessions, + inMemory: sessionStats.inMemorySessions, + maxAllowed: sessionStats.maxSessions, + }, + mcp: { + connected: connectedServers, + failed: failedConnections, + toolCount, + }, + ...(tokenUsage && { tokenUsage }), + ...(estimatedCost !== undefined && { estimatedCost }), + }; + + // Build fallback text + const fallbackLines: string[] = [ + 'System Statistics:', + ` Sessions: ${sessionStats.totalSessions} total, ${sessionStats.inMemorySessions} in memory`, + ` MCP: ${connectedServers} connected, ${toolCount} tools`, + ]; + if (failedConnections > 0) { + fallbackLines.push(` Failed connections: ${failedConnections}`); + } + if (tokenUsage) { + fallbackLines.push( + ` Tokens: ${tokenUsage.totalTokens} total (${tokenUsage.inputTokens} in, ${tokenUsage.outputTokens} out)` + ); + } + + return CommandOutputHelper.styled('stats', styledData, fallbackLines.join('\n')); + } catch (error) { + const errorMsg = `Failed to get statistics: ${error instanceof Error ? error.message : String(error)}`; + agent.logger.error(errorMsg); + return formatForInkCli(`❌ ${errorMsg}`); + } + }, + }, + { + name: 'stream', + description: 'Toggle streaming mode for LLM responses', + usage: '/stream', + category: 'System', + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean | string> => { + // Overlay is handled via commandOverlays.ts mapping + return true; + }, + }, +]; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/tool-commands.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/tool-commands.ts new file mode 100644 index 00000000..a88e32e1 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/tool-commands.ts @@ -0,0 +1,32 @@ +/** + * Tool Commands Module + * + * This module defines tool management slash commands for the Dexto CLI interface. + * These commands provide functionality for listing and managing MCP tools. + * + * Available Tool Commands: + * - /tools - Interactive tool browser + */ + +import type { DextoAgent } from '@dexto/core'; +import type { CommandDefinition, CommandContext } from './command-parser.js'; + +/** + * Tool management commands + */ +export const toolCommands: CommandDefinition[] = [ + { + name: 'tools', + description: 'Browse available tools interactively', + usage: '/tools', + category: 'Tool Management', + handler: async ( + _args: string[], + _agent: DextoAgent, + _ctx: CommandContext + ): Promise<boolean | string> => { + // Overlay is handled via commandOverlays.ts mapping + return true; + }, + }, +]; diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/utils/arg-parser.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/utils/arg-parser.ts new file mode 100644 index 00000000..1f21b38e --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/utils/arg-parser.ts @@ -0,0 +1,98 @@ +/** + * Argument Parsing Utilities + * + * This module provides utilities for parsing command-line arguments in the + * interactive CLI commands. These utilities are designed to be reusable + * across different command modules. + */ + +/** + * Result of parsing command-line arguments + */ +export interface ParsedArguments { + /** Regular arguments (non-option arguments) */ + parsedArgs: string[]; + /** Options in --key=value format */ + options: Record<string, string>; + /** Flags in --flag format (boolean options) */ + flags: Set<string>; +} + +/** + * Parse command-line arguments supporting --key=value options and --flag style flags. + * + * This function separates regular arguments from options and flags: + * - Regular args: any argument that doesn't start with -- + * - Options: --key=value format, parsed into options object + * - Flags: --flag format (no value), parsed into flags set + * + * @param args Array of command-line arguments to parse + * @returns Object containing parsed arguments, options, and flags + * + * @example + * ```typescript + * const result = parseOptions(['server', 'cmd', '--timeout=5000', '--verbose']); + * // result.parsedArgs: ['server', 'cmd'] + * // result.options: { timeout: '5000' } + * // result.flags: Set(['verbose']) + * ``` + */ +export function parseOptions(args: string[]): ParsedArguments { + const parsedArgs: string[] = []; + const options: Record<string, string> = {}; + const flags: Set<string> = new Set(); + + for (const arg of args) { + if (arg.startsWith('--')) { + if (arg.includes('=')) { + // Handle --key=value format + const [key, ...valueParts] = arg.slice(2).split('='); + if (key) { + // Rejoin value parts in case the value contained '=' characters + options[key] = valueParts.join('='); + } + } else { + // Handle --flag format (boolean flags) + flags.add(arg.slice(2)); + } + } else { + // Regular argument (not an option) + parsedArgs.push(arg); + } + } + + return { parsedArgs, options, flags }; +} + +/** + * Convert parsed options back to command-line argument format. + * Useful for debugging or reconstructing command lines. + * + * @param parsed The parsed arguments object + * @returns Array of command-line arguments + * + * @example + * ```typescript + * const args = reconstructArgs({ + * parsedArgs: ['server', 'cmd'], + * options: { timeout: '5000' }, + * flags: new Set(['verbose']) + * }); + * // Result: ['server', 'cmd', '--timeout=5000', '--verbose'] + * ``` + */ +export function reconstructArgs(parsed: ParsedArguments): string[] { + const result: string[] = [...parsed.parsedArgs]; + + // Add options in --key=value format + for (const [key, value] of Object.entries(parsed.options)) { + result.push(`--${key}=${value}`); + } + + // Add flags in --flag format + for (const flag of parsed.flags) { + result.push(`--${flag}`); + } + + return result; +} diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/utils/command-output.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/utils/command-output.ts new file mode 100644 index 00000000..ae8ba61f --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/utils/command-output.ts @@ -0,0 +1,69 @@ +/** + * Command Output Helper + * Utilities for consistent command output handling across all slash commands + */ + +import { formatForInkCli } from './format-output.js'; +import type { StyledOutput } from '../../../ink-cli/services/CommandService.js'; +import type { StyledMessageType, StyledData } from '../../../ink-cli/state/types.js'; + +/** + * Command output helper for consistent display and error handling + * Returns formatted strings for ink-cli to render (no direct console output) + */ +export class CommandOutputHelper { + /** + * Format success message for ink-cli to display + */ + static success(message: string): string { + return formatForInkCli(message); + } + + /** + * Format info message for ink-cli to display + */ + static info(message: string): string { + return formatForInkCli(message); + } + + /** + * Format warning message for ink-cli to display + */ + static warning(message: string): string { + return formatForInkCli(`⚠️ ${message}`); + } + + /** + * Format error message for ink-cli to display + */ + static error(error: unknown, context?: string): string { + const errorMessage = error instanceof Error ? error.message : String(error); + const fullMessage = context ? `❌ ${context}: ${errorMessage}` : `❌ ${errorMessage}`; + return formatForInkCli(fullMessage); + } + + /** + * Format multi-line output for ink-cli to display + */ + static output(lines: string[]): string { + return formatForInkCli(lines.join('\n')); + } + + /** + * Create styled output for rich rendering in ink-cli + * @param styledType - The type of styled rendering + * @param styledData - The structured data for rendering + * @param fallbackText - Plain text fallback for logging/non-ink environments + */ + static styled( + styledType: StyledMessageType, + styledData: StyledData, + fallbackText: string + ): StyledOutput { + return { + styledType, + styledData, + fallbackText: formatForInkCli(fallbackText), + }; + } +} diff --git a/dexto/packages/cli/src/cli/commands/interactive-commands/utils/format-output.ts b/dexto/packages/cli/src/cli/commands/interactive-commands/utils/format-output.ts new file mode 100644 index 00000000..71f95c95 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/interactive-commands/utils/format-output.ts @@ -0,0 +1,21 @@ +/** + * Utility functions for formatting command output for ink-cli + * Strips chalk formatting and provides plain text versions + */ + +import { stripVTControlCharacters } from 'node:util'; + +/** + * Strip ANSI color codes from a string + */ +export function stripAnsi(str: string): string { + return stripVTControlCharacters(str); +} + +/** + * Format output for ink-cli (removes chalk formatting) + * This allows commands to use chalk for regular CLI while returning plain text for ink-cli + */ +export function formatForInkCli(output: string): string { + return stripAnsi(output); +} diff --git a/dexto/packages/cli/src/cli/commands/list-agents.ts b/dexto/packages/cli/src/cli/commands/list-agents.ts new file mode 100644 index 00000000..8a284cc1 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/list-agents.ts @@ -0,0 +1,288 @@ +// packages/cli/src/cli/commands/list-agents.ts + +import { existsSync } from 'fs'; +import { promises as fs } from 'fs'; +import path from 'path'; +import chalk from 'chalk'; +import { z } from 'zod'; +import { + getDextoGlobalPath, + globalPreferencesExist, + loadGlobalPreferences, + loadBundledRegistryAgents, +} from '@dexto/agent-management'; + +// Zod schema for list-agents command validation +const ListAgentsCommandSchema = z + .object({ + verbose: z.boolean().default(false), + installed: z.boolean().default(false), + available: z.boolean().default(false), + }) + .strict(); + +export type ListAgentsCommandOptions = z.output<typeof ListAgentsCommandSchema>; +export type ListAgentsCommandOptionsInput = z.input<typeof ListAgentsCommandSchema>; + +/** + * Information about an installed agent + */ +interface InstalledAgentInfo { + name: string; + description: string; + path: string; + llmProvider?: string; + llmModel?: string; + installedAt?: Date; +} + +/** + * Information about an available agent from registry + */ +interface AvailableAgentInfo { + name: string; + description: string; + author: string; + tags: string[]; + type: 'builtin' | 'custom'; +} + +/** + * Get information about installed agents + */ +async function getInstalledAgents(): Promise<InstalledAgentInfo[]> { + const globalAgentsDir = getDextoGlobalPath('agents'); + + if (!existsSync(globalAgentsDir)) { + return []; + } + + const bundledRegistry = loadBundledRegistryAgents(); + const installedAgents: InstalledAgentInfo[] = []; + + try { + const entries = await fs.readdir(globalAgentsDir, { withFileTypes: true }); + + for (const entry of entries) { + // Skip registry.json and temp files + if (entry.name === 'registry.json' || entry.name.includes('.tmp.')) { + continue; + } + + if (entry.isDirectory()) { + const agentName = entry.name; + const agentPath = path.join(globalAgentsDir, entry.name); + + try { + // For directory agents, try to find main config + // Check bundled registry for main field, default to agent.yml + const bundledEntry = bundledRegistry[agentName]; + const mainFile = bundledEntry?.main || 'agent.yml'; + const mainConfigPath = path.join(agentPath, mainFile); + + // If main config doesn't exist, skip + if (!existsSync(mainConfigPath)) { + console.warn( + `Warning: Could not find main config for agent '${agentName}' at ${mainConfigPath}` + ); + continue; + } + + // Get install date from directory stats + const stats = await fs.stat(agentPath); + + // Try to extract LLM info from config + let llmProvider: string | undefined; + let llmModel: string | undefined; + + try { + const configContent = await fs.readFile(mainConfigPath, 'utf-8'); + const configMatch = configContent.match(/provider:\s*([^\n\r]+)/); + const modelMatch = configContent.match(/model:\s*([^\n\r]+)/); + + llmProvider = configMatch?.[1]?.trim(); + llmModel = modelMatch?.[1]?.trim(); + } catch (_error) { + // Ignore config parsing errors + } + + // Get description from bundled registry if available + const description = bundledEntry?.description || 'Custom agent'; + + const agentInfo: InstalledAgentInfo = { + name: agentName, + description, + path: mainConfigPath, + installedAt: stats.birthtime || stats.mtime, + }; + + if (llmProvider) agentInfo.llmProvider = llmProvider; + if (llmModel) agentInfo.llmModel = llmModel; + + installedAgents.push(agentInfo); + } catch (error) { + // Skip agents that can't be processed + console.warn(`Warning: Could not process agent '${agentName}': ${error}`); + } + } + } + } catch (_error) { + // Return empty array if we can't read the directory + return []; + } + + return installedAgents.sort((a, b) => a.name.localeCompare(b.name)); +} + +/** + * Get information about available agents from registry + */ +function getAvailableAgents(): AvailableAgentInfo[] { + const bundledRegistry = loadBundledRegistryAgents(); + + return Object.entries(bundledRegistry) + .map(([name, data]) => ({ + name, + description: data.description || 'No description', + author: data.author || 'Unknown', + tags: data.tags || [], + type: 'builtin' as const, + })) + .sort((a, b) => a.name.localeCompare(b.name)); +} + +/** + * Handle the list-agents command + */ +export async function handleListAgentsCommand( + options: ListAgentsCommandOptionsInput +): Promise<void> { + // Validate command with Zod + const validated = ListAgentsCommandSchema.parse(options); + + console.log(chalk.cyan('\n📋 Dexto Agents\n')); + + // Get global preferences for LLM info + let globalLLM: string | undefined; + if (globalPreferencesExist()) { + try { + const preferences = await loadGlobalPreferences(); + globalLLM = `${preferences.llm.provider}/${preferences.llm.model}`; + } catch { + // Ignore preference loading errors + } + } + + // Get installed and available agents + const installedAgents = await getInstalledAgents(); + const availableAgents = getAvailableAgents(); + + // Filter based on options + const showInstalled = !validated.available || validated.installed; + const showAvailable = !validated.installed || validated.available; + + // Show installed agents + if (showInstalled && installedAgents.length > 0) { + console.log(chalk.green('✅ Installed Agents:')); + + for (const agent of installedAgents) { + const llmInfo = + agent.llmProvider && agent.llmModel + ? `${agent.llmProvider}/${agent.llmModel}` + : globalLLM || 'Unknown LLM'; + + const llmDisplay = chalk.gray(`(${llmInfo})`); + + if (validated.verbose) { + console.log(` ${chalk.bold(agent.name)} ${llmDisplay}`); + console.log(` ${chalk.gray(agent.description)}`); + console.log(` ${chalk.gray('Path:')} ${agent.path}`); + if (agent.installedAt) { + console.log( + ` ${chalk.gray('Installed:')} ${agent.installedAt.toLocaleDateString()}` + ); + } + console.log(); + } else { + console.log(` • ${chalk.bold(agent.name)} ${llmDisplay} - ${agent.description}`); + } + } + console.log(); + } else if (showInstalled) { + console.log(chalk.rgb(255, 165, 0)('📦 No agents installed yet.')); + console.log( + chalk.gray(' Use `dexto install <agent-name>` to install agents from the registry.\n') + ); + } + + // Show available agents (not installed) + if (showAvailable) { + const availableNotInstalled = availableAgents.filter( + (available) => !installedAgents.some((installed) => installed.name === available.name) + ); + + const builtinAgents = availableNotInstalled.filter((a) => a.type === 'builtin'); + const customAgents = availableNotInstalled.filter((a) => a.type === 'custom'); + + if (builtinAgents.length > 0) { + console.log(chalk.blue('📋 Builtin Agents Available to Install:')); + + for (const agent of builtinAgents) { + if (validated.verbose) { + console.log(` ${chalk.bold(agent.name)}`); + console.log(` ${chalk.gray(agent.description)}`); + console.log(` ${chalk.gray('Author:')} ${agent.author}`); + console.log(` ${chalk.gray('Tags:')} ${agent.tags.join(', ')}`); + console.log(); + } else { + console.log(` • ${chalk.bold(agent.name)} - ${agent.description}`); + } + } + console.log(); + } + + if (customAgents.length > 0) { + console.log(chalk.cyan('🔧 Custom Agents Available:')); + + for (const agent of customAgents) { + if (validated.verbose) { + console.log(` ${chalk.bold(agent.name)}`); + console.log(` ${chalk.gray(agent.description)}`); + console.log(` ${chalk.gray('Author:')} ${agent.author}`); + console.log(` ${chalk.gray('Tags:')} ${agent.tags.join(', ')}`); + console.log(); + } else { + console.log(` • ${chalk.bold(agent.name)} - ${agent.description}`); + } + } + console.log(); + } + } + + // Show summary + const totalInstalled = installedAgents.length; + const availableToInstall = availableAgents.filter( + (a) => !installedAgents.some((i) => i.name === a.name) + ).length; + + if (!validated.verbose) { + console.log( + chalk.gray( + `📊 Summary: ${totalInstalled} installed, ${availableToInstall} available to install` + ) + ); + + if (availableToInstall > 0) { + console.log( + chalk.gray(` Use \`dexto install <agent-name>\` to install more agents.`) + ); + } + + console.log(chalk.gray(` Use \`dexto list-agents --verbose\` for detailed information.`)); + console.log( + chalk.gray(` After installing an agent, use \`dexto -a <agent-name>\` to run it.`) + ); + } + + console.log(); +} diff --git a/dexto/packages/cli/src/cli/commands/plugin.ts b/dexto/packages/cli/src/cli/commands/plugin.ts new file mode 100644 index 00000000..172573d9 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/plugin.ts @@ -0,0 +1,518 @@ +/** + * Plugin CLI Command Handlers + * + * Handles CLI commands for plugin management: + * - dexto plugin list + * - dexto plugin install --path <path> + * - dexto plugin uninstall <name> + * - dexto plugin validate [path] + */ + +import { z } from 'zod'; +import chalk from 'chalk'; +import { + listInstalledPlugins, + installPluginFromPath, + uninstallPlugin, + validatePluginDirectory, + type PluginInstallScope, + // Marketplace + addMarketplace, + removeMarketplace, + updateMarketplace, + listMarketplaces, + listAllMarketplacePlugins, + installPluginFromMarketplace, +} from '@dexto/agent-management'; + +// === Schema Definitions === + +const PluginListCommandSchema = z + .object({ + verbose: z.boolean().default(false).describe('Show detailed plugin information'), + }) + .strict(); + +const PluginInstallCommandSchema = z + .object({ + path: z.string().min(1).describe('Path to the plugin directory'), + scope: z.enum(['user', 'project', 'local']).default('user').describe('Installation scope'), + force: z.boolean().default(false).describe('Force overwrite if already installed'), + }) + .strict(); + +const PluginUninstallCommandSchema = z + .object({ + name: z.string().min(1).describe('Name of the plugin to uninstall'), + }) + .strict(); + +const PluginValidateCommandSchema = z + .object({ + path: z.string().default('.').describe('Path to the plugin directory to validate'), + }) + .strict(); + +// === Marketplace Command Schemas === + +const MarketplaceAddCommandSchema = z + .object({ + source: z + .string() + .min(1) + .describe('Marketplace source (owner/repo, git URL, or local path)'), + name: z.string().optional().describe('Custom name for the marketplace'), + }) + .strict(); + +const MarketplaceRemoveCommandSchema = z + .object({ + name: z.string().min(1).describe('Name of the marketplace to remove'), + }) + .strict(); + +const MarketplaceUpdateCommandSchema = z + .object({ + name: z.string().optional().describe('Name of the marketplace to update (all if omitted)'), + }) + .strict(); + +const MarketplaceListCommandSchema = z + .object({ + verbose: z.boolean().default(false).describe('Show detailed marketplace information'), + }) + .strict(); + +const MarketplaceInstallCommandSchema = z + .object({ + plugin: z.string().min(1).describe('Plugin spec: name or name@marketplace'), + scope: z.enum(['user', 'project', 'local']).default('user').describe('Installation scope'), + force: z.boolean().default(false).describe('Force reinstall if already exists'), + }) + .strict(); + +// === Type Exports === + +export type PluginListCommandOptions = z.output<typeof PluginListCommandSchema>; +export type PluginListCommandOptionsInput = z.input<typeof PluginListCommandSchema>; + +export type PluginInstallCommandOptions = z.output<typeof PluginInstallCommandSchema>; +export type PluginInstallCommandOptionsInput = z.input<typeof PluginInstallCommandSchema>; + +export type PluginUninstallCommandOptions = z.output<typeof PluginUninstallCommandSchema>; +export type PluginUninstallCommandOptionsInput = z.input<typeof PluginUninstallCommandSchema>; + +export type PluginValidateCommandOptions = z.output<typeof PluginValidateCommandSchema>; +export type PluginValidateCommandOptionsInput = z.input<typeof PluginValidateCommandSchema>; + +// Marketplace command types +export type MarketplaceAddCommandOptions = z.output<typeof MarketplaceAddCommandSchema>; +export type MarketplaceAddCommandOptionsInput = z.input<typeof MarketplaceAddCommandSchema>; + +export type MarketplaceRemoveCommandOptions = z.output<typeof MarketplaceRemoveCommandSchema>; +export type MarketplaceRemoveCommandOptionsInput = z.input<typeof MarketplaceRemoveCommandSchema>; + +export type MarketplaceUpdateCommandOptions = z.output<typeof MarketplaceUpdateCommandSchema>; +export type MarketplaceUpdateCommandOptionsInput = z.input<typeof MarketplaceUpdateCommandSchema>; + +export type MarketplaceListCommandOptions = z.output<typeof MarketplaceListCommandSchema>; +export type MarketplaceListCommandOptionsInput = z.input<typeof MarketplaceListCommandSchema>; + +export type MarketplaceInstallCommandOptions = z.output<typeof MarketplaceInstallCommandSchema>; +export type MarketplaceInstallCommandOptionsInput = z.input<typeof MarketplaceInstallCommandSchema>; + +// === Command Handlers === + +/** + * Handles the `dexto plugin list` command. + * Lists all installed plugins managed by Dexto. + */ +export async function handlePluginListCommand( + options: PluginListCommandOptionsInput +): Promise<void> { + const validated = PluginListCommandSchema.parse(options); + const plugins = listInstalledPlugins(); + + if (plugins.length === 0) { + console.log(chalk.yellow('No plugins installed.')); + console.log(''); + console.log('Install a plugin with:'); + console.log(chalk.cyan(' dexto plugin install --path <path-to-plugin>')); + return; + } + + console.log(chalk.bold(`Installed Plugins (${plugins.length}):`)); + console.log(''); + + for (const plugin of plugins) { + const sourceLabel = getSourceLabel(plugin.source); + const scopeLabel = plugin.scope ? ` [${plugin.scope}]` : ''; + + console.log( + ` ${chalk.green(plugin.name)}${chalk.dim('@' + (plugin.version || 'unknown'))} ${sourceLabel}${scopeLabel}` + ); + + if (validated.verbose) { + if (plugin.description) { + console.log(chalk.dim(` ${plugin.description}`)); + } + console.log(chalk.dim(` Path: ${plugin.path}`)); + if (plugin.installedAt) { + const date = new Date(plugin.installedAt).toLocaleDateString(); + console.log(chalk.dim(` Installed: ${date}`)); + } + } + } + + console.log(''); +} + +/** + * Handles the `dexto plugin install --path <path>` command. + * Installs a plugin from a local directory. + */ +export async function handlePluginInstallCommand( + options: PluginInstallCommandOptionsInput +): Promise<void> { + const validated = PluginInstallCommandSchema.parse(options); + + console.log(chalk.cyan(`Installing plugin from ${validated.path}...`)); + console.log(''); + + const result = await installPluginFromPath(validated.path, { + scope: validated.scope as PluginInstallScope, + force: validated.force, + }); + + // Show warnings + if (result.warnings.length > 0) { + console.log(chalk.yellow('Warnings:')); + for (const warning of result.warnings) { + console.log(chalk.yellow(` - ${warning}`)); + } + console.log(''); + } + + console.log(chalk.green(`Successfully installed plugin '${result.pluginName}'`)); + console.log(chalk.dim(` Path: ${result.installPath}`)); + console.log(''); +} + +/** + * Handles the `dexto plugin uninstall <name>` command. + * Uninstalls a plugin by name. + */ +export async function handlePluginUninstallCommand( + options: PluginUninstallCommandOptionsInput +): Promise<void> { + const validated = PluginUninstallCommandSchema.parse(options); + + console.log(chalk.cyan(`Uninstalling plugin '${validated.name}'...`)); + + const result = await uninstallPlugin(validated.name); + + console.log(chalk.green(`Successfully uninstalled plugin '${validated.name}'`)); + if (result.removedPath) { + console.log(chalk.dim(` Removed: ${result.removedPath}`)); + } + console.log(''); +} + +/** + * Handles the `dexto plugin validate [path]` command. + * Validates a plugin directory structure and manifest. + */ +export async function handlePluginValidateCommand( + options: PluginValidateCommandOptionsInput +): Promise<void> { + const validated = PluginValidateCommandSchema.parse(options); + + console.log(chalk.cyan(`Validating plugin at ${validated.path}...`)); + console.log(''); + + const result = validatePluginDirectory(validated.path); + + if (result.valid) { + console.log(chalk.green('Plugin is valid!')); + console.log(''); + + if (result.manifest) { + console.log(chalk.bold('Manifest:')); + console.log(` Name: ${chalk.green(result.manifest.name)}`); + if (result.manifest.description) { + console.log(` Description: ${result.manifest.description}`); + } + if (result.manifest.version) { + console.log(` Version: ${result.manifest.version}`); + } + console.log(''); + } + } else { + console.log(chalk.red('Plugin validation failed!')); + console.log(''); + } + + // Show errors + if (result.errors.length > 0) { + console.log(chalk.red('Errors:')); + for (const error of result.errors) { + console.log(chalk.red(` - ${error}`)); + } + console.log(''); + } + + // Show warnings + if (result.warnings.length > 0) { + console.log(chalk.yellow('Warnings:')); + for (const warning of result.warnings) { + console.log(chalk.yellow(` - ${warning}`)); + } + console.log(''); + } + + // Exit with error code if invalid + if (!result.valid) { + const errorDetails = result.errors.length > 0 ? `: ${result.errors.join(', ')}` : ''; + throw new Error(`Plugin validation failed${errorDetails}`); + } +} + +// === Marketplace Command Handlers === + +/** + * Handles the `dexto plugin marketplace add <source>` command. + * Adds a new marketplace from GitHub, git URL, or local path. + */ +export async function handleMarketplaceAddCommand( + options: MarketplaceAddCommandOptionsInput +): Promise<void> { + const validated = MarketplaceAddCommandSchema.parse(options); + + console.log(chalk.cyan(`Adding marketplace from ${validated.source}...`)); + console.log(''); + + const result = await addMarketplace(validated.source, { + name: validated.name, + }); + + // Show warnings + if (result.warnings.length > 0) { + console.log(chalk.yellow('Warnings:')); + for (const warning of result.warnings) { + console.log(chalk.yellow(` - ${warning}`)); + } + console.log(''); + } + + console.log(chalk.green(`Successfully added marketplace '${result.name}'`)); + console.log(chalk.dim(` Plugins found: ${result.pluginCount}`)); + console.log(''); +} + +/** + * Handles the `dexto plugin marketplace remove <name>` command. + * Removes a registered marketplace. + */ +export async function handleMarketplaceRemoveCommand( + options: MarketplaceRemoveCommandOptionsInput +): Promise<void> { + const validated = MarketplaceRemoveCommandSchema.parse(options); + + console.log(chalk.cyan(`Removing marketplace '${validated.name}'...`)); + + await removeMarketplace(validated.name); + + console.log(chalk.green(`Successfully removed marketplace '${validated.name}'`)); + console.log(''); +} + +/** + * Handles the `dexto plugin marketplace update [name]` command. + * Updates marketplace(s) by pulling latest from git. + */ +export async function handleMarketplaceUpdateCommand( + options: MarketplaceUpdateCommandOptionsInput +): Promise<void> { + const validated = MarketplaceUpdateCommandSchema.parse(options); + + if (validated.name) { + console.log(chalk.cyan(`Updating marketplace '${validated.name}'...`)); + } else { + console.log(chalk.cyan('Updating all marketplaces...')); + } + console.log(''); + + const results = await updateMarketplace(validated.name); + + for (const result of results) { + if (result.hasChanges) { + console.log(chalk.green(`✓ ${result.name}: Updated`)); + if (result.previousSha && result.newSha) { + console.log( + chalk.dim( + ` ${result.previousSha.substring(0, 8)} → ${result.newSha.substring(0, 8)}` + ) + ); + } + } else { + console.log(chalk.dim(`○ ${result.name}: Already up to date`)); + } + + // Show warnings + if (result.warnings.length > 0) { + for (const warning of result.warnings) { + console.log(chalk.yellow(` - ${warning}`)); + } + } + } + + console.log(''); +} + +/** + * Handles the `dexto plugin marketplace list` command. + * Lists all registered marketplaces. + */ +export async function handleMarketplaceListCommand( + options: MarketplaceListCommandOptionsInput +): Promise<void> { + const validated = MarketplaceListCommandSchema.parse(options); + const marketplaces = listMarketplaces(); + + if (marketplaces.length === 0) { + console.log(chalk.yellow('No marketplaces registered.')); + console.log(''); + console.log('Add a marketplace with:'); + console.log(chalk.cyan(' dexto plugin marketplace add <owner/repo>')); + console.log(''); + console.log('Examples:'); + console.log(chalk.dim(' dexto plugin marketplace add anthropics/claude-plugins-official')); + console.log( + chalk.dim(' dexto plugin marketplace add https://github.com/user/plugins.git') + ); + console.log(chalk.dim(' dexto plugin marketplace add ~/my-local-plugins')); + return; + } + + console.log(chalk.bold(`Registered Marketplaces (${marketplaces.length}):`)); + console.log(''); + + for (const marketplace of marketplaces) { + const sourceType = chalk.dim(`[${marketplace.source.type}]`); + console.log(` ${chalk.green(marketplace.name)} ${sourceType}`); + + if (validated.verbose) { + console.log(chalk.dim(` Source: ${marketplace.source.value}`)); + console.log(chalk.dim(` Path: ${marketplace.installLocation}`)); + if (marketplace.lastUpdated) { + const date = new Date(marketplace.lastUpdated).toLocaleDateString(); + console.log(chalk.dim(` Updated: ${date}`)); + } + } + } + + console.log(''); +} + +/** + * Handles the `dexto plugin marketplace plugins [name]` command. + * Lists plugins available in marketplaces. + */ +export async function handleMarketplacePluginsCommand(options: { + marketplace?: string | undefined; + verbose?: boolean | undefined; +}): Promise<void> { + const plugins = listAllMarketplacePlugins(); + + // Filter by marketplace if specified + const filtered = options.marketplace + ? plugins.filter( + (plugin) => plugin.marketplace.toLowerCase() === options.marketplace?.toLowerCase() + ) + : plugins; + + if (filtered.length === 0) { + if (options.marketplace) { + console.log(chalk.yellow(`No plugins found in marketplace '${options.marketplace}'.`)); + } else { + console.log(chalk.yellow('No plugins found in any marketplace.')); + console.log(''); + console.log('Make sure you have marketplaces registered:'); + console.log(chalk.cyan(' dexto plugin marketplace list')); + } + return; + } + + console.log(chalk.bold(`Available Plugins (${filtered.length}):`)); + console.log(''); + + // Group by marketplace + const byMarketplace = new Map<string, typeof filtered>(); + for (const plugin of filtered) { + const list = byMarketplace.get(plugin.marketplace) || []; + list.push(plugin); + byMarketplace.set(plugin.marketplace, list); + } + + for (const [marketplace, marketplacePlugins] of byMarketplace) { + console.log(chalk.cyan(` ${marketplace}:`)); + for (const plugin of marketplacePlugins) { + const version = plugin.version ? chalk.dim(`@${plugin.version}`) : ''; + const category = plugin.category ? chalk.dim(` [${plugin.category}]`) : ''; + console.log(` ${chalk.green(plugin.name)}${version}${category}`); + + if (options.verbose && plugin.description) { + console.log(chalk.dim(` ${plugin.description}`)); + } + } + console.log(''); + } + + console.log('Install a plugin with:'); + console.log(chalk.cyan(' dexto plugin marketplace install <name>@<marketplace>')); + console.log(''); +} + +/** + * Handles the `dexto plugin marketplace install <plugin>` command. + * Installs a plugin from a registered marketplace. + */ +export async function handleMarketplaceInstallCommand( + options: MarketplaceInstallCommandOptionsInput +): Promise<void> { + const validated = MarketplaceInstallCommandSchema.parse(options); + + console.log(chalk.cyan(`Installing plugin '${validated.plugin}' from marketplace...`)); + console.log(''); + + const result = await installPluginFromMarketplace(validated.plugin, { + scope: validated.scope as PluginInstallScope, + force: validated.force, + }); + + // Show warnings + if (result.warnings.length > 0) { + console.log(chalk.yellow('Warnings:')); + for (const warning of result.warnings) { + console.log(chalk.yellow(` - ${warning}`)); + } + console.log(''); + } + + console.log(chalk.green(`Successfully installed plugin '${result.pluginName}'`)); + console.log(chalk.dim(` Marketplace: ${result.marketplace}`)); + console.log(chalk.dim(` Path: ${result.installPath}`)); + if (result.gitCommitSha) { + console.log(chalk.dim(` Version: ${result.gitCommitSha.substring(0, 8)}`)); + } + console.log(''); +} + +// === Helper Functions === + +/** + * Gets a display label for the plugin source. + */ +function getSourceLabel(_source: 'dexto'): string { + return chalk.blue('(dexto)'); +} diff --git a/dexto/packages/cli/src/cli/commands/session-commands.ts b/dexto/packages/cli/src/cli/commands/session-commands.ts new file mode 100644 index 00000000..c1379c55 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/session-commands.ts @@ -0,0 +1,285 @@ +/** + * Non-Interactive Session Management Commands + * + * This module provides CLI commands for managing sessions outside of interactive mode. + * These commands allow users to manage sessions via direct CLI invocation. + */ + +import chalk from 'chalk'; +import { logger, DextoAgent, type SessionMetadata } from '@dexto/core'; +import { formatSessionInfo, formatHistoryMessage } from './helpers/formatters.js'; + +/** + * Escape special regex characters in a string to prevent ReDoS attacks + */ +function escapeRegExp(string: string): string { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Helper to get most recent session info (for non-interactive CLI) + * Non-interactive commands don't have a "current session" concept, + * so we use the most recently active session as a sensible default. + */ +async function getMostRecentSessionInfo( + agent: DextoAgent +): Promise<{ id: string; metadata: SessionMetadata | undefined } | null> { + const sessionIds = await agent.listSessions(); + if (sessionIds.length === 0) { + return null; + } + + // Find most recently active session + let mostRecentId: string | null = null; + let mostRecentActivity = 0; + + for (const sessionId of sessionIds) { + const metadata = await agent.getSessionMetadata(sessionId); + if (metadata && metadata.lastActivity > mostRecentActivity) { + mostRecentActivity = metadata.lastActivity; + mostRecentId = sessionId; + } + } + + if (!mostRecentId) { + return null; + } + + const metadata = await agent.getSessionMetadata(mostRecentId); + return { id: mostRecentId, metadata }; +} + +/** + * Helper to display session history with consistent formatting + */ +async function displaySessionHistory(sessionId: string, agent: DextoAgent): Promise<void> { + console.log(chalk.blue(`\n💬 Session History for: ${chalk.bold(sessionId)}\n`)); + + const history = await agent.getSessionHistory(sessionId); + + if (history.length === 0) { + console.log(chalk.gray(' No messages in this session yet.\n')); + return; + } + + // Display each message with formatting + history.forEach((message, index) => { + console.log(formatHistoryMessage(message, index)); + }); + + console.log(chalk.gray(`\n Total: ${history.length} messages`)); +} + +/** + * List all available sessions + */ +export async function handleSessionListCommand(agent: DextoAgent): Promise<void> { + try { + console.log(chalk.bold.blue('\n📋 Sessions:\n')); + + const sessionIds = await agent.listSessions(); + const mostRecent = await getMostRecentSessionInfo(agent); + + if (sessionIds.length === 0) { + console.log( + chalk.gray( + ' No sessions found. Run `dexto` to start a new session, or use `dexto -c`/`dexto -r <id>`.\n' + ) + ); + return; + } + + // Fetch metadata concurrently; errors do not abort listing + const entries = await Promise.all( + sessionIds.map(async (id) => { + try { + const metadata = await agent.getSessionMetadata(id); + return { id, metadata }; + } catch (e) { + logger.error( + `Failed to fetch metadata for session ${id}: ${e instanceof Error ? e.message : String(e)}`, + null, + 'red' + ); + return { id, metadata: undefined as SessionMetadata | undefined }; + } + }) + ); + + let displayed = 0; + for (const { id, metadata } of entries) { + if (!metadata) continue; + // Mark most recent session with indicator (instead of "current") + const isMostRecent = mostRecent ? id === mostRecent.id : false; + console.log(` ${formatSessionInfo(id, metadata, isMostRecent)}`); + displayed++; + } + + console.log(chalk.gray(`\n Total: ${displayed} of ${sessionIds.length} sessions`)); + console.log(chalk.gray(' 💡 Use `dexto -r <id>` to resume a session\n')); + } catch (error) { + logger.error( + `Failed to list sessions: ${error instanceof Error ? error.message : String(error)}`, + null, + 'red' + ); + throw error; + } +} + +/** + * Show session history + */ +export async function handleSessionHistoryCommand( + agent: DextoAgent, + sessionId?: string +): Promise<void> { + try { + // Use provided session ID or most recent session + let targetSessionId = sessionId; + if (!targetSessionId) { + const recentSession = await getMostRecentSessionInfo(agent); + if (!recentSession) { + console.log(chalk.red('❌ No sessions found')); + console.log(chalk.gray(' Create a session first by running: dexto')); + throw new Error('No sessions found'); + } + targetSessionId = recentSession.id; + console.log(chalk.gray(`Using most recent session: ${targetSessionId}\n`)); + } + + await displaySessionHistory(targetSessionId, agent); + } catch (error) { + if (error instanceof Error && error.message.includes('not found')) { + console.log(chalk.red(`❌ Session not found: ${sessionId || 'current'}`)); + console.log(chalk.gray(' Use `dexto session list` to see available sessions')); + } else if (error instanceof Error && error.message !== 'No sessions found') { + logger.error(`Failed to get session history: ${error.message}`, null, 'red'); + } + throw error; + } +} + +/** + * Delete a session + */ +export async function handleSessionDeleteCommand( + agent: DextoAgent, + sessionId: string +): Promise<void> { + try { + // Note: For non-interactive CLI, there's no concept of "current session" + // We just delete the requested session without restrictions + await agent.deleteSession(sessionId); + console.log(chalk.green(`✅ Deleted session: ${chalk.bold(sessionId)}`)); + } catch (error) { + logger.error( + `Failed to delete session: ${error instanceof Error ? error.message : String(error)}`, + null, + 'red' + ); + throw error; + } +} + +/** + * Search session history + */ +export async function handleSessionSearchCommand( + agent: DextoAgent, + query: string, + options: { + sessionId?: string; + role?: 'user' | 'assistant' | 'system' | 'tool'; + limit?: number; + } = {} +): Promise<void> { + try { + const searchOptions: { + limit: number; + sessionId?: string; + role?: 'user' | 'assistant' | 'system' | 'tool'; + } = { + limit: options.limit || 10, + }; + + if (options.sessionId) { + searchOptions.sessionId = options.sessionId; + } + if (options.role) { + const allowed = new Set(['user', 'assistant', 'system', 'tool']); + if (!allowed.has(options.role)) { + console.log( + chalk.red( + `❌ Invalid role: ${options.role}. Use one of: user, assistant, system, tool` + ) + ); + return; + } + searchOptions.role = options.role; + } + + console.log(chalk.blue(`🔍 Searching for: "${query}"`)); + if (searchOptions.sessionId) { + console.log(chalk.gray(` Session: ${searchOptions.sessionId}`)); + } + if (searchOptions.role) { + console.log(chalk.gray(` Role: ${searchOptions.role}`)); + } + console.log(chalk.gray(` Limit: ${searchOptions.limit}`)); + console.log(); + + const results = await agent.searchMessages(query, searchOptions); + + if (results.results.length === 0) { + console.log(chalk.rgb(255, 165, 0)('📭 No messages found matching your search')); + return; + } + + console.log( + chalk.green(`✅ Found ${results.total} result${results.total === 1 ? '' : 's'}`) + ); + if (results.hasMore) { + console.log(chalk.gray(` Showing first ${results.results.length} results`)); + } + console.log(); + + // Precompile safe regex for highlighting (only if query is not empty) + const highlightRegex = query.trim() + ? new RegExp(`(${escapeRegExp(query.trim().slice(0, 256))})`, 'gi') + : null; + + // Display results + results.results.forEach((result, index) => { + const roleColor = + result.message.role === 'user' + ? chalk.blue + : result.message.role === 'assistant' + ? chalk.green + : chalk.rgb(255, 165, 0); + + console.log( + `${chalk.gray(`${index + 1}.`)} ${chalk.cyan(result.sessionId)} ${roleColor(`[${result.message.role}]`)}` + ); + + // Safe highlighting - only if we have a valid regex + const highlightedContext = highlightRegex + ? result.context.replace(highlightRegex, chalk.inverse('$1')) + : result.context; + + console.log(` ${highlightedContext}`); + console.log(); + }); + + if (results.hasMore) { + console.log(chalk.gray('💡 Use --limit to see more results')); + } + } catch (error) { + logger.error( + `Search failed: ${error instanceof Error ? error.message : String(error)}`, + null, + 'red' + ); + throw error; + } +} diff --git a/dexto/packages/cli/src/cli/commands/setup.test.ts b/dexto/packages/cli/src/cli/commands/setup.test.ts new file mode 100644 index 00000000..a645af08 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/setup.test.ts @@ -0,0 +1,754 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import { tmpdir } from 'os'; +import { handleSetupCommand, type CLISetupOptionsInput } from './setup.js'; + +// Mock only external dependencies that can't be tested directly +vi.mock('@dexto/core', async () => { + const actual = await vi.importActual<typeof import('@dexto/core')>('@dexto/core'); + return { + ...actual, + resolveApiKeyForProvider: vi.fn(), + requiresApiKey: vi.fn(() => true), // Most providers need API keys + }; +}); + +vi.mock('@dexto/agent-management', async () => { + const actual = + await vi.importActual<typeof import('@dexto/agent-management')>('@dexto/agent-management'); + return { + ...actual, + createInitialPreferences: vi.fn((...args: any[]) => { + const options = args[0]; + // Handle new options object signature + if (typeof options === 'object' && 'provider' in options) { + const llmConfig: any = { provider: options.provider, model: options.model }; + if (options.apiKeyVar) { + llmConfig.apiKey = `$${options.apiKeyVar}`; + } + if (options.baseURL) { + llmConfig.baseURL = options.baseURL; + } + return { + llm: llmConfig, + defaults: { + defaultAgent: options.defaultAgent || 'coding-agent', + defaultMode: options.defaultMode || 'web', + }, + setup: { completed: options.setupCompleted ?? true }, + }; + } + // Legacy signature (provider, model, apiKeyVar, defaultAgent) + return { + llm: { provider: options, model: args[1], apiKey: `$${args[2]}` }, + defaults: { defaultAgent: args[3] || 'coding-agent' }, + setup: { completed: true }, + }; + }), + saveGlobalPreferences: vi.fn().mockResolvedValue(undefined), + loadGlobalPreferences: vi.fn().mockResolvedValue(null), + getGlobalPreferencesPath: vi.fn(() => '/tmp/preferences.yml'), + updateGlobalPreferences: vi.fn().mockResolvedValue(undefined), + }; +}); + +vi.mock('../utils/api-key-setup.js', () => ({ + interactiveApiKeySetup: vi.fn().mockResolvedValue({ success: true }), + hasApiKeyConfigured: vi.fn(() => true), +})); + +vi.mock('../utils/provider-setup.js', () => ({ + selectProvider: vi.fn(), + getProviderDisplayName: vi.fn((provider: string) => provider), + getProviderEnvVar: vi.fn((provider: string) => `${provider.toUpperCase()}_API_KEY`), + getProviderInfo: vi.fn(() => ({ apiKeyUrl: 'https://example.com' })), + providerRequiresBaseURL: vi.fn(() => false), + getDefaultModel: vi.fn(() => 'test-model'), +})); + +vi.mock('../utils/setup-utils.js', () => ({ + requiresSetup: vi.fn(), +})); + +vi.mock('@clack/prompts', () => ({ + intro: vi.fn(), + note: vi.fn(), + outro: vi.fn(), + confirm: vi.fn(), + cancel: vi.fn(), + isCancel: vi.fn(), + select: vi.fn().mockResolvedValue('exit'), + text: vi.fn().mockResolvedValue('test'), + password: vi.fn().mockResolvedValue('test-key'), + spinner: vi.fn(() => ({ start: vi.fn(), stop: vi.fn() })), + log: { warn: vi.fn(), success: vi.fn(), error: vi.fn(), info: vi.fn() }, +})); + +describe('Setup Command', () => { + let tempDir: string; + let mockCreateInitialPreferences: any; + let mockSaveGlobalPreferences: any; + let mockInteractiveApiKeySetup: any; + let mockHasApiKeyConfigured: any; + let mockSelectProvider: any; + let mockRequiresSetup: any; + let mockResolveApiKeyForProvider: any; + let mockPrompts: any; + let consoleSpy: any; + let consoleErrorSpy: any; + let processExitSpy: any; + + function createTempDir() { + return fs.mkdtempSync(path.join(tmpdir(), 'setup-test-')); + } + + beforeEach(async () => { + vi.clearAllMocks(); + tempDir = createTempDir(); + + // Get mock functions + const prefLoader = await import('@dexto/agent-management'); + const apiKeySetup = await import('../utils/api-key-setup.js'); + const apiKeyResolver = await import('@dexto/core'); + const providerSetup = await import('../utils/provider-setup.js'); + const setupUtils = await import('../utils/setup-utils.js'); + const prompts = await import('@clack/prompts'); + + mockCreateInitialPreferences = vi.mocked(prefLoader.createInitialPreferences); + mockSaveGlobalPreferences = vi.mocked(prefLoader.saveGlobalPreferences); + mockInteractiveApiKeySetup = vi.mocked(apiKeySetup.interactiveApiKeySetup); + mockHasApiKeyConfigured = vi.mocked(apiKeySetup.hasApiKeyConfigured); + mockResolveApiKeyForProvider = vi.mocked(apiKeyResolver.resolveApiKeyForProvider); + mockSelectProvider = vi.mocked(providerSetup.selectProvider); + mockRequiresSetup = vi.mocked(setupUtils.requiresSetup); + mockPrompts = { + intro: vi.mocked(prompts.intro), + note: vi.mocked(prompts.note), + confirm: vi.mocked(prompts.confirm), + cancel: vi.mocked(prompts.cancel), + isCancel: vi.mocked(prompts.isCancel), + select: vi.mocked(prompts.select), + log: { + warn: vi.mocked(prompts.log.warn), + success: vi.mocked(prompts.log.success), + info: vi.mocked(prompts.log.info), + }, + }; + + // Reset mocks to default behavior - use new options object signature + mockCreateInitialPreferences.mockImplementation((...args: any[]) => { + const options = args[0]; + if (typeof options === 'object' && 'provider' in options) { + const llmConfig: any = { provider: options.provider, model: options.model }; + if (options.apiKeyVar) { + llmConfig.apiKey = `$${options.apiKeyVar}`; + } + return { + llm: llmConfig, + defaults: { + defaultAgent: options.defaultAgent || 'coding-agent', + defaultMode: options.defaultMode || 'web', + }, + setup: { completed: options.setupCompleted ?? true }, + }; + } + return { + llm: { provider: options, model: args[1], apiKey: `$${args[2]}` }, + defaults: { defaultAgent: args[3] || 'coding-agent' }, + setup: { completed: true }, + }; + }); + mockSaveGlobalPreferences.mockResolvedValue(undefined); + mockInteractiveApiKeySetup.mockResolvedValue({ success: true }); + mockHasApiKeyConfigured.mockReturnValue(true); // Default: API key exists + mockResolveApiKeyForProvider.mockReturnValue(undefined); // Default: no API key exists (for analytics) + mockSelectProvider.mockResolvedValue(null); + mockRequiresSetup.mockResolvedValue(true); // Default: setup is required + mockPrompts.isCancel.mockReturnValue(false); + mockPrompts.select.mockResolvedValue('exit'); // Default: exit settings menu + + // Mock console to prevent test output noise + consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + processExitSpy = vi + .spyOn(process, 'exit') + .mockImplementation((code?: string | number | null | undefined) => { + throw new Error(`Process exit called with code ${code}`); + }); + }); + + afterEach(() => { + if (fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + consoleSpy.mockRestore(); + consoleErrorSpy.mockRestore(); + processExitSpy.mockRestore(); + }); + + describe('Non-interactive setup', () => { + it('creates preferences with provided options using new object signature', async () => { + const options: CLISetupOptionsInput = { + provider: 'openai', + model: 'gpt-5', + defaultAgent: 'my-agent', + interactive: false, + }; + + await handleSetupCommand(options); + + expect(mockCreateInitialPreferences).toHaveBeenCalledWith({ + provider: 'openai', + model: 'gpt-5', + apiKeyVar: 'OPENAI_API_KEY', + defaultAgent: 'my-agent', + setupCompleted: true, + }); + expect(mockSaveGlobalPreferences).toHaveBeenCalled(); + expect(mockInteractiveApiKeySetup).not.toHaveBeenCalled(); + }); + + it('uses default model when not specified', async () => { + const options = { + provider: 'anthropic' as const, + interactive: false, + }; + + await handleSetupCommand(options); + + expect(mockCreateInitialPreferences).toHaveBeenCalledWith({ + provider: 'anthropic', + model: 'test-model', // From mocked getDefaultModel + apiKeyVar: 'ANTHROPIC_API_KEY', + defaultAgent: 'coding-agent', + setupCompleted: true, + }); + }); + + it('throws error when provider missing in non-interactive mode', async () => { + const options = { + interactive: false, + }; + + await expect(handleSetupCommand(options)).rejects.toThrow( + 'Provider required in non-interactive mode. Use --provider or --quick-start option.' + ); + }); + + it('exits with error when model required but not provided', async () => { + // Mock getDefaultModel to return empty string for this provider (simulating no default) + const providerSetup = await import('../utils/provider-setup.js'); + vi.mocked(providerSetup.getDefaultModel).mockReturnValueOnce(''); + + const options = { + provider: 'openai-compatible' as const, // Provider with no default model + interactive: false, + }; + + await expect(handleSetupCommand(options)).rejects.toThrow( + 'Process exit called with code 1' + ); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining("Model is required for provider 'openai-compatible'") + ); + }); + }); + + describe('Interactive setup', () => { + it('shows setup type selection when interactive without provider', async () => { + // User selects 'custom' setup, then provider selection happens + // Wizard uses selectProvider for provider selection + mockPrompts.select.mockResolvedValueOnce('custom'); // Setup type + mockSelectProvider.mockResolvedValueOnce('anthropic'); // Provider (via selectProvider) + mockPrompts.select.mockResolvedValueOnce('claude-haiku-4-5-20251001'); // Model + mockPrompts.select.mockResolvedValueOnce('web'); // Default mode + + const options = { + interactive: true, + }; + + await handleSetupCommand(options); + + expect(mockCreateInitialPreferences).toHaveBeenCalledWith( + expect.objectContaining({ + provider: 'anthropic', + defaultMode: 'web', + setupCompleted: true, + }) + ); + }); + + it('handles quick start selection in interactive mode', async () => { + // User selects 'quick' setup + mockPrompts.select.mockResolvedValueOnce('quick'); // Setup type -> quick start + mockPrompts.select.mockResolvedValueOnce('google'); // Provider picker -> google + mockPrompts.confirm.mockResolvedValueOnce(true); // CLI mode confirmation -> yes + mockHasApiKeyConfigured.mockReturnValue(true); // API key already configured + + const options = { + interactive: true, + }; + + await handleSetupCommand(options); + + // Quick start uses Google provider with CLI mode + expect(mockCreateInitialPreferences).toHaveBeenCalledWith( + expect.objectContaining({ + provider: 'google', + defaultMode: 'cli', + setupCompleted: true, + }) + ); + }); + + it('runs interactive API key setup when no API key exists', async () => { + // New wizard flow uses p.select for setup type, selectProvider for provider + mockPrompts.select.mockResolvedValueOnce('custom'); // Setup type + mockSelectProvider.mockResolvedValueOnce('openai'); // Provider (via selectProvider) + mockPrompts.select.mockResolvedValueOnce('gpt-4o'); // Model (must be valid OpenAI model from registry) + mockPrompts.select.mockResolvedValueOnce('web'); // Default mode + mockHasApiKeyConfigured.mockReturnValue(false); // No API key exists + + const options = { + interactive: true, + }; + + await handleSetupCommand(options); + + // API key setup is called with provider and model + expect(mockInteractiveApiKeySetup).toHaveBeenCalledWith( + 'openai', + expect.objectContaining({ + exitOnCancel: false, + }) + ); + }); + + it('skips interactive API key setup when API key already exists', async () => { + mockPrompts.select.mockResolvedValueOnce('custom'); // Setup type + mockSelectProvider.mockResolvedValueOnce('openai'); // Provider (via selectProvider) + mockPrompts.select.mockResolvedValueOnce('gpt-4'); // Model + mockPrompts.select.mockResolvedValueOnce('web'); // Default mode + mockHasApiKeyConfigured.mockReturnValue(true); // API key exists + + const options = { + interactive: true, + }; + + await handleSetupCommand(options); + + expect(mockInteractiveApiKeySetup).not.toHaveBeenCalled(); + expect(mockPrompts.log.success).toHaveBeenCalled(); + }); + + it('cancels setup when user cancels setup type selection', async () => { + mockPrompts.select.mockResolvedValueOnce(Symbol.for('cancel')); // Cancel + mockPrompts.isCancel.mockReturnValue(true); + + const options = { + interactive: true, + }; + + await expect(handleSetupCommand(options)).rejects.toThrow( + 'Process exit called with code 0' + ); + }); + }); + + describe('Validation', () => { + it('validates schema correctly with defaults and uses new options signature', async () => { + // Interactive mode with provider - goes through full setup flow + mockPrompts.select.mockResolvedValueOnce('custom'); // Setup type + mockSelectProvider.mockResolvedValueOnce('google'); // Provider (via selectProvider) + mockPrompts.select.mockResolvedValueOnce('gemini-2.5-pro'); // Model + mockPrompts.select.mockResolvedValueOnce('web'); // Default mode + + const options = { + provider: 'google' as const, + }; + + await handleSetupCommand(options); + + // Should apply defaults: interactive=true, defaultAgent='coding-agent' + expect(mockCreateInitialPreferences).toHaveBeenCalledWith( + expect.objectContaining({ + provider: 'google', + setupCompleted: true, + }) + ); + }); + + it('throws ZodError for invalid provider', async () => { + const options = { + provider: 'invalid-provider', + interactive: false, + } as any; + + await expect(handleSetupCommand(options)).rejects.toThrow(); + }); + + it('throws validation error for empty model name', async () => { + const options = { + provider: 'openai', + model: '', + interactive: false, + } as any; + + await expect(handleSetupCommand(options)).rejects.toThrow(); + }); + + it('throws validation error for empty default agent', async () => { + const options = { + provider: 'openai', + defaultAgent: '', + interactive: false, + } as any; + + await expect(handleSetupCommand(options)).rejects.toThrow(); + }); + + it('handles strict mode validation correctly', async () => { + const options = { + provider: 'openai', + unknownField: 'should-cause-error', + interactive: false, + } as any; + + await expect(handleSetupCommand(options)).rejects.toThrow(); + }); + }); + + describe('Error handling', () => { + it('propagates errors from createInitialPreferences', async () => { + mockCreateInitialPreferences.mockImplementation(() => { + throw new Error('Failed to create preferences'); + }); + + const options = { + provider: 'openai' as const, + interactive: false, + }; + + await expect(handleSetupCommand(options)).rejects.toThrow( + 'Failed to create preferences' + ); + }); + + it('propagates errors from saveGlobalPreferences', async () => { + // Reset createInitialPreferences to new options signature + mockCreateInitialPreferences.mockImplementation((options: any) => ({ + llm: { + provider: options.provider, + model: options.model, + apiKey: `$${options.apiKeyVar}`, + }, + defaults: { + defaultAgent: options.defaultAgent || 'coding-agent', + defaultMode: 'web', + }, + setup: { completed: true }, + })); + mockSaveGlobalPreferences.mockRejectedValue(new Error('Failed to save preferences')); + + const options = { + provider: 'openai' as const, + interactive: false, + }; + + await expect(handleSetupCommand(options)).rejects.toThrow('Failed to save preferences'); + }); + + it('propagates errors from interactiveApiKeySetup', async () => { + // Reset to new options signature + mockCreateInitialPreferences.mockImplementation((options: any) => ({ + llm: { + provider: options.provider, + model: options.model, + apiKey: `$${options.apiKeyVar}`, + }, + defaults: { + defaultAgent: options.defaultAgent || 'coding-agent', + defaultMode: 'web', + }, + setup: { completed: true }, + })); + mockSaveGlobalPreferences.mockResolvedValue(undefined); + mockHasApiKeyConfigured.mockReturnValue(false); // No API key exists + // Simulate a thrown error (not just a failed result) + mockInteractiveApiKeySetup.mockRejectedValue(new Error('API key setup failed')); + + // Setup mocks for interactive flow + mockPrompts.select.mockResolvedValueOnce('custom'); // Setup type + mockSelectProvider.mockResolvedValueOnce('openai'); // Provider (via selectProvider) + mockPrompts.select.mockResolvedValueOnce('gpt-4'); // Model + mockPrompts.select.mockResolvedValueOnce('web'); // Mode (won't be reached due to error) + + const options = { + interactive: true, + }; + + await expect(handleSetupCommand(options)).rejects.toThrow('API key setup failed'); + }); + }); + + describe('Edge cases', () => { + it('works correctly with multiple providers in non-interactive mode', async () => { + const testCases = [ + { + provider: 'openai', + expectedKey: 'OPENAI_API_KEY', + }, + { + provider: 'anthropic', + expectedKey: 'ANTHROPIC_API_KEY', + }, + { + provider: 'google', + expectedKey: 'GOOGLE_API_KEY', + }, + ] as const; + + for (const testCase of testCases) { + // Reset mocks for each test case + mockCreateInitialPreferences.mockClear(); + mockSaveGlobalPreferences.mockClear(); + mockInteractiveApiKeySetup.mockClear(); + + const options = { + provider: testCase.provider, + interactive: false, + }; + + await handleSetupCommand(options); + + expect(mockCreateInitialPreferences).toHaveBeenCalledWith({ + provider: testCase.provider, + model: 'test-model', // From mocked getDefaultModel + apiKeyVar: testCase.expectedKey, + defaultAgent: 'coding-agent', + setupCompleted: true, + }); + } + }); + + it('preserves user-provided model over default', async () => { + // Reset to new options signature + mockCreateInitialPreferences.mockImplementation((options: any) => ({ + llm: { + provider: options.provider, + model: options.model, + apiKey: `$${options.apiKeyVar}`, + }, + defaults: { + defaultAgent: options.defaultAgent || 'coding-agent', + defaultMode: 'web', + }, + setup: { completed: true }, + })); + + const options = { + provider: 'openai' as const, + model: 'gpt-5-mini', + interactive: false, + }; + + await handleSetupCommand(options); + + expect(mockCreateInitialPreferences).toHaveBeenCalledWith({ + provider: 'openai', + model: 'gpt-5-mini', // User-specified model, not default + apiKeyVar: 'OPENAI_API_KEY', + defaultAgent: 'coding-agent', + setupCompleted: true, + }); + }); + }); + + describe('Re-setup scenarios', () => { + beforeEach(() => { + // Setup is already complete for these tests + mockRequiresSetup.mockResolvedValue(false); + }); + + describe('Non-interactive re-setup', () => { + it('errors without --force flag when setup is already complete', async () => { + const options = { + provider: 'openai' as const, + interactive: false, + force: false, + }; + + await expect(handleSetupCommand(options)).rejects.toThrow( + 'Process exit called with code 1' + ); + + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('Setup is already complete') + ); + expect(mockCreateInitialPreferences).not.toHaveBeenCalled(); + }); + + it('proceeds with --force flag when setup is already complete', async () => { + const options = { + provider: 'openai' as const, + interactive: false, + force: true, + }; + + await handleSetupCommand(options); + + expect(mockCreateInitialPreferences).toHaveBeenCalledWith({ + provider: 'openai', + model: 'test-model', // From mocked getDefaultModel + apiKeyVar: 'OPENAI_API_KEY', + defaultAgent: 'coding-agent', + setupCompleted: true, + }); + expect(mockSaveGlobalPreferences).toHaveBeenCalled(); + expect(processExitSpy).not.toHaveBeenCalled(); + }); + }); + + describe('Interactive re-setup (Settings Menu)', () => { + it('shows settings menu when setup is already complete', async () => { + // User selects 'exit' from settings menu + mockPrompts.select.mockResolvedValueOnce('exit'); + + const options = { + interactive: true, + }; + + await handleSetupCommand(options); + + // Should show settings menu intro + expect(mockPrompts.intro).toHaveBeenCalledWith(expect.stringContaining('Settings')); + // Should not try to create new preferences when exiting + expect(mockCreateInitialPreferences).not.toHaveBeenCalled(); + }); + + it('exits gracefully when user cancels from settings menu', async () => { + mockPrompts.select.mockResolvedValueOnce(Symbol.for('cancel')); + mockPrompts.isCancel.mockReturnValue(true); + + const options = { + interactive: true, + }; + + await handleSetupCommand(options); + + // Should not throw, just exit gracefully + expect(mockCreateInitialPreferences).not.toHaveBeenCalled(); + }); + }); + + it('proceeds normally when setup is required despite preferences existing', async () => { + // Edge case: preferences exist but are incomplete/corrupted + mockRequiresSetup.mockResolvedValue(true); + + const options = { + provider: 'openai' as const, + interactive: false, + }; + + await handleSetupCommand(options); + + expect(mockCreateInitialPreferences).toHaveBeenCalled(); + expect(mockSaveGlobalPreferences).toHaveBeenCalled(); + }); + }); + + describe('Quick start flow', () => { + it('handles --quick-start flag in non-interactive mode', async () => { + mockPrompts.select.mockResolvedValueOnce('google'); // Provider picker + mockPrompts.confirm.mockResolvedValueOnce(true); // CLI mode confirmation + mockHasApiKeyConfigured.mockReturnValue(true); + + const options = { + quickStart: true, + interactive: false, + }; + + // Note: quickStart triggers the quick start flow even in non-interactive + await handleSetupCommand(options); + + expect(mockCreateInitialPreferences).toHaveBeenCalledWith( + expect.objectContaining({ + provider: 'google', + defaultMode: 'cli', + setupCompleted: true, + }) + ); + }); + + it('prompts for API key if not configured during quick start', async () => { + mockPrompts.select.mockResolvedValueOnce('google'); // Provider picker + mockPrompts.confirm.mockResolvedValueOnce(true); // CLI mode confirmation + mockHasApiKeyConfigured.mockReturnValue(false); + mockInteractiveApiKeySetup.mockResolvedValue({ success: true }); + + const options = { + quickStart: true, + }; + + await handleSetupCommand(options); + + expect(mockInteractiveApiKeySetup).toHaveBeenCalledWith( + 'google', + expect.objectContaining({ + exitOnCancel: false, + }) + ); + }); + + it('handles API key skip during quick start', async () => { + mockPrompts.select.mockResolvedValueOnce('google'); // Provider picker + mockPrompts.confirm.mockResolvedValueOnce(true); // CLI mode confirmation + mockHasApiKeyConfigured.mockReturnValue(false); + mockInteractiveApiKeySetup.mockResolvedValue({ success: true, skipped: true }); + + const options = { + quickStart: true, + }; + + await handleSetupCommand(options); + + // Should save preferences with apiKeyPending flag set to true + expect(mockCreateInitialPreferences).toHaveBeenCalledWith( + expect.objectContaining({ + provider: 'google', + apiKeyPending: true, + setupCompleted: true, + }) + ); + expect(mockSaveGlobalPreferences).toHaveBeenCalled(); + }); + + it('sets apiKeyPending to false when API key is provided', async () => { + // Reset mocks to ensure clean state + mockPrompts.select.mockReset(); + mockPrompts.confirm.mockReset(); + mockPrompts.select.mockResolvedValueOnce('google'); // Provider picker + mockPrompts.confirm.mockResolvedValueOnce(true); // CLI mode confirmation + + mockHasApiKeyConfigured.mockReturnValue(false); + // interactiveApiKeySetup returns success without skipped flag - API key was provided + mockInteractiveApiKeySetup.mockResolvedValue({ success: true, apiKey: 'test-key' }); + + const options = { + quickStart: true, + }; + + await handleSetupCommand(options); + + // Should save preferences with apiKeyPending false + expect(mockCreateInitialPreferences).toHaveBeenCalledWith( + expect.objectContaining({ + provider: 'google', + apiKeyPending: false, + setupCompleted: true, + }) + ); + }); + }); +}); diff --git a/dexto/packages/cli/src/cli/commands/setup.ts b/dexto/packages/cli/src/cli/commands/setup.ts new file mode 100644 index 00000000..f314e099 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/setup.ts @@ -0,0 +1,1632 @@ +// packages/cli/src/cli/commands/setup.ts + +import chalk from 'chalk'; +import { z } from 'zod'; +import { + getDefaultModelForProvider, + LLM_PROVIDERS, + LLM_REGISTRY, + isValidProviderModel, + getSupportedModels, + acceptsAnyModel, + supportsCustomModels, + requiresApiKey, + isReasoningCapableModel, +} from '@dexto/core'; +import { resolveApiKeyForProvider } from '@dexto/core'; +import { + createInitialPreferences, + saveGlobalPreferences, + loadGlobalPreferences, + getGlobalPreferencesPath, + updateGlobalPreferences, + setActiveModel, + isDextoAuthEnabled, + type CreatePreferencesOptions, +} from '@dexto/agent-management'; +import { interactiveApiKeySetup, hasApiKeyConfigured } from '../utils/api-key-setup.js'; +import { + selectProvider, + getProviderDisplayName, + getProviderEnvVar, + providerRequiresBaseURL, + getDefaultModel, +} from '../utils/provider-setup.js'; +import { + setupLocalModels, + setupOllamaModels, + hasSelectedModel, + getModelFromResult, +} from '../utils/local-model-setup.js'; +import { requiresSetup } from '../utils/setup-utils.js'; +import { canUseDextoProvider } from '../utils/dexto-setup.js'; +import { handleBrowserLogin } from './auth/login.js'; +import * as p from '@clack/prompts'; +import { logger } from '@dexto/core'; +import { capture } from '../../analytics/index.js'; +import type { LLMProvider } from '@dexto/core'; + +// Zod schema for setup command validation +const SetupCommandSchema = z + .object({ + provider: z + .enum(LLM_PROVIDERS) + .optional() + .describe('AI provider identifier to use for LLM calls'), + model: z + .string() + .min(1, 'Model name cannot be empty') + .optional() + .describe('Preferred model name for the selected provider'), + defaultAgent: z + .string() + .min(1, 'Default agent name cannot be empty') + .default('coding-agent') + .describe('Registry agent id to use when none is specified'), + interactive: z.boolean().default(true).describe('Enable interactive prompts'), + force: z + .boolean() + .default(false) + .describe('Overwrite existing setup when already configured'), + quickStart: z + .boolean() + .default(false) + .describe('Use quick start with Google Gemini (recommended for new users)'), + }) + .strict() + .superRefine((data, ctx) => { + // Validate model against provider when both are provided + if (data.provider && data.model) { + // Skip validation for providers that accept any model + if (!acceptsAnyModel(data.provider) && !supportsCustomModels(data.provider)) { + if (!isValidProviderModel(data.provider, data.model)) { + const supportedModels = getSupportedModels(data.provider); + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['model'], + message: `Model '${data.model}' is not supported by provider '${data.provider}'. Supported models: ${supportedModels.join(', ')}`, + }); + } + } + } + }); + +export type CLISetupOptions = z.output<typeof SetupCommandSchema>; +export type CLISetupOptionsInput = z.input<typeof SetupCommandSchema>; + +// ============================================================================ +// Setup Wizard Types and Helpers +// ============================================================================ + +/** + * Setup wizard step identifiers + */ +type SetupStep = + | 'setupType' + | 'provider' + | 'model' + | 'reasoningEffort' + | 'apiKey' + | 'mode' + | 'complete'; + +/** + * Setup wizard state - accumulates data as user progresses through steps + * Note: Properties explicitly allow undefined for back navigation to clear values + */ +interface SetupWizardState { + step: SetupStep; + setupType?: 'quick' | 'custom' | undefined; + provider?: LLMProvider | undefined; + model?: string | undefined; + baseURL?: string | undefined; + reasoningEffort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | undefined; + apiKeySkipped?: boolean | undefined; + defaultMode?: 'cli' | 'web' | 'server' | 'discord' | 'telegram' | 'mcp' | undefined; + /** Quick start handles its own preferences saving */ + quickStartHandled?: boolean | undefined; +} + +/** + * Get the steps to display for the current provider and model. + * - Local/Ollama providers skip the API Key step. + * - Reasoning-capable models (o1, o3, codex, gpt-5.x) show the Reasoning step. + */ +function getWizardSteps( + provider?: LLMProvider, + model?: string +): Array<{ key: SetupStep; label: string }> { + const isLocalProvider = provider === 'local' || provider === 'ollama'; + const showReasoningStep = model && isReasoningCapableModel(model); + + if (isLocalProvider) { + const steps: Array<{ key: SetupStep; label: string }> = [ + { key: 'provider', label: 'Provider' }, + { key: 'model', label: 'Model' }, + ]; + if (showReasoningStep) { + steps.push({ key: 'reasoningEffort', label: 'Reasoning' }); + } + steps.push({ key: 'mode', label: 'Mode' }); + return steps; + } + + const steps: Array<{ key: SetupStep; label: string }> = [ + { key: 'provider', label: 'Provider' }, + { key: 'model', label: 'Model' }, + ]; + if (showReasoningStep) { + steps.push({ key: 'reasoningEffort', label: 'Reasoning' }); + } + steps.push({ key: 'apiKey', label: 'API Key' }); + steps.push({ key: 'mode', label: 'Mode' }); + return steps; +} + +/** + * Display step progress indicator for the setup wizard + */ +function showStepProgress(currentStep: SetupStep, provider?: LLMProvider, model?: string): void { + // Don't show progress for setupType (it's the entry point) + if (currentStep === 'setupType' || currentStep === 'complete') { + return; + } + + const steps = getWizardSteps(provider, model); + const currentIndex = steps.findIndex((s) => s.key === currentStep); + + if (currentIndex === -1) { + return; + } + + const progress = steps + .map((step, i) => { + if (i < currentIndex) return chalk.green(`✓ ${step.label}`); + if (i === currentIndex) return chalk.cyan(`● ${step.label}`); + return chalk.gray(`○ ${step.label}`); + }) + .join(' '); + + console.log(`\n ${progress}\n`); +} + +// ============================================================================ +// Setup Command Validation +// ============================================================================ + +/** + * Validate setup command options with comprehensive validation + */ +function validateSetupCommand(options: Partial<CLISetupOptionsInput>): CLISetupOptions { + const validated = SetupCommandSchema.parse(options); + + // Business logic validation + if (!validated.interactive && !validated.provider && !validated.quickStart) { + throw new Error( + 'Provider required in non-interactive mode. Use --provider or --quick-start option.' + ); + } + + return validated; +} + +/** + * Main setup command handler + */ +export async function handleSetupCommand(options: Partial<CLISetupOptionsInput>): Promise<void> { + const validated = validateSetupCommand(options); + logger.debug(`Validated setup command options: ${JSON.stringify(validated, null, 2)}`); + + // Check if setup is already complete + const needsSetup = await requiresSetup(); + + if (!needsSetup && !validated.force) { + if (!validated.interactive) { + console.error(chalk.red('❌ Setup is already complete.')); + console.error( + chalk.gray(' Use --force to overwrite, or run interactively for options.') + ); + process.exit(1); + } + + // Interactive mode: show settings menu + await showSettingsMenu(); + return; + } + + // Handle quick start + if (validated.quickStart) { + await handleQuickStart(); + return; + } + + // Handle interactive full setup + if (validated.interactive && !validated.provider) { + await handleInteractiveSetup(validated); + return; + } + + // Handle non-interactive setup with provided options + await handleNonInteractiveSetup(validated); +} + +/** + * Quick start flow - pick a free provider with minimal prompts + */ +async function handleQuickStart(): Promise<void> { + console.log(chalk.cyan('\n🚀 Quick Start\n')); + + p.intro(chalk.cyan('Quick Setup')); + + // Let user pick from popular free providers + const quickProvider = await p.select({ + message: 'Choose a provider', + options: [ + { + value: 'google' as const, + label: `${chalk.green('●')} Google Gemini`, + hint: 'Free, 1M+ context (recommended)', + }, + { + value: 'groq' as const, + label: `${chalk.green('●')} Groq`, + hint: 'Free, ultra-fast', + }, + { + value: 'local' as const, + label: `${chalk.cyan('●')} Local Models`, + hint: 'Free, private, runs on your machine', + }, + ], + }); + + if (p.isCancel(quickProvider)) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + // Handle local models with dedicated setup flow + if (quickProvider === 'local') { + const localResult = await setupLocalModels(); + if (!hasSelectedModel(localResult)) { + p.cancel('Setup cancelled'); + process.exit(0); + } + const model = getModelFromResult(localResult); + + // CLI mode confirmation for local + const useCli = await p.confirm({ + message: 'Start in Terminal mode? (You can change this later)', + initialValue: true, + }); + + if (p.isCancel(useCli)) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + const defaultMode = useCli ? 'cli' : await selectDefaultMode(); + if (defaultMode === null) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + // Sync the active model for local provider + await setActiveModel(model); + + const preferences = createInitialPreferences({ + provider: 'local', + model, + defaultMode, + setupCompleted: true, + apiKeyPending: false, + }); + + await saveGlobalPreferences(preferences); + + capture('dexto_setup', { + provider: 'local', + model, + setupMode: 'interactive', + setupVariant: 'quick-start', + defaultMode, + apiKeySkipped: false, + }); + + showSetupComplete('local', model, defaultMode, false); + return; + } + + // Cloud provider flow (google or groq) + const provider: LLMProvider = quickProvider; + const model = + getDefaultModelForProvider(provider) || + (provider === 'google' ? 'gemini-2.5-pro' : 'llama-3.3-70b-versatile'); + const apiKeyVar = getProviderEnvVar(provider); + let apiKeySkipped = false; + + // Check if API key exists + const hasKey = hasApiKeyConfigured(provider); + + if (!hasKey) { + const providerName = getProviderDisplayName(provider); + p.note( + `${providerName} is ${chalk.green('free')} to use!\n\n` + + `We'll help you get an API key in just a few seconds.`, + 'Free AI Access' + ); + + const result = await interactiveApiKeySetup(provider, { + exitOnCancel: false, // Don't exit - allow skipping + model, + }); + + if (result.cancelled) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + if (result.skipped || !result.success) { + apiKeySkipped = true; + } + } else { + p.log.success(`API key for ${getProviderDisplayName(provider)} already configured`); + } + + // CLI mode confirmation + const useCli = await p.confirm({ + message: 'Start in Terminal mode? (You can change this later)', + initialValue: true, + }); + + if (p.isCancel(useCli)) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + const defaultMode = useCli ? 'cli' : await selectDefaultMode(); + + // Handle cancellation + if (defaultMode === null) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + // Save preferences + const preferencesOptions: CreatePreferencesOptions = { + provider, + model, + defaultMode, + setupCompleted: true, + apiKeyPending: apiKeySkipped, + }; + // Only include apiKeyVar if not skipped + if (!apiKeySkipped) { + preferencesOptions.apiKeyVar = apiKeyVar; + } + const preferences = createInitialPreferences(preferencesOptions); + + await saveGlobalPreferences(preferences); + + capture('dexto_setup', { + provider, + model, + setupMode: 'interactive', + setupVariant: 'quick-start', + defaultMode, + apiKeySkipped, + }); + + showSetupComplete(provider, model, defaultMode, apiKeySkipped); +} + +/** + * Dexto setup flow - login if needed, select model, save preferences + * + * Config storage: + * - provider: 'dexto' (the gateway provider) + * - model: OpenRouter-style ID (e.g., 'anthropic/claude-haiku-4.5') + * + * Runtime handles routing requests through the Dexto gateway to the underlying provider. + */ +async function handleDextoProviderSetup(): Promise<void> { + console.log(chalk.magenta('\n★ Dexto Setup\n')); + + // Check if user already has DEXTO_API_KEY + const hasKey = await canUseDextoProvider(); + + if (!hasKey) { + p.note( + `Dexto gives you instant access to ${chalk.cyan('all AI models')} with a single account.\n\n` + + `We'll open your browser to sign in or create an account.`, + 'Login Required' + ); + + const shouldLogin = await p.confirm({ + message: 'Continue with browser login?', + initialValue: true, + }); + + if (p.isCancel(shouldLogin) || !shouldLogin) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + try { + await handleBrowserLogin(); + // Verify key was actually provisioned (provisionKeys silently catches errors) + if (!(await canUseDextoProvider())) { + p.log.error( + 'API key provisioning failed. Please try again or use `dexto setup` with a different provider.' + ); + process.exit(1); + } + p.log.success('Login successful! Continuing with setup...'); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + p.log.error(`Login failed: ${errorMessage}`); + p.cancel('Setup cancelled - login required for Dexto'); + process.exit(1); + } + } else { + p.log.success('Already logged in to Dexto'); + } + + // Model selection - show popular models in OpenRouter format + // NOTE: This list is intentionally hardcoded (not from registry) to include + // curated hints for onboarding UX. Keep model IDs in sync with: + // packages/core/src/llm/registry.ts (LLM_REGISTRY.dexto.models) + const model = await p.select({ + message: 'Select a model to start with', + options: [ + // Claude models (Anthropic via Dexto gateway) + { + value: 'anthropic/claude-haiku-4.5', + label: 'Claude 4.5 Haiku', + hint: 'Fast & affordable (recommended)', + }, + { + value: 'anthropic/claude-sonnet-4.5', + label: 'Claude 4.5 Sonnet', + hint: 'Balanced performance and cost', + }, + { + value: 'anthropic/claude-opus-4.5', + label: 'Claude 4.5 Opus', + hint: 'Most capable Claude model', + }, + // OpenAI models (via Dexto gateway) + { + value: 'openai/gpt-5.2', + label: 'GPT-5.2', + hint: 'OpenAI flagship model', + }, + { + value: 'openai/gpt-5.2-codex', + label: 'GPT-5.2 Codex', + hint: 'Optimized for coding', + }, + // Google models (via Dexto gateway) + { + value: 'google/gemini-3-pro-preview', + label: 'Gemini 3 Pro', + hint: 'Google flagship model', + }, + { + value: 'google/gemini-3-flash-preview', + label: 'Gemini 3 Flash', + hint: 'Fast and efficient', + }, + // Free models (via Dexto gateway) + { + value: 'qwen/qwen3-coder:free', + label: 'Qwen3 Coder (Free)', + hint: 'Free coding model, 262k context', + }, + { + value: 'deepseek/deepseek-r1-0528:free', + label: 'DeepSeek R1 (Free)', + hint: 'Free reasoning model, 163k context', + }, + ], + }); + + if (p.isCancel(model)) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + // Dexto setup always uses 'dexto' provider with OpenRouter model IDs + const provider: LLMProvider = 'dexto'; + + // Cast model to string (prompts library typing) + const selectedModel = model as string; + + p.log.info(`${chalk.dim('Tip:')} You can switch models anytime with ${chalk.cyan('/model')}`); + + // Ask about default mode + const defaultMode = await selectDefaultMode(); + + if (defaultMode === null) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + // Save preferences with explicit dexto provider and OpenRouter model ID + const preferences = createInitialPreferences({ + provider, + model: selectedModel, + defaultMode, + setupCompleted: true, + apiKeyPending: false, + apiKeyVar: 'DEXTO_API_KEY', + }); + + await saveGlobalPreferences(preferences); + + capture('dexto_setup', { + provider, + model: selectedModel, + setupMode: 'interactive', + setupVariant: 'dexto', + defaultMode, + }); + + showSetupComplete(provider, selectedModel, defaultMode, false); +} + +/** + * Full interactive setup flow with wizard navigation. + * Users can go back to previous steps to change their selections. + */ +async function handleInteractiveSetup(_options: CLISetupOptions): Promise<void> { + console.log(chalk.cyan('\n🗿 Dexto Setup\n')); + + p.intro(chalk.cyan("Let's configure your AI agent")); + + // Initialize wizard state + let state: SetupWizardState = { step: 'setupType' }; + + // Wizard loop - process steps until complete + while (state.step !== 'complete') { + switch (state.step) { + case 'setupType': + state = await wizardStepSetupType(state); + break; + case 'provider': + state = await wizardStepProvider(state); + break; + case 'model': + state = await wizardStepModel(state); + break; + case 'reasoningEffort': + state = await wizardStepReasoningEffort(state); + break; + case 'apiKey': + state = await wizardStepApiKey(state); + break; + case 'mode': + state = await wizardStepMode(state); + break; + } + } + + // Save preferences and show completion + // Quick start handles its own saving, so skip for that path + if (!state.quickStartHandled) { + await saveWizardPreferences(state); + } +} + +/** + * Wizard Step: Setup Type (Quick Start vs Custom) + */ +async function wizardStepSetupType(state: SetupWizardState): Promise<SetupWizardState> { + // Build options list - only show Dexto Credits when feature is enabled + const options: Array<{ value: string; label: string; hint: string }> = []; + + if (isDextoAuthEnabled()) { + options.push({ + value: 'dexto', + label: `${chalk.magenta('★')} Dexto Credits`, + hint: 'All models, one account - login to get started (recommended)', + }); + } + + options.push( + { + value: 'quick', + label: `${chalk.green('●')} Quick Start`, + hint: 'Google Gemini (free) - no account needed', + }, + { + value: 'custom', + label: `${chalk.blue('●')} Custom Setup`, + hint: 'Choose your provider (OpenAI, Anthropic, Ollama, etc.)', + } + ); + + const setupType = await p.select({ + message: 'How would you like to set up Dexto?', + options, + }); + + if (p.isCancel(setupType)) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + if (setupType === 'dexto') { + // Handle Dexto Credits flow - login if needed, then proceed to model selection + await handleDextoProviderSetup(); + return { ...state, step: 'complete', quickStartHandled: true }; + } + + if (setupType === 'quick') { + // Quick start bypasses the wizard - handle it directly + await handleQuickStart(); + return { ...state, step: 'complete', quickStartHandled: true }; + } + + return { ...state, step: 'provider', setupType: 'custom' }; +} + +/** + * Wizard Step: Provider Selection + */ +async function wizardStepProvider(state: SetupWizardState): Promise<SetupWizardState> { + showStepProgress('provider', state.provider); + + const provider = await selectProvider(); + + if (provider === null) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + if (provider === '_back') { + return { ...state, step: 'setupType', provider: undefined }; + } + + return { ...state, step: 'model', provider }; +} + +/** + * Wizard Step: Model Selection + */ +async function wizardStepModel(state: SetupWizardState): Promise<SetupWizardState> { + const provider = state.provider!; + showStepProgress('model', provider); + + // Handle local providers with special setup flow + if (provider === 'local') { + const localResult = await setupLocalModels(); + // Only proceed if user actually selected a model + // (handles cancelled, back, skipped, and any other incomplete states) + if (!hasSelectedModel(localResult)) { + return { ...state, step: 'provider', model: undefined }; + } + const model = getModelFromResult(localResult); + // Check if model supports reasoning effort + const nextStep = isReasoningCapableModel(model) ? 'reasoningEffort' : 'mode'; + return { ...state, step: nextStep, model }; + } + + if (provider === 'ollama') { + const ollamaResult = await setupOllamaModels(); + // Only proceed if user actually selected a model + if (!hasSelectedModel(ollamaResult)) { + return { ...state, step: 'provider', model: undefined }; + } + const model = getModelFromResult(ollamaResult); + // Check if model supports reasoning effort + const nextStep = isReasoningCapableModel(model) ? 'reasoningEffort' : 'mode'; + return { ...state, step: nextStep, model }; + } + + // Handle baseURL for providers that need it + let baseURL: string | undefined; + if (providerRequiresBaseURL(provider)) { + const result = await promptForBaseURL(provider); + if (result === null) { + // User cancelled - go back to provider selection + return { ...state, step: 'provider', model: undefined, baseURL: undefined }; + } + baseURL = result; + } + + // Cloud provider model selection with back option + const model = await selectModelWithBack(provider); + + if (model === '_back') { + return { ...state, step: 'provider', model: undefined, baseURL: undefined }; + } + + // Check if model supports reasoning effort + const nextStep = isReasoningCapableModel(model) ? 'reasoningEffort' : 'apiKey'; + return { ...state, step: nextStep, model, baseURL }; +} + +/** + * Wizard Step: Reasoning Effort Selection (for OpenAI reasoning models) + */ +async function wizardStepReasoningEffort(state: SetupWizardState): Promise<SetupWizardState> { + const provider = state.provider!; + const model = state.model!; + const isLocalProvider = provider === 'local' || provider === 'ollama'; + showStepProgress('reasoningEffort', provider, model); + + const result = await p.select({ + message: 'Select reasoning effort level', + options: [ + { + value: 'medium' as const, + label: 'Medium (Recommended)', + hint: 'Balanced reasoning for most tasks', + }, + { value: 'low' as const, label: 'Low', hint: 'Light reasoning, fast responses' }, + { + value: 'minimal' as const, + label: 'Minimal', + hint: 'Barely any reasoning, very fast', + }, + { value: 'high' as const, label: 'High', hint: 'More thorough reasoning' }, + { + value: 'xhigh' as const, + label: 'Extra High', + hint: 'Maximum reasoning, slower responses', + }, + { value: 'none' as const, label: 'None', hint: 'Disable reasoning' }, + { value: '_back' as const, label: chalk.gray('← Back'), hint: 'Change model' }, + ], + }); + + if (p.isCancel(result)) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + if (result === '_back') { + return { ...state, step: 'model', reasoningEffort: undefined }; + } + + // Determine next step based on provider type + const nextStep = isLocalProvider ? 'mode' : 'apiKey'; + return { ...state, step: nextStep, reasoningEffort: result }; +} + +/** + * Wizard Step: API Key Configuration + */ +async function wizardStepApiKey(state: SetupWizardState): Promise<SetupWizardState> { + const provider = state.provider!; + const model = state.model!; + showStepProgress('apiKey', provider, model); + + const hasKey = hasApiKeyConfigured(provider); + const needsApiKey = requiresApiKey(provider); + + if (needsApiKey && !hasKey) { + const result = await interactiveApiKeySetup(provider, { + exitOnCancel: false, + model, + }); + + if (result.cancelled) { + // Go back to reasoning effort if model supports it, otherwise model selection + const prevStep = isReasoningCapableModel(model) ? 'reasoningEffort' : 'model'; + return { ...state, step: prevStep, apiKeySkipped: undefined }; + } + + const apiKeySkipped = result.skipped || !result.success; + return { ...state, step: 'mode', apiKeySkipped }; + } else if (needsApiKey && hasKey) { + p.log.success(`API key for ${getProviderDisplayName(provider)} already configured`); + } else if (!needsApiKey) { + p.log.info(`${getProviderDisplayName(provider)} does not require an API key`); + } + + return { ...state, step: 'mode', apiKeySkipped: false }; +} + +/** + * Wizard Step: Default Mode Selection + */ +async function wizardStepMode(state: SetupWizardState): Promise<SetupWizardState> { + const provider = state.provider!; + const model = state.model!; + const isLocalProvider = provider === 'local' || provider === 'ollama'; + const hasReasoningStep = isReasoningCapableModel(model); + showStepProgress('mode', provider, model); + + const mode = await selectDefaultModeWithBack(); + + if (mode === '_back') { + // Go back to previous step based on provider type and model capabilities + if (isLocalProvider) { + // Local: reasoning effort -> model + return { + ...state, + step: hasReasoningStep ? 'reasoningEffort' : 'model', + defaultMode: undefined, + }; + } + // Cloud: always go back to apiKey (reasoning effort comes before apiKey) + return { ...state, step: 'apiKey', defaultMode: undefined }; + } + + return { ...state, step: 'complete', defaultMode: mode }; +} + +/** + * Select model with back option + */ +async function selectModelWithBack(provider: LLMProvider): Promise<string | '_back'> { + const providerInfo = LLM_REGISTRY[provider]; + + if (providerInfo?.models && providerInfo.models.length > 0) { + const modelOptions = providerInfo.models.map((m) => ({ + value: m.name, + label: m.displayName || m.name, + })); + + const result = await p.select({ + message: `Select a model for ${getProviderDisplayName(provider)}`, + options: [ + ...modelOptions, + { value: '_back' as const, label: chalk.gray('← Back'), hint: 'Change provider' }, + ], + }); + + if (p.isCancel(result)) { + p.cancel('Setup cancelled'); + process.exit(0); + } + + return result as string | '_back'; + } + + // For providers that accept any model, show text input with back hint + p.log.info(chalk.gray('Press Ctrl+C to go back')); + const defaultModel = providerInfo?.models?.find((m) => m.default)?.name; + const model = await p.text({ + message: `Enter model name for ${getProviderDisplayName(provider)}`, + placeholder: defaultModel || 'e.g., gpt-4-turbo', + validate: (value) => { + if (!value.trim()) return 'Model name is required'; + return undefined; + }, + }); + + if (p.isCancel(model)) { + return '_back'; + } + + return model as string; +} + +/** + * Select default mode with back option + */ +async function selectDefaultModeWithBack(): Promise< + 'cli' | 'web' | 'server' | 'discord' | 'telegram' | 'mcp' | '_back' +> { + const result = await p.select({ + message: 'How do you want to use Dexto by default?', + options: [ + { + value: 'cli' as const, + label: `${chalk.green('●')} Terminal`, + hint: 'Chat in your terminal (most popular)', + }, + { + value: 'web' as const, + label: `${chalk.blue('●')} Browser`, + hint: 'Web UI at localhost:3000', + }, + { + value: 'server' as const, + label: `${chalk.cyan('●')} API Server`, + hint: 'REST API for integrations', + }, + { value: '_back' as const, label: chalk.gray('← Back'), hint: 'Go to previous step' }, + ], + }); + + if (p.isCancel(result)) { + return '_back'; + } + + return result as 'cli' | 'web' | 'server' | 'discord' | 'telegram' | 'mcp' | '_back'; +} + +/** + * Save wizard state to preferences + */ +async function saveWizardPreferences(state: SetupWizardState): Promise<void> { + const provider = state.provider!; + const model = state.model!; + const defaultMode = state.defaultMode!; + const apiKeySkipped = state.apiKeySkipped || false; + + const needsApiKey = requiresApiKey(provider); + const apiKeyVar = getProviderEnvVar(provider); + + // For local provider, sync the active model in state.json + if (provider === 'local') { + await setActiveModel(model); + } + + // Build preferences options + const preferencesOptions: CreatePreferencesOptions = { + provider, + model, + defaultMode, + setupCompleted: true, + apiKeyPending: apiKeySkipped, + }; + + if (needsApiKey && !apiKeySkipped) { + preferencesOptions.apiKeyVar = apiKeyVar; + } + if (state.baseURL) { + preferencesOptions.baseURL = state.baseURL; + } + if (state.reasoningEffort) { + preferencesOptions.reasoningEffort = state.reasoningEffort; + } + + const preferences = createInitialPreferences(preferencesOptions); + await saveGlobalPreferences(preferences); + + // Analytics + capture('dexto_setup', { + provider, + model, + setupMode: 'interactive', + setupVariant: 'custom', + defaultMode, + hasBaseURL: Boolean(state.baseURL), + apiKeySkipped, + }); + + showSetupComplete(provider, model, defaultMode, apiKeySkipped); +} + +/** + * Non-interactive setup with CLI options + */ +async function handleNonInteractiveSetup(options: CLISetupOptions): Promise<void> { + const provider = options.provider!; + const model = options.model || getDefaultModel(provider); + + if (!model) { + console.error(chalk.red(`❌ Model is required for provider '${provider}'.`)); + console.error(chalk.gray(` Use --model option to specify a model.`)); + process.exit(1); + } + + const apiKeyVar = getProviderEnvVar(provider); + const hadApiKeyBefore = Boolean(resolveApiKeyForProvider(provider)); + + const preferences = createInitialPreferences({ + provider, + model, + apiKeyVar, + defaultAgent: options.defaultAgent, + setupCompleted: true, + }); + + await saveGlobalPreferences(preferences); + + // For local provider, sync the active model in state.json + if (provider === 'local') { + await setActiveModel(model); + } + + capture('dexto_setup', { + provider, + model, + hadApiKeyBefore, + setupMode: 'non-interactive', + }); + + console.log(chalk.green('\n✨ Setup complete! Dexto is ready to use.\n')); +} + +/** + * Settings menu for users who already have setup complete. + * Loops back to menu after each action until user exits. + */ +async function showSettingsMenu(): Promise<void> { + p.intro(chalk.cyan('⚙️ Dexto Settings')); + + // Settings menu loop - returns to menu after each action + while (true) { + // Load current preferences (refresh each iteration) + let currentPrefs; + try { + currentPrefs = await loadGlobalPreferences(); + } catch { + currentPrefs = null; + } + + // Show current configuration + if (currentPrefs) { + const currentConfig = [ + `Provider: ${chalk.cyan(getProviderDisplayName(currentPrefs.llm.provider))}`, + `Model: ${chalk.cyan(currentPrefs.llm.model)}`, + `Default Mode: ${chalk.cyan(currentPrefs.defaults.defaultMode)}`, + ...(currentPrefs.llm.baseURL + ? [`Base URL: ${chalk.cyan(currentPrefs.llm.baseURL)}`] + : []), + ...(currentPrefs.llm.reasoningEffort + ? [`Reasoning Effort: ${chalk.cyan(currentPrefs.llm.reasoningEffort)}`] + : []), + ].join('\n'); + + p.note(currentConfig, 'Current Configuration'); + } + + const action = await p.select({ + message: 'What would you like to do?', + options: [ + { + value: 'model', + label: 'Change model', + hint: `Currently: ${currentPrefs?.llm.provider || 'not set'} / ${currentPrefs?.llm.model || 'not set'}`, + }, + { + value: 'mode', + label: 'Change default mode', + hint: `Currently: ${currentPrefs?.defaults.defaultMode || 'web'}`, + }, + { + value: 'apikey', + label: 'Update API key', + hint: 'Re-enter your API key', + }, + { + value: 'reset', + label: 'Reset to defaults', + hint: 'Start fresh with a new configuration', + }, + { + value: 'file', + label: 'View preferences file', + hint: 'See where your settings are stored', + }, + { + value: 'exit', + label: 'Exit', + hint: 'Done making changes', + }, + ], + }); + + // Exit conditions + if (p.isCancel(action) || action === 'exit') { + p.outro(chalk.gray('Settings closed')); + return; + } + + // Execute action and loop back (except for reset which exits) + switch (action) { + case 'model': + await changeModel(); // Always prompt for provider selection + break; + case 'mode': + await changeDefaultMode(); + break; + case 'apikey': + await updateApiKey(currentPrefs?.llm.provider); + break; + case 'reset': { + // Reset exits the menu after completion, but returns to menu if cancelled + const resetCompleted = await resetSetup(); + if (resetCompleted) { + return; + } + break; + } + case 'file': + showPreferencesFilePath(); + break; + } + + // Add separator before next iteration + console.log(''); + } +} + +/** + * Change model setting (includes provider selection) + */ +async function changeModel(currentProvider?: LLMProvider): Promise<void> { + let provider: LLMProvider | null | '_back' = currentProvider ?? null; + + // If no provider specified, show selection + // When Dexto auth is enabled, show Dexto/Other choice first (matching first-time setup flow) + if (!provider && isDextoAuthEnabled()) { + const providerChoice = await p.select({ + message: 'Choose your model source', + options: [ + { + value: 'dexto', + label: `${chalk.magenta('★')} Dexto Credits`, + hint: 'All models, one account', + }, + { + value: 'other', + label: `${chalk.blue('●')} Other providers`, + hint: 'OpenAI, Anthropic, Gemini, Ollama, etc.', + }, + ], + }); + + if (p.isCancel(providerChoice)) { + p.log.warn('Model change cancelled'); + return; + } + + if (providerChoice === 'dexto') { + // Use the same Dexto setup flow as first-time setup + await handleDextoProviderSetup(); + return; + } + + // 'other' - fall through to normal provider selection + } + + // Get provider if not already set + if (!provider) { + provider = await selectProvider(); + } + + // Handle cancellation or back from selectProvider + if (provider === null || provider === '_back') { + p.log.warn('Model change cancelled'); + return; + } + + // Special handling for local providers - use dedicated setup flows + if (provider === 'local') { + const localResult = await setupLocalModels(); + // Only proceed if user actually selected a model + // (handles cancelled, back, skipped - returns to menu without saving) + if (!hasSelectedModel(localResult)) { + p.log.warn('Model change cancelled'); + return; + } + const model = getModelFromResult(localResult); + const llmUpdate: { + provider: LLMProvider; + model: string; + reasoningEffort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; + } = { + provider, + model, + }; + + // Ask for reasoning effort if applicable + if (isReasoningCapableModel(model)) { + const reasoningEffort = await selectReasoningEffort(); + if (reasoningEffort !== null) { + llmUpdate.reasoningEffort = reasoningEffort; + } + } + + await updateGlobalPreferences({ llm: llmUpdate }); + p.log.success(`Model changed to ${model}`); + return; + } + + if (provider === 'ollama') { + const ollamaResult = await setupOllamaModels(); + // Only proceed if user actually selected a model + if (!hasSelectedModel(ollamaResult)) { + p.log.warn('Model change cancelled'); + return; + } + const model = getModelFromResult(ollamaResult); + const llmUpdate: { + provider: LLMProvider; + model: string; + reasoningEffort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; + } = { + provider, + model, + }; + + // Ask for reasoning effort if applicable + if (isReasoningCapableModel(model)) { + const reasoningEffort = await selectReasoningEffort(); + if (reasoningEffort !== null) { + llmUpdate.reasoningEffort = reasoningEffort; + } + } + + await updateGlobalPreferences({ llm: llmUpdate }); + p.log.success(`Model changed to ${model}`); + return; + } + + // Standard flow for cloud providers + const model = await selectModel(provider); + + // Handle cancellation from selectModel + if (model === null) { + p.log.warn('Model change cancelled'); + return; + } + + const apiKeyVar = getProviderEnvVar(provider); + const needsApiKey = requiresApiKey(provider); + + const llmUpdate: { + provider: LLMProvider; + model: string; + apiKey?: string; + reasoningEffort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; + } = { + provider, + model, + }; + // Only include apiKey for providers that need it + if (needsApiKey) { + llmUpdate.apiKey = `$${apiKeyVar}`; + } + + // Ask for reasoning effort if applicable + if (isReasoningCapableModel(model)) { + const reasoningEffort = await selectReasoningEffort(); + if (reasoningEffort !== null) { + llmUpdate.reasoningEffort = reasoningEffort; + } + } + + await updateGlobalPreferences({ llm: llmUpdate }); + + p.log.success(`Model changed to ${model}`); +} + +/** + * Change default mode setting + */ +async function changeDefaultMode(): Promise<void> { + const mode = await selectDefaultMode(); + + // Handle cancellation + if (mode === null) { + p.log.warn('Mode change cancelled'); + return; + } + + await updateGlobalPreferences({ + defaults: { defaultMode: mode }, + }); + + p.log.success(`Default mode changed to ${mode}`); +} + +/** + * Update API key for current provider + */ +async function updateApiKey(currentProvider?: LLMProvider): Promise<void> { + const provider = currentProvider || (await selectProvider()); + + // Handle cancellation or back from selectProvider + if (provider === null || provider === '_back') { + p.log.warn('API key update cancelled'); + return; + } + + // Handle providers that use non-API-key authentication + if (provider === 'vertex') { + p.note( + `Google Vertex AI uses Application Default Credentials (ADC).\n\n` + + `To authenticate:\n` + + ` 1. Install gcloud CLI: https://cloud.google.com/sdk/docs/install\n` + + ` 2. Run: gcloud auth application-default login\n` + + ` 3. Set GOOGLE_VERTEX_PROJECT environment variable`, + 'Google Cloud Authentication' + ); + return; + } + + if (provider === 'bedrock') { + p.note( + `Amazon Bedrock uses AWS credentials.\n\n` + + `To authenticate, set these environment variables:\n` + + ` • AWS_REGION (required)\n` + + ` • AWS_ACCESS_KEY_ID (required)\n` + + ` • AWS_SECRET_ACCESS_KEY (required)\n` + + ` • AWS_SESSION_TOKEN (optional, for temporary credentials)`, + 'AWS Authentication' + ); + return; + } + + // For openai-compatible and litellm, API keys are optional but allowed + // Show a note but still allow setting one + if (provider === 'openai-compatible' || provider === 'litellm') { + const wantsKey = await p.confirm({ + message: `API key is optional for ${getProviderDisplayName(provider)}. Set one anyway?`, + initialValue: true, + }); + + if (p.isCancel(wantsKey) || !wantsKey) { + p.log.info('Skipped API key setup'); + return; + } + } + + const result = await interactiveApiKeySetup(provider, { + exitOnCancel: false, + skipVerification: false, + }); + + if (result.success) { + p.log.success(`API key updated for ${getProviderDisplayName(provider)}`); + } else { + p.log.warn('API key update cancelled'); + } +} + +/** + * Reset setup to start fresh + */ +async function resetSetup(): Promise<boolean> { + const confirm = await p.confirm({ + message: 'This will erase your current configuration. Continue?', + initialValue: false, + }); + + if (p.isCancel(confirm) || !confirm) { + p.log.info('Reset cancelled'); + return false; // Indicate reset was cancelled + } + + // Run full setup - this will complete a fresh setup + await handleInteractiveSetup({ + interactive: true, + force: true, + defaultAgent: 'coding-agent', + quickStart: false, + }); + + return true; // Indicate reset completed +} + +/** + * Show preferences file path + */ +function showPreferencesFilePath(): void { + const prefsPath = getGlobalPreferencesPath(); + + p.note( + [ + `Your preferences are stored at:`, + ``, + ` ${chalk.cyan(prefsPath)}`, + ``, + `You can edit this file directly with any text editor.`, + `Changes take effect on the next run of dexto.`, + ``, + chalk.gray('Example commands:'), + chalk.gray(` code ${prefsPath} # Open in VS Code`), + chalk.gray(` nano ${prefsPath} # Edit in terminal`), + chalk.gray(` cat ${prefsPath} # View contents`), + ].join('\n'), + 'Preferences File Location' + ); +} + +/** + * Select default mode interactively + * Returns null if user cancels + */ +async function selectDefaultMode(): Promise< + 'cli' | 'web' | 'server' | 'discord' | 'telegram' | 'mcp' | null +> { + const mode = await p.select({ + message: 'How do you want to use Dexto by default?', + options: [ + { + value: 'cli' as const, + label: `${chalk.green('●')} Terminal`, + hint: 'Chat in your terminal (most popular)', + }, + { + value: 'web' as const, + label: `${chalk.blue('●')} Browser`, + hint: 'Web UI at localhost:3000', + }, + { + value: 'server' as const, + label: `${chalk.cyan('●')} API Server`, + hint: 'REST API for integrations', + }, + ], + }); + + if (p.isCancel(mode)) { + return null; + } + + return mode; +} + +/** + * Select reasoning effort level for reasoning-capable models + * Used in settings menu when changing to a reasoning-capable model + */ +async function selectReasoningEffort(): Promise< + 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | null +> { + const effort = await p.select({ + message: 'Select reasoning effort level', + options: [ + { + value: 'medium' as const, + label: 'Medium (Recommended)', + hint: 'Balanced reasoning for most tasks', + }, + { value: 'low' as const, label: 'Low', hint: 'Light reasoning, fast responses' }, + { + value: 'minimal' as const, + label: 'Minimal', + hint: 'Barely any reasoning, very fast', + }, + { value: 'high' as const, label: 'High', hint: 'More thorough reasoning' }, + { + value: 'xhigh' as const, + label: 'Extra High', + hint: 'Maximum reasoning, slower responses', + }, + { value: 'none' as const, label: 'None', hint: 'Disable reasoning' }, + ], + }); + + if (p.isCancel(effort)) { + return null; + } + + return effort; +} + +/** + * Select model interactively + * Returns null if user cancels + */ +async function selectModel(provider: LLMProvider): Promise<string | null> { + const providerInfo = LLM_REGISTRY[provider]; + + // For providers with a fixed model list + if (providerInfo?.models && providerInfo.models.length > 0) { + const options = providerInfo.models.map((m) => { + const option: { value: string; label: string; hint?: string } = { + value: m.name, + label: m.displayName || m.name, + }; + if (m.default) { + option.hint = '(default)'; + } + return option; + }); + + const selected = await p.select({ + message: `Select a model for ${getProviderDisplayName(provider)}`, + options, + initialValue: providerInfo.models.find((m) => m.default)?.name, + }); + + if (p.isCancel(selected)) { + return null; + } + + return selected as string; + } + + // For providers that accept any model (openai-compatible, openrouter, etc.) + const modelInput = await p.text({ + message: `Enter model name for ${getProviderDisplayName(provider)}`, + placeholder: + provider === 'openrouter' ? 'e.g., anthropic/claude-3.5-sonnet' : 'e.g., llama-3-70b', + validate: (value) => { + if (!value || value.trim().length === 0) { + return 'Model name is required'; + } + return undefined; + }, + }); + + if (p.isCancel(modelInput)) { + return null; + } + + return modelInput.trim(); +} + +/** + * Prompt for base URL for custom endpoints + * Returns null if user cancels (to go back) + */ +async function promptForBaseURL(provider: LLMProvider): Promise<string | null> { + p.log.info(chalk.gray('Press Ctrl+C to go back')); + + const placeholder = + provider === 'openai-compatible' + ? 'http://localhost:11434/v1' + : provider === 'litellm' + ? 'http://localhost:4000' + : 'https://your-api-endpoint.com/v1'; + + const baseURL = await p.text({ + message: `Enter base URL for ${getProviderDisplayName(provider)}`, + placeholder, + validate: (value) => { + if (!value || value.trim().length === 0) { + return 'Base URL is required for this provider'; + } + try { + new URL(value.trim()); + } catch { + return 'Please enter a valid URL'; + } + return undefined; + }, + }); + + if (p.isCancel(baseURL)) { + return null; + } + + return baseURL.trim(); +} + +/** + * Show setup complete message + */ +function showSetupComplete( + provider: LLMProvider, + model: string, + defaultMode: string, + apiKeySkipped: boolean = false +): void { + const modeCommand = defaultMode === 'web' ? 'dexto' : `dexto --mode ${defaultMode}`; + const isLocalProvider = provider === 'local' || provider === 'ollama'; + + if (apiKeySkipped) { + console.log(chalk.rgb(255, 165, 0)('\n⚠️ Setup complete (API key pending)\n')); + } else { + console.log(chalk.green('\n✨ Setup complete! Dexto is ready to use.\n')); + } + + const summary = [ + `${chalk.bold('Configuration:')}`, + ` Provider: ${chalk.cyan(getProviderDisplayName(provider))}`, + ` Model: ${chalk.cyan(model)}`, + ` Mode: ${chalk.cyan(defaultMode)}`, + ...(apiKeySkipped + ? [ + ` API Key: ${chalk.rgb(255, 165, 0)('Not configured')}`, + ``, + `${chalk.bold('To complete setup:')}`, + ` Run ${chalk.cyan('dexto setup')} to add your API key`, + ` Or set ${chalk.cyan(getProviderEnvVar(provider))} in your environment`, + ] + : []), + ``, + `${chalk.bold('Next steps:')}`, + ` Run ${chalk.cyan(modeCommand)} to start`, + ` Run ${chalk.cyan('dexto setup')} to change settings`, + ...(isLocalProvider + ? [` Run ${chalk.cyan('dexto setup')} again to manage local models`] + : []), + ` Run ${chalk.cyan('dexto --help')} for more options`, + ].join('\n'); + + console.log(summary); + console.log(''); +} diff --git a/dexto/packages/cli/src/cli/commands/sync-agents.test.ts b/dexto/packages/cli/src/cli/commands/sync-agents.test.ts new file mode 100644 index 00000000..f4a7c302 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/sync-agents.test.ts @@ -0,0 +1,212 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { promises as fs } from 'fs'; +import { shouldPromptForSync, markSyncDismissed, clearSyncDismissed } from './sync-agents.js'; + +// Mock fs module +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + promises: { + readFile: vi.fn(), + writeFile: vi.fn(), + mkdir: vi.fn(), + readdir: vi.fn(), + stat: vi.fn(), + access: vi.fn(), + unlink: vi.fn(), + }, + }; +}); + +// Mock agent-management +const mockLoadBundledRegistryAgents = vi.fn(); +const mockResolveBundledScript = vi.fn(); + +vi.mock('@dexto/agent-management', () => ({ + getDextoGlobalPath: vi.fn((type: string, filename?: string) => { + if (type === 'agents') return '/mock/.dexto/agents'; + if (type === 'cache') + return filename ? `/mock/.dexto/cache/${filename}` : '/mock/.dexto/cache'; + return '/mock/.dexto'; + }), + loadBundledRegistryAgents: () => mockLoadBundledRegistryAgents(), + resolveBundledScript: (path: string) => mockResolveBundledScript(path), + copyDirectory: vi.fn(), +})); + +describe('sync-agents', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe('shouldPromptForSync', () => { + it('returns false when sync was dismissed for current version', async () => { + vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify({ version: '1.0.0' })); + + const result = await shouldPromptForSync('1.0.0'); + + expect(result).toBe(false); + }); + + it('checks for updates when not dismissed', async () => { + // Dismissed file doesn't exist + vi.mocked(fs.readFile).mockRejectedValue(new Error('ENOENT')); + + // No installed agents + vi.mocked(fs.readdir).mockResolvedValue([]); + + mockLoadBundledRegistryAgents.mockReturnValue({ + 'test-agent': { id: 'test-agent', name: 'Test Agent', source: 'test-agent/' }, + }); + + const result = await shouldPromptForSync('1.0.0'); + + expect(result).toBe(false); // No installed agents = nothing to sync + }); + + it('returns true when installed agent differs from bundled', async () => { + // Not dismissed + vi.mocked(fs.readFile).mockImplementation(async (path) => { + if (String(path).includes('sync-dismissed')) { + throw new Error('ENOENT'); + } + // Return different content for bundled vs installed to simulate hash mismatch + if (String(path).includes('bundled')) { + return Buffer.from('bundled content v2'); + } + return Buffer.from('installed content v1'); + }); + + // One installed agent + vi.mocked(fs.readdir).mockResolvedValue([ + { name: 'test-agent', isDirectory: () => true } as any, + ]); + + // Mock stat to return file (not directory) for simpler hash + vi.mocked(fs.stat).mockResolvedValue({ isDirectory: () => false } as any); + + mockLoadBundledRegistryAgents.mockReturnValue({ + 'test-agent': { + id: 'test-agent', + name: 'Test Agent', + source: 'test-agent.yml', + description: 'A test agent', + author: 'Test', + tags: [], + }, + }); + + mockResolveBundledScript.mockReturnValue('/bundled/agents/test-agent.yml'); + + const result = await shouldPromptForSync('1.0.0'); + + expect(result).toBe(true); + }); + + it('returns false when all installed agents match bundled', async () => { + // Not dismissed + vi.mocked(fs.readFile).mockImplementation(async (path) => { + if (String(path).includes('sync-dismissed')) { + throw new Error('ENOENT'); + } + // Return same content for both + return Buffer.from('same content'); + }); + + vi.mocked(fs.readdir).mockResolvedValue([ + { name: 'test-agent', isDirectory: () => true } as any, + ]); + + vi.mocked(fs.stat).mockResolvedValue({ isDirectory: () => false } as any); + + mockLoadBundledRegistryAgents.mockReturnValue({ + 'test-agent': { + id: 'test-agent', + name: 'Test Agent', + source: 'test-agent.yml', + description: 'A test agent', + author: 'Test', + tags: [], + }, + }); + + mockResolveBundledScript.mockReturnValue('/bundled/agents/test-agent.yml'); + + const result = await shouldPromptForSync('1.0.0'); + + expect(result).toBe(false); + }); + + it('skips custom agents not in bundled registry', async () => { + // Not dismissed + vi.mocked(fs.readFile).mockRejectedValue(new Error('ENOENT')); + + // Custom agent installed but not in bundled registry + vi.mocked(fs.readdir).mockResolvedValue([ + { name: 'my-custom-agent', isDirectory: () => true } as any, + ]); + + mockLoadBundledRegistryAgents.mockReturnValue({ + 'test-agent': { id: 'test-agent', name: 'Test Agent', source: 'test-agent/' }, + }); + + const result = await shouldPromptForSync('1.0.0'); + + expect(result).toBe(false); // Custom agent is skipped + }); + + it('returns false on error', async () => { + vi.mocked(fs.readFile).mockRejectedValue(new Error('Some error')); + mockLoadBundledRegistryAgents.mockImplementation(() => { + throw new Error('Registry error'); + }); + + const result = await shouldPromptForSync('1.0.0'); + + expect(result).toBe(false); + }); + }); + + describe('markSyncDismissed', () => { + it('writes dismissed state to file', async () => { + vi.mocked(fs.mkdir).mockResolvedValue(undefined); + vi.mocked(fs.writeFile).mockResolvedValue(); + + await markSyncDismissed('1.5.0'); + + expect(fs.writeFile).toHaveBeenCalledWith( + expect.stringContaining('sync-dismissed.json'), + JSON.stringify({ version: '1.5.0' }) + ); + }); + + it('does not throw on error', async () => { + vi.mocked(fs.mkdir).mockRejectedValue(new Error('Permission denied')); + + await expect(markSyncDismissed('1.5.0')).resolves.not.toThrow(); + }); + }); + + describe('clearSyncDismissed', () => { + it('removes dismissed state file', async () => { + vi.mocked(fs.unlink).mockResolvedValue(); + + await clearSyncDismissed(); + + expect(fs.unlink).toHaveBeenCalledWith(expect.stringContaining('sync-dismissed.json')); + }); + + it('does not throw when file does not exist', async () => { + const error = new Error('ENOENT') as NodeJS.ErrnoException; + error.code = 'ENOENT'; + vi.mocked(fs.unlink).mockRejectedValue(error); + + await expect(clearSyncDismissed()).resolves.not.toThrow(); + }); + }); +}); diff --git a/dexto/packages/cli/src/cli/commands/sync-agents.ts b/dexto/packages/cli/src/cli/commands/sync-agents.ts new file mode 100644 index 00000000..a94ac73d --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/sync-agents.ts @@ -0,0 +1,594 @@ +// packages/cli/src/cli/commands/sync-agents.ts + +import { promises as fs } from 'fs'; +import { createHash } from 'crypto'; +import path from 'path'; +import chalk from 'chalk'; +import * as p from '@clack/prompts'; +import { logger } from '@dexto/core'; +import { + getDextoGlobalPath, + resolveBundledScript, + copyDirectory, + loadBundledRegistryAgents, + type AgentRegistryEntry, +} from '@dexto/agent-management'; + +/** + * Options for the sync-agents command + */ +export interface SyncAgentsCommandOptions { + /** Just list status without updating */ + list?: boolean; + /** Update all without prompting */ + force?: boolean; + /** Minimal output - used when called from startup prompt */ + quiet?: boolean; +} + +/** + * Agent sync status + */ +type AgentStatus = + | 'up_to_date' + | 'changes_available' + | 'not_installed' + | 'custom' // User-installed, not in bundled registry + | 'error'; + +interface AgentInfo { + id: string; + name: string; + description?: string | undefined; + status: AgentStatus; + error?: string | undefined; +} + +/** + * Calculate SHA256 hash of a file + */ +async function hashFile(filePath: string): Promise<string> { + const content = await fs.readFile(filePath); + return createHash('sha256').update(content).digest('hex'); +} + +/** + * Calculate combined hash for a directory + * Hashes all files recursively and combines them + */ +async function hashDirectory(dirPath: string): Promise<string> { + const hash = createHash('sha256'); + const files: string[] = []; + + async function collectFiles(dir: string): Promise<void> { + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + await collectFiles(fullPath); + } else { + files.push(fullPath); + } + } + } + + await collectFiles(dirPath); + + // Sort for consistent ordering + files.sort(); + + for (const file of files) { + const relativePath = path.relative(dirPath, file); + const content = await fs.readFile(file); + hash.update(relativePath); + hash.update(content); + } + + return hash.digest('hex'); +} + +/** + * Get hash of bundled agent + */ +async function getBundledAgentHash(agentEntry: AgentRegistryEntry): Promise<string | null> { + try { + const sourcePath = resolveBundledScript(`agents/${agentEntry.source}`); + const stat = await fs.stat(sourcePath); + + if (stat.isDirectory()) { + return await hashDirectory(sourcePath); + } else { + return await hashFile(sourcePath); + } + } catch (error) { + logger.debug( + `Failed to hash bundled agent: ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } +} + +/** + * Get hash of installed agent + */ +async function getInstalledAgentHash(agentId: string): Promise<string | null> { + try { + const installedPath = path.join(getDextoGlobalPath('agents'), agentId); + const stat = await fs.stat(installedPath); + + if (stat.isDirectory()) { + return await hashDirectory(installedPath); + } else { + return await hashFile(installedPath); + } + } catch (error) { + logger.debug( + `Failed to hash installed agent: ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } +} + +/** + * Check if an agent is installed + */ +async function isAgentInstalled(agentId: string): Promise<boolean> { + try { + const installedPath = path.join(getDextoGlobalPath('agents'), agentId); + await fs.access(installedPath); + return true; + } catch { + return false; + } +} + +/** + * Get list of all installed agent directories + */ +async function getInstalledAgentIds(): Promise<string[]> { + try { + const agentsDir = getDextoGlobalPath('agents'); + const entries = await fs.readdir(agentsDir, { withFileTypes: true }); + return entries.filter((e) => e.isDirectory()).map((e) => e.name); + } catch (error) { + logger.debug( + `Failed to list installed agents: ${error instanceof Error ? error.message : String(error)}` + ); + return []; + } +} + +/** + * Get agent status by comparing bundled vs installed + */ +async function getAgentStatus(agentId: string, agentEntry: AgentRegistryEntry): Promise<AgentInfo> { + const installed = await isAgentInstalled(agentId); + + if (!installed) { + return { + id: agentId, + name: agentEntry.name, + description: agentEntry.description, + status: 'not_installed', + }; + } + + try { + const bundledHash = await getBundledAgentHash(agentEntry); + const installedHash = await getInstalledAgentHash(agentId); + + if (!bundledHash || !installedHash) { + return { + id: agentId, + name: agentEntry.name, + description: agentEntry.description, + status: 'error', + error: 'Could not compute hash', + }; + } + + if (bundledHash === installedHash) { + return { + id: agentId, + name: agentEntry.name, + description: agentEntry.description, + status: 'up_to_date', + }; + } else { + return { + id: agentId, + name: agentEntry.name, + description: agentEntry.description, + status: 'changes_available', + }; + } + } catch (error) { + return { + id: agentId, + name: agentEntry.name, + description: agentEntry.description, + status: 'error', + error: error instanceof Error ? error.message : String(error), + }; + } +} + +// Store sync dismissed state in cache directory +const SYNC_DISMISSED_PATH = getDextoGlobalPath('cache', 'sync-dismissed.json'); + +/** + * Check if sync was dismissed for current version + */ +async function wasSyncDismissed(currentVersion: string): Promise<boolean> { + try { + const content = await fs.readFile(SYNC_DISMISSED_PATH, 'utf-8'); + const data = JSON.parse(content) as { version: string }; + return data.version === currentVersion; + } catch (error) { + logger.debug( + `Could not read sync dismissed state: ${error instanceof Error ? error.message : String(error)}` + ); + return false; + } +} + +/** + * Mark sync as dismissed for current version + */ +export async function markSyncDismissed(currentVersion: string): Promise<void> { + try { + await fs.mkdir(path.dirname(SYNC_DISMISSED_PATH), { recursive: true }); + await fs.writeFile(SYNC_DISMISSED_PATH, JSON.stringify({ version: currentVersion })); + } catch (error) { + logger.debug( + `Could not save sync dismissed state: ${error instanceof Error ? error.message : String(error)}` + ); + } +} + +/** + * Clear sync dismissed state (called after successful sync) + */ +export async function clearSyncDismissed(): Promise<void> { + try { + await fs.unlink(SYNC_DISMISSED_PATH); + } catch (error) { + // File might not exist - only log if it's a different error + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + logger.debug( + `Could not clear sync dismissed state: ${error instanceof Error ? error.message : String(error)}` + ); + } + } +} + +/** + * Quick check if any installed agents have updates available + * + * Used at CLI startup to prompt for sync without full command output. + * Returns true if at least one installed agent differs from bundled + * AND the user hasn't dismissed the prompt for this version. + * + * @param currentVersion Current CLI version to check dismissal state + * @returns true if should prompt for sync + */ +export async function shouldPromptForSync(currentVersion: string): Promise<boolean> { + try { + // Check if user already dismissed for this version + if (await wasSyncDismissed(currentVersion)) { + return false; + } + + const bundledAgents = loadBundledRegistryAgents(); + const installedAgentIds = await getInstalledAgentIds(); + + for (const agentId of installedAgentIds) { + const agentEntry = bundledAgents[agentId]; + // Skip custom agents (not in bundled registry) + if (!agentEntry) continue; + + const bundledHash = await getBundledAgentHash(agentEntry); + const installedHash = await getInstalledAgentHash(agentId); + + if (bundledHash && installedHash && bundledHash !== installedHash) { + return true; + } + } + + return false; + } catch (error) { + logger.debug( + `shouldPromptForSync check failed: ${error instanceof Error ? error.message : String(error)}` + ); + return false; + } +} + +/** + * Update an agent from bundled to installed + */ +async function updateAgent(agentId: string, agentEntry: AgentRegistryEntry): Promise<void> { + const agentsDir = getDextoGlobalPath('agents'); + const targetDir = path.join(agentsDir, agentId); + const sourcePath = resolveBundledScript(`agents/${agentEntry.source}`); + + // Ensure agents directory exists + await fs.mkdir(agentsDir, { recursive: true }); + + // Remove old installation + try { + await fs.rm(targetDir, { recursive: true, force: true }); + } catch { + // Ignore if doesn't exist + } + + // Copy from bundled source + const stat = await fs.stat(sourcePath); + + if (stat.isDirectory()) { + await copyDirectory(sourcePath, targetDir); + } else { + await fs.mkdir(targetDir, { recursive: true }); + const targetFile = path.join(targetDir, path.basename(sourcePath)); + await fs.copyFile(sourcePath, targetFile); + } +} + +/** + * Display agent status with appropriate colors + */ +function formatStatus(status: AgentStatus): string { + switch (status) { + case 'up_to_date': + return chalk.green('Up to date'); + case 'changes_available': + return chalk.yellow('Changes available'); + case 'not_installed': + return chalk.gray('Not installed'); + case 'custom': + return chalk.blue('Custom (user-installed)'); + case 'error': + return chalk.red('Error'); + default: + return chalk.gray('Unknown'); + } +} + +/** + * Main handler for the sync-agents command + * + * @param options Command options + * + * @example + * ```bash + * dexto sync-agents # Interactive - prompt for each + * dexto sync-agents --list # Show what would be updated + * dexto sync-agents --force # Update all without prompting + * ``` + */ +export async function handleSyncAgentsCommand(options: SyncAgentsCommandOptions): Promise<void> { + const { list = false, force = false, quiet = false } = options; + + if (!quiet) { + p.intro(chalk.cyan('Agent Sync')); + } + + const spinner = p.spinner(); + spinner.start('Checking agent configs...'); + + try { + // Load bundled registry (uses existing function from agent-management) + const bundledAgents = loadBundledRegistryAgents(); + const bundledAgentIds = Object.keys(bundledAgents); + + // Get installed agents + const installedAgentIds = await getInstalledAgentIds(); + + // Find custom agents (installed but not in bundled registry) + const customAgentIds = installedAgentIds.filter((id) => !bundledAgents[id]); + + // Check status of all bundled agents + const agentInfos: AgentInfo[] = []; + + for (const agentId of bundledAgentIds) { + const entry = bundledAgents[agentId]; + if (entry) { + const info = await getAgentStatus(agentId, entry); + agentInfos.push(info); + } + } + + // Add custom agents + for (const agentId of customAgentIds) { + agentInfos.push({ + id: agentId, + name: agentId, + status: 'custom', + }); + } + + const updatableAgents = agentInfos.filter((a) => a.status === 'changes_available'); + const upToDateAgents = agentInfos.filter((a) => a.status === 'up_to_date'); + const notInstalledAgents = agentInfos.filter((a) => a.status === 'not_installed'); + const customAgents = agentInfos.filter((a) => a.status === 'custom'); + const errorAgents = agentInfos.filter((a) => a.status === 'error'); + + spinner.stop('Agent check complete'); + + // Quiet mode with force - minimal output for startup prompt + if (quiet && force) { + if (updatableAgents.length === 0) { + p.log.success('All agents up to date'); + return; + } + + const updatedNames: string[] = []; + const failedNames: string[] = []; + + for (const agent of updatableAgents) { + const entry = bundledAgents[agent.id]; + if (entry) { + try { + await updateAgent(agent.id, entry); + updatedNames.push(agent.id); + } catch (error) { + failedNames.push(agent.id); + logger.debug( + `Failed to update ${agent.id}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + } + + if (updatedNames.length > 0) { + p.log.success(`Updated: ${updatedNames.join(', ')}`); + } + if (failedNames.length > 0) { + p.log.warn(`Failed to update: ${failedNames.join(', ')}`); + } + return; + } + + // Display full status (non-quiet mode) + console.log(''); + console.log(chalk.bold('Agent Status:')); + console.log(''); + + // Show updatable first + for (const agent of updatableAgents) { + console.log(` ${chalk.cyan(agent.id)}:`); + console.log(` Status: ${formatStatus(agent.status)}`); + if (agent.description) { + console.log(` ${chalk.gray(agent.description)}`); + } + console.log(''); + } + + // Show up-to-date + for (const agent of upToDateAgents) { + console.log(` ${chalk.green(agent.id)}: ${formatStatus(agent.status)}`); + } + + // Show not installed (summarized) + if (notInstalledAgents.length > 0) { + console.log(''); + console.log( + chalk.gray( + ` ${notInstalledAgents.length} agents not installed: ${notInstalledAgents.map((a) => a.id).join(', ')}` + ) + ); + } + + // Show custom + if (customAgents.length > 0) { + console.log(''); + for (const agent of customAgents) { + console.log(` ${chalk.blue(agent.id)}: ${formatStatus(agent.status)}`); + } + } + + // Show errors + for (const agent of errorAgents) { + console.log(` ${chalk.red(agent.id)}: ${formatStatus(agent.status)}`); + if (agent.error) { + console.log(` ${chalk.red(agent.error)}`); + } + } + + console.log(''); + + // Summary + console.log(chalk.bold('Summary:')); + console.log(` Up to date: ${chalk.green(upToDateAgents.length.toString())}`); + console.log(` Changes available: ${chalk.yellow(updatableAgents.length.toString())}`); + console.log(` Not installed: ${chalk.gray(notInstalledAgents.length.toString())}`); + if (customAgents.length > 0) { + console.log(` Custom: ${chalk.blue(customAgents.length.toString())}`); + } + console.log(''); + + // If list mode, stop here + if (list) { + p.outro('Use `dexto sync-agents` to update agents'); + return; + } + + // No updates needed + if (updatableAgents.length === 0) { + p.outro(chalk.green('All installed agents are up to date!')); + return; + } + + // Force mode - update all without prompting + if (force) { + const updateSpinner = p.spinner(); + updateSpinner.start(`Updating ${updatableAgents.length} agents...`); + + let successCount = 0; + let failCount = 0; + + for (const agent of updatableAgents) { + const entry = bundledAgents[agent.id]; + if (entry) { + try { + await updateAgent(agent.id, entry); + successCount++; + } catch (error) { + failCount++; + logger.error( + `Failed to update ${agent.id}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + } + + updateSpinner.stop(`Updated ${successCount} agents`); + + if (failCount > 0) { + p.log.warn(`${failCount} agents failed to update`); + } + + p.outro(chalk.green('Sync complete!')); + return; + } + + // Interactive mode - prompt for each + for (const agent of updatableAgents) { + const shouldUpdate = await p.confirm({ + message: `Update ${chalk.cyan(agent.name)} (${agent.id})?`, + initialValue: true, + }); + + if (p.isCancel(shouldUpdate)) { + p.cancel('Sync cancelled'); + return; + } + + if (shouldUpdate) { + const entry = bundledAgents[agent.id]; + if (entry) { + try { + const updateSpinner = p.spinner(); + updateSpinner.start(`Updating ${agent.id}...`); + await updateAgent(agent.id, entry); + updateSpinner.stop(`Updated ${agent.id}`); + } catch (error) { + p.log.error( + `Failed to update ${agent.id}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + } else { + p.log.info(`Skipped ${agent.id}`); + } + } + + p.outro(chalk.green('Sync complete!')); + } catch (error) { + spinner.stop('Error'); + p.log.error( + `Failed to check agents: ${error instanceof Error ? error.message : String(error)}` + ); + throw error; + } +} diff --git a/dexto/packages/cli/src/cli/commands/uninstall.test.ts b/dexto/packages/cli/src/cli/commands/uninstall.test.ts new file mode 100644 index 00000000..aead2f6b --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/uninstall.test.ts @@ -0,0 +1,191 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; + +// Mock the agent-helpers module +vi.mock('../../utils/agent-helpers.js', () => ({ + uninstallAgent: vi.fn(), + listInstalledAgents: vi.fn(), +})); + +// Mock analytics +vi.mock('../../analytics/index.js', () => ({ + capture: vi.fn(), +})); + +// Import SUT after mocks +import { handleUninstallCommand } from './uninstall.js'; +import { uninstallAgent, listInstalledAgents } from '../../utils/agent-helpers.js'; + +describe('Uninstall Command', () => { + let consoleSpy: any; + let consoleErrorSpy: any; + + beforeEach(() => { + vi.clearAllMocks(); + + // Mock console + consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + consoleSpy.mockRestore(); + consoleErrorSpy.mockRestore(); + }); + + describe('Command validation', () => { + it('rejects when no agents specified and all flag is false', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['some-agent']); + + await expect(handleUninstallCommand([], {})).rejects.toThrow(/No agents specified/); + }); + + it('rejects when no agents are installed', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue([]); + + await expect(handleUninstallCommand(['any-agent'], {})).rejects.toThrow( + /No agents are currently installed/ + ); + expect(uninstallAgent).not.toHaveBeenCalled(); + }); + + it('rejects uninstalling agents that are not installed', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['real-agent']); + + await expect(handleUninstallCommand(['fake-agent'], {})).rejects.toThrow( + /not installed/ + ); + }); + }); + + describe('Single agent uninstall', () => { + it('successfully uninstalls existing agent', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['test-agent']); + vi.mocked(uninstallAgent).mockResolvedValue(undefined); + + await expect(handleUninstallCommand(['test-agent'], {})).resolves.not.toThrow(); + + expect(uninstallAgent).toHaveBeenCalledWith('test-agent'); + expect(consoleSpy).toHaveBeenCalled(); + }); + + it('uninstalls agent without force flag', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['default-agent']); + vi.mocked(uninstallAgent).mockResolvedValue(undefined); + + await handleUninstallCommand(['default-agent'], {}); + + expect(uninstallAgent).toHaveBeenCalledWith('default-agent'); + expect(uninstallAgent).toHaveBeenCalledTimes(1); + }); + }); + + describe('Bulk uninstall', () => { + it('uninstalls all agents when --all flag is used', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['agent1', 'agent2', 'agent3']); + vi.mocked(uninstallAgent).mockResolvedValue(undefined); + + await expect(handleUninstallCommand([], { all: true })).resolves.not.toThrow(); + + expect(uninstallAgent).toHaveBeenCalledTimes(3); + expect(uninstallAgent).toHaveBeenCalledWith('agent1'); + expect(uninstallAgent).toHaveBeenCalledWith('agent2'); + expect(uninstallAgent).toHaveBeenCalledWith('agent3'); + // Multiple agents show summary + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('📊 Uninstallation Summary') + ); + }); + + it('uninstalls multiple specified agents', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['agent1', 'agent2', 'agent3']); + vi.mocked(uninstallAgent).mockResolvedValue(undefined); + + await handleUninstallCommand(['agent1', 'agent2'], {}); + + expect(uninstallAgent).toHaveBeenCalledTimes(2); + expect(uninstallAgent).toHaveBeenCalledWith('agent1'); + expect(uninstallAgent).toHaveBeenCalledWith('agent2'); + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('📊 Uninstallation Summary') + ); + }); + }); + + describe('Error handling', () => { + it('continues with other agents when one fails', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['good-agent', 'bad-agent']); + vi.mocked(uninstallAgent).mockImplementation((agent: string) => { + if (agent === 'bad-agent') { + throw new Error('Cannot remove protected agent'); + } + return Promise.resolve(undefined); + }); + + await expect( + handleUninstallCommand(['good-agent', 'bad-agent'], {}) + ).resolves.not.toThrow(); + + expect(uninstallAgent).toHaveBeenCalledTimes(2); + // Should show summary for multiple agents + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('📊 Uninstallation Summary') + ); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('Failed to uninstall bad-agent') + ); + }); + + it('throws when single agent uninstall fails', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['bad-agent']); + vi.mocked(uninstallAgent).mockRejectedValue(new Error('Protection error')); + + // Single agent failure should propagate the error directly + await expect(handleUninstallCommand(['bad-agent'], {})).rejects.toThrow(); + }); + + it('shows error in summary when all agents fail in bulk operation', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['agent1', 'agent2']); + vi.mocked(uninstallAgent).mockRejectedValue(new Error('Protection error')); + + await expect(handleUninstallCommand(['agent1', 'agent2'], {})).rejects.toThrow( + /All uninstallations failed/ + ); + + expect(uninstallAgent).toHaveBeenCalledTimes(2); + }); + + it('shows partial success when some agents fail in bulk operation', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['agent1', 'agent2', 'agent3']); + vi.mocked(uninstallAgent).mockImplementation((agent: string) => { + if (agent === 'agent2') { + throw new Error('Failed to uninstall'); + } + return Promise.resolve(undefined); + }); + + await expect( + handleUninstallCommand(['agent1', 'agent2', 'agent3'], {}) + ).resolves.not.toThrow(); + + expect(uninstallAgent).toHaveBeenCalledTimes(3); + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('📊 Uninstallation Summary') + ); + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('✅ Successfully')); + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('❌ Failed')); + }); + }); + + describe('Force flag handling', () => { + it('accepts force flag in options', async () => { + vi.mocked(listInstalledAgents).mockResolvedValue(['test-agent']); + vi.mocked(uninstallAgent).mockResolvedValue(undefined); + + // Force flag is in the options but doesn't affect uninstallAgent call + await handleUninstallCommand(['test-agent'], { force: true }); + + // uninstallAgent only takes agentId, no force parameter + expect(uninstallAgent).toHaveBeenCalledWith('test-agent'); + }); + }); +}); diff --git a/dexto/packages/cli/src/cli/commands/uninstall.ts b/dexto/packages/cli/src/cli/commands/uninstall.ts new file mode 100644 index 00000000..ad8c56d0 --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/uninstall.ts @@ -0,0 +1,160 @@ +// packages/cli/src/cli/commands/uninstall.ts + +import { z } from 'zod'; +import { uninstallAgent, listInstalledAgents } from '../../utils/agent-helpers.js'; +import { capture } from '../../analytics/index.js'; + +// Zod schema for uninstall command validation +const UninstallCommandSchema = z + .object({ + agents: z.array(z.string().min(1, 'Agent name cannot be empty')), + all: z.boolean().default(false), + force: z.boolean().default(false), + }) + .strict(); + +export type UninstallCommandOptions = z.output<typeof UninstallCommandSchema>; + +/** + * Validate uninstall command arguments + */ +async function validateUninstallCommand( + agents: string[], + options: Partial<UninstallCommandOptions> +): Promise<UninstallCommandOptions> { + // Basic structure validation + const validated = UninstallCommandSchema.parse({ + ...options, + agents, + }); + + // Business logic validation + const installedAgents = await listInstalledAgents(); + + if (installedAgents.length === 0) { + throw new Error('No agents are currently installed.'); + } + + if (!validated.all && validated.agents.length === 0) { + throw new Error( + `No agents specified. Use agent names or --all flag. Installed agents: ${installedAgents.join(', ')}` + ); + } + + return validated; +} + +export async function handleUninstallCommand( + agents: string[], + options: Partial<UninstallCommandOptions> +): Promise<void> { + // Validate command with Zod + const validated = await validateUninstallCommand(agents, options); + const installedAgents = await listInstalledAgents(); + + if (installedAgents.length === 0) { + console.log('📋 No agents are currently installed.'); + return; + } + + // Determine which agents to uninstall + let agentsToUninstall: string[]; + if (validated.all) { + agentsToUninstall = installedAgents; + console.log(`📋 Uninstalling all ${agentsToUninstall.length} installed agents...`); + } else { + agentsToUninstall = validated.agents; + + // Validate all specified agents are actually installed + const notInstalled = agentsToUninstall.filter((agent) => !installedAgents.includes(agent)); + if (notInstalled.length > 0) { + throw new Error( + `Agents not installed: ${notInstalled.join(', ')}. ` + + `Installed agents: ${installedAgents.join(', ')}` + ); + } + } + + console.log(`🗑️ Uninstalling ${agentsToUninstall.length} agents...`); + + let successCount = 0; + let errorCount = 0; + const errors: string[] = []; + const uninstalled: string[] = []; + const failed: string[] = []; + + // Uninstall each agent + for (const agentName of agentsToUninstall) { + try { + console.log(`\n🗑️ Uninstalling ${agentName}...`); + await uninstallAgent(agentName); + successCount++; + console.log(`✅ ${agentName} uninstalled successfully`); + uninstalled.push(agentName); + // Per-agent analytics for successful uninstall + try { + capture('dexto_uninstall_agent', { + agent: agentName, + status: 'uninstalled', + force: validated.force, + }); + } catch { + // Analytics failures should not block CLI execution. + } + } catch (error) { + errorCount++; + const errorMsg = `Failed to uninstall ${agentName}: ${error instanceof Error ? error.message : String(error)}`; + errors.push(errorMsg); + failed.push(agentName); + console.error(`❌ ${errorMsg}`); + // Per-agent analytics for failed uninstall + try { + capture('dexto_uninstall_agent', { + agent: agentName, + status: 'failed', + error_message: error instanceof Error ? error.message : String(error), + force: validated.force, + }); + } catch { + // Analytics failures should not block CLI execution. + } + } + } + + // Emit analytics for both single- and multi-agent cases + try { + capture('dexto_uninstall', { + requested: agentsToUninstall, + uninstalled, + failed, + successCount, + errorCount, + }); + } catch { + // Analytics failures should not block CLI execution. + } + + // For single agent operations, throw error if it failed (after emitting analytics) + if (agentsToUninstall.length === 1) { + if (errorCount > 0) { + throw new Error(errors[0]); + } + return; + } + + // Show summary if more than 1 agent uninstalled + console.log(`\n📊 Uninstallation Summary:`); + console.log(`✅ Successfully uninstalled: ${successCount}`); + if (errorCount > 0) { + console.log(`❌ Failed to uninstall: ${errorCount}`); + errors.forEach((error) => console.log(` • ${error}`)); + } + + if (errorCount > 0 && successCount === 0) { + throw new Error('All uninstallations failed'); + } else if (errorCount > 0) { + console.log(`⚠️ Some uninstallations failed, but ${successCount} succeeded.`); + } else { + console.log(`🎉 All agents uninstalled successfully!`); + } +} diff --git a/dexto/packages/cli/src/cli/commands/which.ts b/dexto/packages/cli/src/cli/commands/which.ts new file mode 100644 index 00000000..0a2873ab --- /dev/null +++ b/dexto/packages/cli/src/cli/commands/which.ts @@ -0,0 +1,50 @@ +// packages/cli/src/cli/commands/which.ts + +import { readFileSync } from 'fs'; +import chalk from 'chalk'; +import { z } from 'zod'; +import { resolveAgentPath, resolveBundledScript } from '@dexto/agent-management'; + +// Zod schema for which command validation +const WhichCommandSchema = z + .object({ + agentName: z.string().min(1, 'Agent name cannot be empty'), + }) + .strict(); + +export type WhichCommandOptions = z.output<typeof WhichCommandSchema>; + +/** + * Load available agent names from bundled registry + */ +function getAvailableAgentNames(): string[] { + try { + const registryPath = resolveBundledScript('agents/agent-registry.json'); + const content = readFileSync(registryPath, 'utf-8'); + const registry = JSON.parse(content); + return Object.keys(registry.agents || {}); + } catch (_error) { + return []; + } +} + +/** + * Handle the which command + */ +export async function handleWhichCommand(agentName: string): Promise<void> { + // Validate command with Zod + const validated = WhichCommandSchema.parse({ agentName }); + const availableAgents = getAvailableAgentNames(); + + try { + const resolvedPath = await resolveAgentPath(validated.agentName, false); // Don't auto-install + console.log(resolvedPath); + } catch (error) { + console.error( + chalk.red( + `❌ dexto which command failed: ${error instanceof Error ? error.message : String(error)}. Available agents: ${availableAgents.join(', ')}` + ) + ); + process.exit(1); + } +} diff --git a/dexto/packages/cli/src/cli/ink-cli/AGENTS.md b/dexto/packages/cli/src/cli/ink-cli/AGENTS.md new file mode 100644 index 00000000..be9990c5 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/AGENTS.md @@ -0,0 +1,182 @@ +# Ink-CLI Architecture + +React-based terminal UI built with [Ink](https://github.com/vadimdemedes/ink). + +## Entry Point + +`InkCLIRefactored.tsx` → `startInkCliRefactored()` + +Two rendering modes (controlled by `USE_ALTERNATE_BUFFER` constant): +- **StaticCLI** (default): Uses Ink's `<Static>` for copy-friendly terminal scrollback +- **AlternateBufferCLI**: Fullscreen with VirtualizedList and mouse support + +## State Management + +State is managed via **multiple useState hooks** in `useCLIState.ts`. + +### Message State (separate arrays for render ordering) + +```typescript +messages: Message[] // Finalized → rendered in <Static> +pendingMessages: Message[] // Streaming → rendered dynamically +dequeuedBuffer: Message[] // User messages after pending (ordering fix) +queuedMessages: QueuedMessage[] // Waiting to be processed +``` + +### CLIState + +```typescript +interface CLIState { + input: InputState // value, history, images, pastedBlocks + ui: UIState // isProcessing, activeOverlay, exitWarning + session: SessionState // id, modelName + approval: ApprovalRequest | null + approvalQueue: ApprovalRequest[] +} +``` + +### Message Interface + +```typescript +interface Message { + id: string + role: 'user' | 'assistant' | 'system' | 'tool' + content: string + timestamp: Date + isStreaming?: boolean + toolResult?: string + toolStatus?: 'running' | 'finished' + isError?: boolean // Tool execution failed + styledType?: StyledMessageType // config, stats, help, session-list, etc. + styledData?: StyledData + isContinuation?: boolean // Split message continuation +} +``` + +## Component Hierarchy + +```text +InkCLIRefactored +├── KeypressProvider +├── MouseProvider (alternate buffer only) +├── ScrollProvider (alternate buffer only) +└── StaticCLI / AlternateBufferCLI + ├── Header + ├── Messages (Static or VirtualizedList) + ├── PendingMessages (dynamic, outside Static) + ├── StatusBar + ├── OverlayContainer + ├── InputContainer → TextBufferInput + └── Footer +``` + +## Input Architecture + +### Keyboard Flow + +1. `KeypressContext` captures raw stdin +2. `useInputOrchestrator` routes to handlers based on focus +3. Main text input uses its own `useKeypress()` in `TextBufferInput` + +### Focus Priority + +1. **Approval prompt** (if visible) +2. **Overlay** (if active) +3. **Global shortcuts** (Ctrl+C, Escape) +4. **Main input** (TextBufferInput) + +### Global Shortcuts + +- **Ctrl+C**: Cancel processing → clear input → exit warning → exit +- **Escape**: Clear exit warning → cancel processing → close overlay +- **Ctrl+S**: Toggle copy mode (alternate buffer only) + +## Overlay System + +### All Overlay Types + +```typescript +type OverlayType = + | 'none' + | 'slash-autocomplete' // /command + | 'resource-autocomplete' // @resource + | 'model-selector' // /model + | 'session-selector' // /session, /resume + | 'session-subcommand-selector' + | 'mcp-selector' + | 'mcp-add-selector' + | 'mcp-remove-selector' + | 'mcp-custom-type-selector' + | 'mcp-custom-wizard' + | 'log-level-selector' + | 'approval' +``` + +### Detection + +Overlays detected by input content (debounced 50ms): +- `/` prefix → slash-autocomplete (or specific selector) +- `@` anywhere → resource-autocomplete + +## Key Files + +| File | Purpose | +|------|---------| +| `InkCLIRefactored.tsx` | Entry point, provider setup | +| `hooks/useCLIState.ts` | All state management | +| `hooks/useInputOrchestrator.ts` | Keyboard routing | +| `hooks/useAgentEvents.ts` | Event subscriptions | +| `services/processStream.ts` | Streaming handler | +| `services/CommandService.ts` | Command execution | +| `containers/OverlayContainer.tsx` | Overlay management | +| `components/shared/text-buffer.ts` | Input text buffer | + +## Critical Rules + +### Do + +- Use `TextBuffer` as source of truth for input +- Pass explicit `sessionId` to all agent calls +- Check `key.escape` BEFORE checking item count in selectors +- Return `boolean` from handlers (`true` = consumed) +- Use refs + `useImperativeHandle` for component coordination + +### Don't + +```typescript +// Adding useInput directly to components +useInput((input, key) => { ... }); // Don't do this + +// Checking item count before escape +if (items.length === 0) return false; +if (key.escape) { onClose(); return true; } // Never reached! + +// Using agent.getCurrentSessionId() +const sessionId = agent.getCurrentSessionId(); // Stale! +// Use state.session.id instead +``` + +## Common Tasks + +### Add New Overlay + +1. Add to `OverlayType` in `state/types.ts` +2. Add detection in `useCLIState.ts` +3. Create component in `components/overlays/` +4. Register in `OverlayContainer.tsx` +5. **If overlay has its own text input** (wizard, search, etc.): Add to `overlaysWithOwnInput` array in `InputContainer.tsx` (~line 659) to disable main input while overlay is active + +### Add New Slash Command + +1. Add to `commands/interactive-commands/commands.ts` +2. Add handler in `CommandService.ts` if needed +3. Appears in autocomplete automatically + +## Streaming Architecture + +`processStream.ts` handles the async iterator from `agent.stream()`: + +- Streaming content → `pendingMessages` (redrawn each frame) +- Finalized content → `messages` (in `<Static>`, rendered once) +- Large content split at markdown boundaries to reduce flickering +- Uses `localPending` mirror to avoid React batching race conditions diff --git a/dexto/packages/cli/src/cli/ink-cli/CLAUDE.md b/dexto/packages/cli/src/cli/ink-cli/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/dexto/packages/cli/src/cli/ink-cli/GEMINI.md b/dexto/packages/cli/src/cli/ink-cli/GEMINI.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/GEMINI.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/dexto/packages/cli/src/cli/ink-cli/InkCLIRefactored.tsx b/dexto/packages/cli/src/cli/ink-cli/InkCLIRefactored.tsx new file mode 100644 index 00000000..fd6edb01 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/InkCLIRefactored.tsx @@ -0,0 +1,213 @@ +/** + * InkCLI Component (Refactored) + * + * Entry point for the Ink-based CLI. Selects between two rendering modes: + * - AlternateBufferCLI: VirtualizedList with mouse scroll, keyboard scroll, copy mode + * - StaticCLI: Static pattern with native terminal scrollback and selection + * + * The mode is selected via USE_ALTERNATE_BUFFER constant. + */ + +import React, { useCallback, useState } from 'react'; +import { render } from 'ink'; +import type { DextoAgent } from '@dexto/core'; +import { registerGracefulShutdown } from '../../utils/graceful-shutdown.js'; +import { enableBracketedPaste, disableBracketedPaste } from './utils/bracketedPaste.js'; + +// Types +import type { StartupInfo } from './state/types.js'; + +// Contexts (keyboard/mouse providers) +import { + KeypressProvider, + MouseProvider, + ScrollProvider, + SoundProvider, +} from './contexts/index.js'; + +// Sound notification +import type { SoundNotificationService } from './utils/soundNotification.js'; + +// Components +import { ErrorBoundary } from './components/ErrorBoundary.js'; +import { AlternateBufferCLI, StaticCLI } from './components/modes/index.js'; + +// Hooks +import { useStreaming } from './hooks/useStreaming.js'; + +// Utils +import { getStartupInfo } from './utils/messageFormatting.js'; + +// Rendering mode: true = alternate buffer with VirtualizedList, false = Static pattern +// Toggle this to switch between modes for testing +//const USE_ALTERNATE_BUFFER = true; +const USE_ALTERNATE_BUFFER = false; + +interface InkCLIProps { + agent: DextoAgent; + initialSessionId: string | null; + startupInfo: StartupInfo; + soundService: SoundNotificationService | null; +} + +/** + * Inner component that wraps the mode-specific component with providers + */ +function InkCLIInner({ agent, initialSessionId, startupInfo, soundService }: InkCLIProps) { + // Selection hint callback for alternate buffer mode + const [, setSelectionHintShown] = useState(false); + + // Streaming mode - can be toggled via /stream command + const { streaming } = useStreaming(); + + const handleSelectionAttempt = useCallback(() => { + setSelectionHintShown(true); + }, []); + + if (USE_ALTERNATE_BUFFER) { + return ( + <SoundProvider soundService={soundService}> + <ScrollProvider onSelectionAttempt={handleSelectionAttempt}> + <AlternateBufferCLI + agent={agent} + initialSessionId={initialSessionId} + startupInfo={startupInfo} + onSelectionAttempt={handleSelectionAttempt} + useStreaming={streaming} + /> + </ScrollProvider> + </SoundProvider> + ); + } + + // Static mode - no ScrollProvider needed + return ( + <SoundProvider soundService={soundService}> + <StaticCLI + agent={agent} + initialSessionId={initialSessionId} + startupInfo={startupInfo} + useStreaming={streaming} + /> + </SoundProvider> + ); +} + +/** + * Modern CLI interface using React Ink + * + * Wraps the CLI with: + * - ErrorBoundary for graceful error handling + * - KeypressProvider for unified keyboard input + * - MouseProvider (only in alternate buffer mode) + */ +export function InkCLIRefactored({ + agent, + initialSessionId, + startupInfo, + soundService, +}: InkCLIProps) { + return ( + <ErrorBoundary> + <KeypressProvider> + {/* Mouse events only in alternate buffer mode - Static mode uses native terminal selection */} + <MouseProvider mouseEventsEnabled={USE_ALTERNATE_BUFFER}> + <InkCLIInner + agent={agent} + initialSessionId={initialSessionId} + startupInfo={startupInfo} + soundService={soundService} + /> + </MouseProvider> + </KeypressProvider> + </ErrorBoundary> + ); +} + +/** + * Options for starting the Ink CLI + */ +export interface InkCLIOptions { + /** Update info if a newer version is available */ + updateInfo?: { current: string; latest: string; updateCommand: string } | undefined; + /** True if installed agents differ from bundled and user should sync */ + needsAgentSync?: boolean | undefined; +} + +/** + * Start the modern Ink-based CLI + */ +export async function startInkCliRefactored( + agent: DextoAgent, + initialSessionId: string | null, + options: InkCLIOptions = {} +): Promise<void> { + registerGracefulShutdown(() => agent, { inkMode: true }); + + // Enable bracketed paste mode so we can detect pasted text + // This wraps pastes with escape sequences that our KeypressContext handles + enableBracketedPaste(); + + const baseStartupInfo = await getStartupInfo(agent); + const startupInfo = { + ...baseStartupInfo, + updateInfo: options.updateInfo, + needsAgentSync: options.needsAgentSync, + }; + + // Initialize sound service from preferences + const { SoundNotificationService } = await import('./utils/soundNotification.js'); + const { globalPreferencesExist, loadGlobalPreferences } = await import( + '@dexto/agent-management' + ); + + let soundService: SoundNotificationService | null = null; + // Initialize sound config with defaults (enabled by default even without preferences file) + let soundConfig = { + enabled: true, + onApprovalRequired: true, + onTaskComplete: true, + }; + // Override with user preferences if they exist + if (globalPreferencesExist()) { + try { + const preferences = await loadGlobalPreferences(); + soundConfig = { + enabled: preferences.sounds?.enabled ?? soundConfig.enabled, + onApprovalRequired: + preferences.sounds?.onApprovalRequired ?? soundConfig.onApprovalRequired, + onTaskComplete: preferences.sounds?.onTaskComplete ?? soundConfig.onTaskComplete, + }; + } catch (error) { + // Log at debug level to help troubleshoot sound configuration issues + // Continue with default sounds - this is non-critical functionality + agent.logger.debug( + `Sound preferences could not be loaded: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + if (soundConfig.enabled) { + soundService = new SoundNotificationService(soundConfig); + } + + const inkApp = render( + <InkCLIRefactored + agent={agent} + initialSessionId={initialSessionId} + startupInfo={startupInfo} + soundService={soundService} + />, + { + exitOnCtrlC: false, + alternateBuffer: USE_ALTERNATE_BUFFER, + // Incremental rendering works better with VirtualizedList + // Static pattern doesn't need it (and may work better without) + incrementalRendering: USE_ALTERNATE_BUFFER, + } + ); + + await inkApp.waitUntilExit(); + + // Disable bracketed paste mode to restore normal terminal behavior + disableBracketedPaste(); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/ApprovalPrompt.tsx b/dexto/packages/cli/src/cli/ink-cli/components/ApprovalPrompt.tsx new file mode 100644 index 00000000..9c18f82c --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/ApprovalPrompt.tsx @@ -0,0 +1,462 @@ +import React, { + forwardRef, + useState, + useImperativeHandle, + useRef, + useEffect, + useMemo, +} from 'react'; +import { Box, Text } from 'ink'; +import type { ToolDisplayData, ElicitationMetadata } from '@dexto/core'; +import type { Key } from '../hooks/useInputOrchestrator.js'; +import { ElicitationForm, type ElicitationFormHandle } from './ElicitationForm.js'; +import { DiffPreview, CreateFilePreview } from './renderers/index.js'; +import { isEditWriteTool } from '../utils/toolUtils.js'; +import { formatToolHeader } from '../utils/messageFormatting.js'; + +export interface ApprovalRequest { + approvalId: string; + type: string; + sessionId?: string; + timeout?: number; + timestamp: Date; + metadata: Record<string, unknown>; +} + +export interface ApprovalPromptHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * Options passed when approving a request + */ +export interface ApprovalOptions { + /** Remember this tool for the entire session (approves ALL uses) */ + rememberChoice?: boolean; + /** Remember a specific command pattern for bash (e.g., "git *") */ + rememberPattern?: string; + /** Form data for elicitation requests */ + formData?: Record<string, unknown>; + /** Enable "accept all edits" mode (auto-approve future edit_file/write_file) */ + enableAcceptEditsMode?: boolean; + /** Remember directory access for the session */ + rememberDirectory?: boolean; +} + +interface ApprovalPromptProps { + approval: ApprovalRequest; + onApprove: (options: ApprovalOptions) => void; + onDeny: (feedback?: string) => void; + onCancel: () => void; +} + +/** + * Selection option type - supports both simple yes/no and pattern-based options + */ +type SelectionOption = + | 'yes' + | 'yes-session' + | 'yes-accept-edits' + | 'no' + | `pattern-${number}` + // Plan review specific options + | 'plan-approve' + | 'plan-approve-accept-edits' + | 'plan-reject'; // Single reject option with feedback input + +/** + * Compact approval prompt component that displays above the input area + * Shows options based on approval type: + * - Tool confirmation: Yes, Yes (Session), No + * - Bash with patterns: Yes (once), pattern options, Yes (all bash), No + * - Elicitation: Form with input fields + */ +export const ApprovalPrompt = forwardRef<ApprovalPromptHandle, ApprovalPromptProps>( + ({ approval, onApprove, onDeny, onCancel }, ref) => { + const isCommandConfirmation = approval.type === 'command_confirmation'; + const isElicitation = approval.type === 'elicitation'; + const isDirectoryAccess = approval.type === 'directory_access'; + + // Extract tool metadata + const toolName = approval.metadata.toolName as string | undefined; + const toolArgs = (approval.metadata.args as Record<string, unknown>) || {}; + + // Check if this is a plan_review tool (shows custom approval options) + const isPlanReview = + toolName === 'custom--plan_review' || + toolName === 'internal--plan_review' || + toolName === 'plan_review'; + + // Extract suggested patterns for bash tools + const suggestedPatterns = + (approval.metadata.suggestedPatterns as string[] | undefined) ?? []; + const hasBashPatterns = suggestedPatterns.length > 0; + + // Check if this is an edit/write file tool + const isEditOrWriteTool = isEditWriteTool(toolName); + + // Format tool header using shared utility (same format as tool messages) + const formattedTool = useMemo(() => { + if (!toolName) return null; + return formatToolHeader(toolName, toolArgs); + }, [toolName, toolArgs]); + + const [selectedIndex, setSelectedIndex] = useState(0); + + // State for plan review feedback input + const [showFeedbackInput, setShowFeedbackInput] = useState(false); + const [feedbackText, setFeedbackText] = useState(''); + + // Ref for elicitation form + const elicitationFormRef = useRef<ElicitationFormHandle>(null); + + // Use ref to avoid stale closure issues in handleInput + const selectedIndexRef = useRef(0); + + // Build the list of options based on approval type + const options: Array<{ id: SelectionOption; label: string }> = []; + + if (isPlanReview) { + // Plan review - show plan-specific options (2 options + feedback input) + options.push({ id: 'plan-approve', label: 'Approve' }); + options.push({ id: 'plan-approve-accept-edits', label: 'Approve + Accept All Edits' }); + // Third "option" is the feedback input (handled specially in render) + } else if (hasBashPatterns) { + // Bash tool with pattern suggestions + options.push({ id: 'yes', label: 'Yes (once)' }); + suggestedPatterns.forEach((pattern, i) => { + options.push({ + id: `pattern-${i}` as SelectionOption, + label: `Yes, allow "${pattern}"`, + }); + }); + options.push({ id: 'yes-session', label: 'Yes, allow all bash' }); + options.push({ id: 'no', label: 'No' }); + } else if (isCommandConfirmation) { + // Command confirmation (no session option) + options.push({ id: 'yes', label: 'Yes' }); + options.push({ id: 'no', label: 'No' }); + } else if (isDirectoryAccess) { + // Directory access - offer session-scoped access + const parentDir = approval.metadata.parentDir as string | undefined; + const dirLabel = parentDir ? ` "${parentDir}"` : ''; + options.push({ id: 'yes', label: 'Yes (once)' }); + options.push({ id: 'yes-session', label: `Yes, allow${dirLabel} (session)` }); + options.push({ id: 'no', label: 'No' }); + } else if (isEditOrWriteTool) { + // Edit/write file tools - offer "accept all edits" mode instead of session + options.push({ id: 'yes', label: 'Yes' }); + options.push({ id: 'yes-accept-edits', label: 'Yes, and accept all edits' }); + options.push({ id: 'no', label: 'No' }); + } else { + // Standard tool confirmation + options.push({ id: 'yes', label: 'Yes' }); + options.push({ id: 'yes-session', label: 'Yes (Session)' }); + options.push({ id: 'no', label: 'No' }); + } + + // Keep ref in sync with state + useEffect(() => { + selectedIndexRef.current = selectedIndex; + }, [selectedIndex]); + + // Helper to get the option at current index + const getCurrentOption = () => options[selectedIndexRef.current]; + + // Expose handleInput method via ref + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key) => { + // For elicitation, delegate to the form + if (isElicitation && elicitationFormRef.current) { + return elicitationFormRef.current.handleInput(input, key); + } + + // For plan review, calculate total options including feedback input + const totalOptions = isPlanReview ? options.length + 1 : options.length; + const isFeedbackSelected = + isPlanReview && selectedIndexRef.current === options.length; + + // Handle typing when feedback input is selected + if (isFeedbackSelected) { + if (key.return) { + // Submit rejection with feedback + onDeny(feedbackText || undefined); + return true; + } else if (key.backspace || key.delete) { + setFeedbackText((prev) => prev.slice(0, -1)); + return true; + } else if (key.upArrow) { + // Navigate up from feedback input + setSelectedIndex(options.length - 1); + return true; + } else if (key.downArrow) { + // Wrap to first option + setSelectedIndex(0); + return true; + } else if (key.escape) { + onCancel(); + return true; + } else if (input && !key.ctrl && !key.meta) { + // Add typed character to feedback + setFeedbackText((prev) => prev + input); + return true; + } + return true; // Consume all input when feedback is selected + } + + if (key.upArrow) { + setSelectedIndex((current) => + current === 0 ? totalOptions - 1 : current - 1 + ); + return true; + } else if (key.downArrow) { + setSelectedIndex((current) => + current === totalOptions - 1 ? 0 : current + 1 + ); + return true; + } else if (key.return) { + const option = getCurrentOption(); + if (!option) return false; + + // Plan review options + if (option.id === 'plan-approve') { + onApprove({}); + } else if (option.id === 'plan-approve-accept-edits') { + onApprove({ enableAcceptEditsMode: true }); + } else if (option.id === 'yes') { + onApprove({}); + } else if (option.id === 'yes-session') { + // For directory access, remember the directory; otherwise remember the tool + if (isDirectoryAccess) { + onApprove({ rememberDirectory: true }); + } else { + onApprove({ rememberChoice: true }); + } + } else if (option.id === 'yes-accept-edits') { + // Approve and enable "accept all edits" mode + onApprove({ enableAcceptEditsMode: true }); + } else if (option.id === 'no') { + onDeny(); + } else if (option.id.startsWith('pattern-')) { + // Extract pattern index and get the pattern string + const patternIndex = parseInt(option.id.replace('pattern-', ''), 10); + const pattern = suggestedPatterns[patternIndex]; + if (pattern) { + onApprove({ rememberPattern: pattern }); + } else { + onApprove({}); + } + } + return true; + } else if (key.shift && key.tab && isEditOrWriteTool) { + // Shift+Tab on edit/write tool: approve and enable "accept all edits" mode + onApprove({ enableAcceptEditsMode: true }); + return true; + } else if (key.escape) { + onCancel(); + return true; + } + return false; + }, + }), + [ + isElicitation, + isEditOrWriteTool, + isDirectoryAccess, + isPlanReview, + options, + suggestedPatterns, + onApprove, + onDeny, + onCancel, + feedbackText, + ] + ); + + // For elicitation, render the form + if (isElicitation) { + const metadata = approval.metadata as unknown as ElicitationMetadata; + return ( + <ElicitationForm + ref={elicitationFormRef} + metadata={metadata} + onSubmit={(formData) => onApprove({ formData })} + onCancel={onCancel} + /> + ); + } + + // Extract information from metadata based on approval type + const command = approval.metadata.command as string | undefined; + const displayPreview = approval.metadata.displayPreview as ToolDisplayData | undefined; + + // Render preview based on display type + const renderPreview = () => { + if (!displayPreview) return null; + + switch (displayPreview.type) { + case 'diff': { + const isOverwrite = + toolName === 'custom--write_file' || + toolName === 'internal--write_file' || + toolName === 'write_file'; + return ( + <DiffPreview + data={displayPreview} + headerType={isOverwrite ? 'overwrite' : 'edit'} + /> + ); + } + case 'shell': + // For shell preview, just show the command (no output yet) + return ( + <Box marginBottom={1} flexDirection="row"> + <Text color="gray">$ </Text> + <Text color="yellowBright">{displayPreview.command}</Text> + {displayPreview.isBackground && <Text color="gray"> (background)</Text>} + </Box> + ); + case 'file': + // Use enhanced file preview with full content for file creation + if (displayPreview.operation === 'create' && displayPreview.content) { + return <CreateFilePreview data={displayPreview} />; + } + // For plan_review (read operation with content), show full content for review + if ( + displayPreview.operation === 'read' && + displayPreview.content && + isPlanReview + ) { + return <CreateFilePreview data={displayPreview} header="Review plan" />; + } + // Fallback for other file operations + return ( + <Box marginBottom={1}> + <Text color="gray"> + {displayPreview.operation === 'read' && + `Read ${displayPreview.lineCount ?? 'file'} lines`} + {displayPreview.operation === 'write' && + `Write to ${displayPreview.path}`} + {displayPreview.operation === 'delete' && + `Delete ${displayPreview.path}`} + </Text> + </Box> + ); + default: + return null; + } + }; + + // Extract directory access metadata + const directoryPath = approval.metadata.path as string | undefined; + const parentDir = approval.metadata.parentDir as string | undefined; + const operation = approval.metadata.operation as string | undefined; + + return ( + <Box paddingX={0} paddingY={0} flexDirection="column"> + {/* Compact header with context */} + <Box flexDirection="column" marginBottom={0}> + {isDirectoryAccess ? ( + <> + <Box flexDirection="row"> + <Text color="yellowBright" bold> + 🔐 Directory Access:{' '} + </Text> + <Text color="cyan">{parentDir || directoryPath}</Text> + </Box> + <Box flexDirection="row" marginTop={0}> + <Text color="gray">{' '}</Text> + <Text color="gray"> + {formattedTool ? `"${formattedTool.displayName}"` : 'Tool'}{' '} + wants to {operation || 'access'} files outside working directory + </Text> + </Box> + </> + ) : ( + <> + <Box flexDirection="row"> + <Text color="yellowBright" bold> + 🔐 Approval:{' '} + </Text> + {formattedTool && <Text color="cyan">{formattedTool.header}</Text>} + </Box> + {isCommandConfirmation && command && ( + <Box flexDirection="row" marginTop={0}> + <Text color="gray">{' Command: '}</Text> + <Text color="red">{command}</Text> + </Box> + )} + </> + )} + </Box> + + {/* Preview section - shown BEFORE approval options */} + {renderPreview()} + + {/* Vertical selection options */} + <Box flexDirection="column" marginTop={0}> + {options.map((option, index) => { + const isSelected = index === selectedIndex; + const isNo = option.id === 'no'; + + return ( + <Box key={option.id}> + {isSelected ? ( + <Text color={isNo ? 'red' : 'green'} bold> + {' ▶ '} + {option.label} + </Text> + ) : ( + <Text color="gray"> + {' '} + {option.label} + </Text> + )} + </Box> + ); + })} + + {/* Feedback input as third option for plan review */} + {isPlanReview && ( + <Box> + {selectedIndex === options.length ? ( + // Selected - show editable input + <Box flexDirection="row"> + <Text color="red" bold> + {' ▶ '} + </Text> + {feedbackText ? ( + <Text color="white"> + {feedbackText} + <Text color="cyan">▋</Text> + </Text> + ) : ( + <Text color="gray"> + What changes would you like? + <Text color="cyan">▋</Text> + </Text> + )} + </Box> + ) : ( + // Not selected - show placeholder + <Text color="gray"> + {' '} + {feedbackText || 'What changes would you like?'} + </Text> + )} + </Box> + )} + </Box> + + {/* Compact instructions */} + <Box marginTop={0}> + <Text color="gray">{' '}↑↓ to select • Enter to confirm • Esc to cancel</Text> + </Box> + </Box> + ); + } +); + +ApprovalPrompt.displayName = 'ApprovalPrompt'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/CustomInput.tsx b/dexto/packages/cli/src/cli/ink-cli/components/CustomInput.tsx new file mode 100644 index 00000000..f12ed85d --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/CustomInput.tsx @@ -0,0 +1,95 @@ +import { useInput, Text } from 'ink'; + +interface CustomInputProps { + value: string; + onChange: (value: string) => void; + onSubmit: (value: string) => void; + placeholder?: string; + isProcessing?: boolean; + onWordDelete?: () => void; + onLineDelete?: () => void; + onToggleMultiLine?: () => void; +} + +/** + * Custom input component that handles keyboard shortcuts + * Fully custom implementation without TextInput to properly handle shortcuts + */ +export default function CustomInput({ + value, + onChange, + onSubmit, + placeholder, + isProcessing = false, + onWordDelete, + onLineDelete, + onToggleMultiLine, +}: CustomInputProps) { + // Handle all keyboard input directly + useInput( + (inputChar, key) => { + if (isProcessing) return; + + // Shift+Enter = toggle multi-line mode + if (key.return && key.shift) { + onToggleMultiLine?.(); + return; + } + + // Enter = submit + if (key.return) { + onSubmit(value); + return; + } + + // Ctrl+U = line delete (Unix standard, also what Cmd+Backspace becomes) + if (key.ctrl && inputChar === 'u') { + onLineDelete?.(); + return; + } + + // Ctrl+W = word delete (Unix standard, also what Option+Backspace becomes) + if (key.ctrl && inputChar === 'w') { + onWordDelete?.(); + return; + } + + // Regular backspace/delete + if (key.backspace || key.delete) { + onChange(value.slice(0, -1)); + return; + } + + // Regular character input + if (inputChar && !key.ctrl && !key.meta) { + onChange(value + inputChar); + } + }, + { isActive: true } + ); + + // Render with block cursor highlighting the character at cursor position + if (!value && placeholder) { + // Empty input - highlight first character of placeholder + const firstChar = placeholder[0] || ' '; + const rest = placeholder.slice(1); + return ( + <Text> + <Text color="black" backgroundColor="green"> + {firstChar} + </Text> + <Text color="gray">{rest}</Text> + </Text> + ); + } + + // Has value - highlight character after end (space) + return ( + <Text> + {value} + <Text color="black" backgroundColor="green"> + {' '} + </Text> + </Text> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/CustomTextInput.tsx b/dexto/packages/cli/src/cli/ink-cli/components/CustomTextInput.tsx new file mode 100644 index 00000000..2042f99c --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/CustomTextInput.tsx @@ -0,0 +1,69 @@ +import React from 'react'; +import { useInput } from 'ink'; +import TextInput from 'ink-text-input'; + +interface CustomTextInputProps { + value: string; + onChange: (value: string) => void; + onSubmit: (value: string) => void; + placeholder?: string; + onWordDelete?: () => void; + onLineDelete?: () => void; + onNewline?: () => void; +} + +/** + * Custom TextInput wrapper that handles keyboard shortcuts + * before TextInput consumes them + */ +export default function CustomTextInput({ + value, + onChange, + onSubmit, + placeholder, + onWordDelete, + onLineDelete, + onNewline, +}: CustomTextInputProps) { + // Use useInput to intercept keyboard shortcuts + // This needs to run with isActive: true to intercept before TextInput + useInput( + (inputChar, key) => { + // Handle Shift+Enter or Ctrl+E to toggle multi-line mode + // Note: Shift+Enter may not work in all terminals, Ctrl+E is more reliable + if ((key.return && key.shift) || (key.ctrl && inputChar === 'e')) { + onNewline?.(); + return; + } + + // Handle word deletion (Cmd+Delete or Cmd+Backspace on Mac, Ctrl+Delete or Ctrl+Backspace on Windows/Linux) + // Note: On Mac, Cmd+Backspace is the standard for word deletion + if ((key.delete || key.backspace) && (key.meta || key.ctrl)) { + onWordDelete?.(); + return; + } + + // Handle line deletion (Cmd+Shift+Delete or Ctrl+U) + if ((key.delete && key.meta && key.shift) || (key.ctrl && inputChar === 'u')) { + onLineDelete?.(); + return; + } + + // Handle Ctrl+Shift+Delete as additional word deletion shortcut (Windows/Linux) + if (key.delete && key.ctrl && key.shift) { + onWordDelete?.(); + return; + } + }, + { isActive: true } + ); + + return ( + <TextInput + value={value} + onChange={onChange} + onSubmit={onSubmit} + {...(placeholder ? { placeholder } : {})} + /> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/EditableMultiLineInput.tsx b/dexto/packages/cli/src/cli/ink-cli/components/EditableMultiLineInput.tsx new file mode 100644 index 00000000..3fde6297 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/EditableMultiLineInput.tsx @@ -0,0 +1,193 @@ +/** + * Editable multi-line input component + * Simple, reliable multi-line editor without complex box layouts + */ + +import React, { useState, useEffect, useMemo } from 'react'; +import { Box, Text, useInput } from 'ink'; + +interface EditableMultiLineInputProps { + value: string; + onChange: (value: string) => void; + onSubmit: (value: string) => void; + placeholder?: string; + isProcessing?: boolean; + onToggleSingleLine?: () => void; +} + +/** + * Multi-line input with cursor navigation + * Uses simple text rendering without nested boxes for reliable terminal display + */ +export default function EditableMultiLineInput({ + value, + onChange, + onSubmit, + placeholder, + isProcessing = false, + onToggleSingleLine, +}: EditableMultiLineInputProps) { + const [cursorPos, setCursorPos] = useState(value.length); + + // Keep cursor valid when value changes externally + useEffect(() => { + if (cursorPos > value.length) { + setCursorPos(value.length); + } + }, [value, cursorPos]); + + // Calculate line info from cursor position + const { lines, currentLine, currentCol, lineStartIndices } = useMemo(() => { + const lines = value.split('\n'); + const lineStartIndices: number[] = []; + let pos = 0; + for (const line of lines) { + lineStartIndices.push(pos); + pos += line.length + 1; + } + + let currentLine = 0; + for (let i = 0; i < lineStartIndices.length; i++) { + const lineEnd = + i < lineStartIndices.length - 1 ? lineStartIndices[i + 1]! - 1 : value.length; + if (cursorPos <= lineEnd || i === lineStartIndices.length - 1) { + currentLine = i; + break; + } + } + + const currentCol = cursorPos - (lineStartIndices[currentLine] ?? 0); + return { lines, currentLine, currentCol, lineStartIndices }; + }, [value, cursorPos]); + + useInput( + (inputChar, key) => { + if (isProcessing) return; + + // Cmd/Ctrl+Enter = submit + if (key.return && (key.meta || key.ctrl)) { + onSubmit(value); + return; + } + + // Shift+Enter = toggle back to single-line + // Note: Ctrl+E is reserved for standard "move to end of line" behavior + if (key.return && key.shift) { + onToggleSingleLine?.(); + return; + } + + // Enter = newline + if (key.return) { + const newValue = value.slice(0, cursorPos) + '\n' + value.slice(cursorPos); + onChange(newValue); + setCursorPos(cursorPos + 1); + return; + } + + // Backspace + if (key.backspace && cursorPos > 0) { + onChange(value.slice(0, cursorPos - 1) + value.slice(cursorPos)); + setCursorPos(cursorPos - 1); + return; + } + + // Delete + if (key.delete && cursorPos < value.length) { + onChange(value.slice(0, cursorPos) + value.slice(cursorPos + 1)); + return; + } + + // Arrow navigation + if (key.leftArrow) { + setCursorPos(Math.max(0, cursorPos - 1)); + return; + } + if (key.rightArrow) { + setCursorPos(Math.min(value.length, cursorPos + 1)); + return; + } + if (key.upArrow && currentLine > 0) { + const prevLineStart = lineStartIndices[currentLine - 1]!; + const prevLineLen = lines[currentLine - 1]!.length; + setCursorPos(prevLineStart + Math.min(currentCol, prevLineLen)); + return; + } + if (key.downArrow && currentLine < lines.length - 1) { + const nextLineStart = lineStartIndices[currentLine + 1]!; + const nextLineLen = lines[currentLine + 1]!.length; + setCursorPos(nextLineStart + Math.min(currentCol, nextLineLen)); + return; + } + + // Character input + if (inputChar && !key.ctrl && !key.meta) { + onChange(value.slice(0, cursorPos) + inputChar + value.slice(cursorPos)); + setCursorPos(cursorPos + inputChar.length); + } + }, + { isActive: true } + ); + + // Render each line with cursor + const renderLine = (line: string, lineIdx: number) => { + const lineStart = lineStartIndices[lineIdx]!; + const isCursorLine = lineIdx === currentLine; + const cursorCol = isCursorLine ? cursorPos - lineStart : -1; + + const prefix = lineIdx === 0 ? '» ' : ' '; + + if (cursorCol < 0) { + // No cursor on this line + return ( + <Text key={lineIdx}> + <Text color="cyan">{prefix}</Text> + <Text>{line || ' '}</Text> + </Text> + ); + } + + // Cursor on this line - highlight character at cursor + const before = line.slice(0, cursorCol); + const atCursor = line.charAt(cursorCol) || ' '; + const after = line.slice(cursorCol + 1); + + return ( + <Text key={lineIdx}> + <Text color="cyan">{prefix}</Text> + <Text>{before}</Text> + <Text backgroundColor="green" color="black"> + {atCursor} + </Text> + <Text>{after}</Text> + </Text> + ); + }; + + // Show placeholder if empty + if (!value && placeholder) { + return ( + <Box flexDirection="column"> + <Text> + <Text color="cyan">{'» '}</Text> + <Text backgroundColor="green" color="black"> + {' '} + </Text> + <Text color="gray"> {placeholder}</Text> + </Text> + <Text color="gray"> + Multi-line mode • Cmd/Ctrl+Enter to submit • Shift+Enter for single-line + </Text> + </Box> + ); + } + + return ( + <Box flexDirection="column"> + {lines.map((line, idx) => renderLine(line, idx))} + <Text color="gray"> + Multi-line mode • Cmd/Ctrl+Enter to submit • Shift+Enter for single-line + </Text> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/ElicitationForm.tsx b/dexto/packages/cli/src/cli/ink-cli/components/ElicitationForm.tsx new file mode 100644 index 00000000..68c02411 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/ElicitationForm.tsx @@ -0,0 +1,557 @@ +/** + * ElicitationForm Component + * Renders a form for ask_user/elicitation requests in the CLI + * Supports string, number, boolean, and enum field types + */ + +import React, { useState, forwardRef, useImperativeHandle, useCallback, useMemo } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../hooks/useInputOrchestrator.js'; +import type { ElicitationMetadata } from '@dexto/core'; + +export interface ElicitationFormHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ElicitationFormProps { + metadata: ElicitationMetadata; + onSubmit: (formData: Record<string, unknown>) => void; + onCancel: () => void; +} + +interface FormField { + name: string; + label: string; // title if available, otherwise name + type: 'string' | 'number' | 'boolean' | 'enum' | 'array-enum'; + description: string | undefined; + required: boolean; + enumValues: unknown[] | undefined; +} + +/** + * Form component for elicitation/ask_user requests + */ +export const ElicitationForm = forwardRef<ElicitationFormHandle, ElicitationFormProps>( + ({ metadata, onSubmit, onCancel }, ref) => { + // Parse schema into form fields + const fields = useMemo((): FormField[] => { + const schema = metadata.schema; + if (!schema?.properties) return []; + + const required = schema.required || []; + return Object.entries(schema.properties) + .filter( + (entry): entry is [string, Exclude<(typeof entry)[1], boolean>] => + typeof entry[1] !== 'boolean' + ) + .map(([name, prop]) => { + let type: FormField['type'] = 'string'; + let enumValues: unknown[] | undefined; + + if (prop.type === 'boolean') { + type = 'boolean'; + } else if (prop.type === 'number' || prop.type === 'integer') { + type = 'number'; + } else if (prop.enum && Array.isArray(prop.enum)) { + type = 'enum'; + enumValues = prop.enum; + } else if ( + prop.type === 'array' && + typeof prop.items === 'object' && + prop.items && + 'enum' in prop.items + ) { + type = 'array-enum'; + enumValues = prop.items.enum as unknown[]; + } + + return { + name, + label: prop.title || name, + type, + description: prop.description, + required: required.includes(name), + enumValues, + }; + }); + }, [metadata.schema]); + + // Form state + const [activeFieldIndex, setActiveFieldIndex] = useState(0); + const [formData, setFormData] = useState<Record<string, unknown>>({}); + const [currentInput, setCurrentInput] = useState(''); + const [enumIndex, setEnumIndex] = useState(0); // For enum selection + const [arraySelections, setArraySelections] = useState<Set<number>>(new Set()); // For array-enum + const [errors, setErrors] = useState<Record<string, string>>({}); + const [isReviewing, setIsReviewing] = useState(false); // Confirmation step before submit + + const activeField = fields[activeFieldIndex]; + + // Update a field value + const updateField = useCallback((name: string, value: unknown) => { + setFormData((prev) => ({ ...prev, [name]: value })); + setErrors((prev) => { + const newErrors = { ...prev }; + delete newErrors[name]; + return newErrors; + }); + }, []); + + // Validate and enter review mode (or submit if already reviewing) + // Accepts optional currentFieldValue to handle async state update timing + const handleSubmit = useCallback( + (currentFieldValue?: { name: string; value: unknown }) => { + const newErrors: Record<string, string> = {}; + // Merge current field value since React state update is async + const finalFormData = currentFieldValue + ? { ...formData, [currentFieldValue.name]: currentFieldValue.value } + : formData; + + for (const field of fields) { + if (field.required) { + const value = finalFormData[field.name]; + if (value === undefined || value === null || value === '') { + newErrors[field.name] = 'Required'; + } + } + } + + if (Object.keys(newErrors).length > 0) { + setErrors(newErrors); + // Focus first error field + const firstErrorField = fields.findIndex((f) => newErrors[f.name]); + if (firstErrorField >= 0) { + setActiveFieldIndex(firstErrorField); + } + return; + } + + // Update formData with final value and enter review mode + if (currentFieldValue) { + setFormData(finalFormData); + } + setIsReviewing(true); + }, + [fields, formData] + ); + + // Final submission after review + const confirmSubmit = useCallback(() => { + onSubmit(formData); + }, [formData, onSubmit]); + + // Navigate to next/previous field + const nextField = useCallback(() => { + if (activeFieldIndex < fields.length - 1) { + // Save current input for string/number fields + if (activeField?.type === 'string' || activeField?.type === 'number') { + if (currentInput.trim()) { + const value = + activeField.type === 'number' ? Number(currentInput) : currentInput; + updateField(activeField.name, value); + } + } + setActiveFieldIndex((prev) => prev + 1); + setCurrentInput(''); + setEnumIndex(0); + setArraySelections(new Set()); + } + }, [activeFieldIndex, fields.length, activeField, currentInput, updateField]); + + const prevField = useCallback(() => { + if (activeFieldIndex > 0) { + setActiveFieldIndex((prev) => prev - 1); + setCurrentInput(''); + setEnumIndex(0); + setArraySelections(new Set()); + } + }, [activeFieldIndex]); + + // Handle keyboard input + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + // Review mode handling + if (isReviewing) { + if (key.return) { + confirmSubmit(); + return true; + } + // Backspace to go back to editing + if (key.backspace || key.delete) { + setIsReviewing(false); + return true; + } + // Esc to cancel entirely + if (key.escape) { + onCancel(); + return true; + } + return false; + } + + // Escape to cancel + if (key.escape) { + onCancel(); + return true; + } + + if (!activeField) return false; + + // Shift+Tab or Up to previous field (check BEFORE plain Tab) + if ( + (key.tab && key.shift) || + (key.upArrow && + activeField.type !== 'enum' && + activeField.type !== 'array-enum') + ) { + prevField(); + return true; + } + + // Tab (without Shift) or Down to next field + if ( + (key.tab && !key.shift) || + (key.downArrow && + activeField.type !== 'enum' && + activeField.type !== 'array-enum') + ) { + nextField(); + return true; + } + + // Field-specific handling + switch (activeField.type) { + case 'boolean': { + // Space or Enter to toggle + if (input === ' ' || key.return) { + const current = formData[activeField.name] === true; + const newValue = !current; + updateField(activeField.name, newValue); + if (key.return) { + if (activeFieldIndex === fields.length - 1) { + handleSubmit({ name: activeField.name, value: newValue }); + } else { + nextField(); + } + } + return true; + } + // Left/Right to toggle + if (key.leftArrow || key.rightArrow) { + const current = formData[activeField.name] === true; + updateField(activeField.name, !current); + return true; + } + break; + } + + case 'enum': { + const values = activeField.enumValues || []; + // Up/Down to navigate enum + if (key.upArrow) { + setEnumIndex((prev) => (prev > 0 ? prev - 1 : values.length - 1)); + return true; + } + if (key.downArrow) { + setEnumIndex((prev) => (prev < values.length - 1 ? prev + 1 : 0)); + return true; + } + // Enter to select and move to next (or submit if last) + if (key.return) { + const selectedValue = values[enumIndex]; + updateField(activeField.name, selectedValue); + if (activeFieldIndex === fields.length - 1) { + handleSubmit({ name: activeField.name, value: selectedValue }); + } else { + nextField(); + } + return true; + } + break; + } + + case 'array-enum': { + const values = activeField.enumValues || []; + // Up/Down to navigate + if (key.upArrow) { + setEnumIndex((prev) => (prev > 0 ? prev - 1 : values.length - 1)); + return true; + } + if (key.downArrow) { + setEnumIndex((prev) => (prev < values.length - 1 ? prev + 1 : 0)); + return true; + } + // Space to toggle selection + if (input === ' ') { + setArraySelections((prev) => { + const next = new Set(prev); + if (next.has(enumIndex)) { + next.delete(enumIndex); + } else { + next.add(enumIndex); + } + // Update form data + const selected = Array.from(next).map((i) => values[i]); + updateField(activeField.name, selected); + return next; + }); + return true; + } + // Enter to confirm and move to next (or submit if last) + if (key.return) { + // Get current selections for submit + const selected = Array.from(arraySelections).map((i) => values[i]); + if (activeFieldIndex === fields.length - 1) { + handleSubmit({ name: activeField.name, value: selected }); + } else { + nextField(); + } + return true; + } + break; + } + + case 'string': + case 'number': { + // Enter to confirm field and move to next (or submit if last) + if (key.return) { + const value = currentInput.trim() + ? activeField.type === 'number' + ? Number(currentInput) + : currentInput + : formData[activeField.name]; // Use existing value if no new input + if (currentInput.trim()) { + updateField(activeField.name, value); + } + if (activeFieldIndex === fields.length - 1) { + // Last field - submit with current value + handleSubmit( + value !== undefined + ? { name: activeField.name, value } + : undefined + ); + } else { + nextField(); + } + return true; + } + // Backspace + if (key.backspace || key.delete) { + setCurrentInput((prev) => prev.slice(0, -1)); + return true; + } + // Regular character input + if (input && !key.ctrl && !key.meta) { + // For number type, only allow digits and decimal + if (activeField.type === 'number') { + if (/^[\d.-]$/.test(input)) { + setCurrentInput((prev) => prev + input); + } + } else { + setCurrentInput((prev) => prev + input); + } + return true; + } + break; + } + } + + return false; + }, + }), + [ + activeField, + activeFieldIndex, + arraySelections, + confirmSubmit, + currentInput, + enumIndex, + fields.length, + formData, + handleSubmit, + isReviewing, + nextField, + onCancel, + prevField, + updateField, + ] + ); + + if (fields.length === 0) { + return ( + <Box flexDirection="column" paddingX={1}> + <Text color="red">Invalid form schema</Text> + </Box> + ); + } + + const prompt = metadata.prompt; + + // Review mode - show summary of choices + if (isReviewing) { + return ( + <Box flexDirection="column" paddingX={0} paddingY={0}> + <Box marginBottom={1}> + <Text color="green" bold> + ✓ Review your answers: + </Text> + </Box> + + {fields.map((field) => { + const value = formData[field.name]; + const displayValue = Array.isArray(value) + ? value.join(', ') + : value === true + ? 'Yes' + : value === false + ? 'No' + : String(value ?? ''); + return ( + <Box key={field.name} marginBottom={0}> + <Text> + <Text color="cyan">{field.label}</Text> + <Text>: </Text> + <Text color="green">{displayValue}</Text> + </Text> + </Box> + ); + })} + + <Box marginTop={1}> + <Text color="gray"> + Enter to submit • Backspace to edit • Esc to cancel + </Text> + </Box> + </Box> + ); + } + + return ( + <Box flexDirection="column" paddingX={0} paddingY={0}> + {/* Header */} + <Box marginBottom={1}> + <Text color="yellowBright" bold> + 📝 {prompt} + </Text> + </Box> + + {/* Form fields */} + {fields.map((field, index) => { + const isActive = index === activeFieldIndex; + const value = formData[field.name]; + const error = errors[field.name]; + + return ( + <Box key={field.name} flexDirection="column" marginBottom={1}> + {/* Field label */} + <Box> + <Text color={isActive ? 'cyan' : 'white'} bold={isActive}> + {isActive ? '▶ ' : ' '} + {field.label} + {field.required && <Text color="red">*</Text>} + {': '} + </Text> + + {/* Field value display */} + {field.type === 'boolean' && ( + <Text color={value === true ? 'green' : 'gray'}> + {value === true ? '[✓] Yes' : '[ ] No'} + {isActive && <Text color="gray"> (Space to toggle)</Text>} + </Text> + )} + + {field.type === 'string' && !isActive && value !== undefined && ( + <Text color="green">{String(value)}</Text> + )} + + {field.type === 'number' && !isActive && value !== undefined && ( + <Text color="green">{String(value)}</Text> + )} + + {field.type === 'enum' && !isActive && value !== undefined && ( + <Text color="green">{String(value)}</Text> + )} + + {field.type === 'array-enum' && + !isActive && + Array.isArray(value) && + value.length > 0 && ( + <Text color="green">{value.join(', ')}</Text> + )} + </Box> + + {/* Active field input */} + {isActive && (field.type === 'string' || field.type === 'number') && ( + <Box marginLeft={2}> + <Text color="cyan">> </Text> + <Text>{currentInput}</Text> + <Text color="cyan">_</Text> + </Box> + )} + + {/* Enum selection */} + {isActive && field.type === 'enum' && field.enumValues && ( + <Box flexDirection="column" marginLeft={2}> + {field.enumValues.map((opt, i) => ( + <Box key={String(opt)}> + <Text color={i === enumIndex ? 'green' : 'gray'}> + {i === enumIndex ? ' ▶ ' : ' '} + {String(opt)} + </Text> + </Box> + ))} + </Box> + )} + + {/* Array-enum multi-select */} + {isActive && field.type === 'array-enum' && field.enumValues && ( + <Box flexDirection="column" marginLeft={2}> + <Text color="gray"> (Space to select, Enter to confirm)</Text> + {field.enumValues.map((opt, i) => { + const isSelected = arraySelections.has(i); + return ( + <Box key={String(opt)}> + <Text color={i === enumIndex ? 'cyan' : 'gray'}> + {i === enumIndex ? ' ▶ ' : ' '} + <Text color={isSelected ? 'green' : 'gray'}> + {isSelected ? '[✓]' : '[ ]'} + </Text>{' '} + {String(opt)} + </Text> + </Box> + ); + })} + </Box> + )} + + {/* Field description */} + {isActive && field.description && ( + <Box marginLeft={2}> + <Text color="gray">{field.description}</Text> + </Box> + )} + + {/* Error message */} + {error && ( + <Box marginLeft={2}> + <Text color="red">{error}</Text> + </Box> + )} + </Box> + ); + })} + + {/* Help text */} + <Box marginTop={1}> + <Text color="gray"> + Tab/↓ next field • Shift+Tab/↑ prev • Enter to confirm • Esc to cancel + </Text> + </Box> + </Box> + ); + } +); + +ElicitationForm.displayName = 'ElicitationForm'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/ErrorBoundary.tsx b/dexto/packages/cli/src/cli/ink-cli/components/ErrorBoundary.tsx new file mode 100644 index 00000000..5a8a60f5 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/ErrorBoundary.tsx @@ -0,0 +1,51 @@ +/** + * Error Boundary Component + * Catches and displays errors in the component tree + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import { logger } from '@dexto/core'; + +interface ErrorBoundaryProps { + children: React.ReactNode; +} + +interface ErrorBoundaryState { + hasError: boolean; + error: Error | null; +} + +export class ErrorBoundary extends React.Component<ErrorBoundaryProps, ErrorBoundaryState> { + constructor(props: ErrorBoundaryProps) { + super(props); + this.state = { hasError: false, error: null }; + } + + static getDerivedStateFromError(error: Error): ErrorBoundaryState { + return { hasError: true, error }; + } + + override componentDidCatch(error: Error, errorInfo: React.ErrorInfo): void { + logger.error(`Error in ErrorBoundary: ${error.message}`, { + error, + componentStack: errorInfo.componentStack, + }); + } + + override render(): React.ReactNode { + if (this.state.hasError) { + return ( + <Box flexDirection="column" padding={1} borderStyle="round" borderColor="red"> + <Text color="red" bold> + ❌ CLI Error + </Text> + <Text color="red">{this.state.error?.message || 'Unknown error'}</Text> + <Text color="yellowBright">Press Ctrl+C to exit</Text> + </Box> + ); + } + + return this.props.children; + } +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/Footer.tsx b/dexto/packages/cli/src/cli/ink-cli/components/Footer.tsx new file mode 100644 index 00000000..9623a726 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/Footer.tsx @@ -0,0 +1,152 @@ +/** + * Footer Component + * Status line at the bottom showing CWD, branch, and model info. + */ + +import React, { useEffect, useState } from 'react'; +import path from 'node:path'; +import { Box, Text } from 'ink'; +import { getModelDisplayName, type DextoAgent } from '@dexto/core'; + +interface FooterProps { + agent: DextoAgent; + sessionId: string | null; + modelName: string; + cwd?: string; + branchName?: string; + autoApproveEdits?: boolean; + planModeActive?: boolean; + /** Whether user is in shell command mode (input starts with !) */ + isShellMode?: boolean; +} + +function getDirectoryName(cwd: string): string { + const base = path.basename(cwd); + return base || cwd; +} + +/** + * Pure presentational component for footer status line + */ +export function Footer({ + agent, + sessionId, + modelName, + cwd, + branchName, + autoApproveEdits, + planModeActive, + isShellMode, +}: FooterProps) { + const displayPath = cwd ? getDirectoryName(cwd) : ''; + const displayModelName = getModelDisplayName(modelName); + const [contextLeft, setContextLeft] = useState<{ + percentLeft: number; + } | null>(null); + + // Check if Dexto is actually the active provider (not just if API key exists) + const viaDexto = agent.getCurrentLLMConfig().provider === 'dexto'; + + useEffect(() => { + if (!sessionId) { + setContextLeft(null); + return; + } + + let cancelled = false; + let refreshId = 0; + + const refreshContext = async () => { + const requestId = ++refreshId; + try { + const stats = await agent.getContextStats(sessionId); + if (cancelled || requestId !== refreshId) return; + const percentLeft = Math.max(0, Math.min(100, 100 - stats.usagePercent)); + setContextLeft({ + percentLeft, + }); + } catch { + if (!cancelled) { + setContextLeft(null); + } + } + }; + + refreshContext(); + + const bus = agent.agentEventBus; + const controller = new AbortController(); + const { signal } = controller; + const sessionEvents = [ + 'llm:response', + 'context:compacted', + 'context:pruned', + 'context:cleared', + 'message:dequeued', + 'session:reset', + ] as const; + + const handleEvent = (payload: { sessionId?: string }) => { + if (payload.sessionId && payload.sessionId !== sessionId) return; + refreshContext(); + }; + + for (const eventName of sessionEvents) { + bus.on(eventName, handleEvent, { signal }); + } + + return () => { + cancelled = true; + controller.abort(); + }; + }, [agent, sessionId]); + + // Shell mode changes the path color to yellow as indicator + const pathColor = isShellMode ? 'yellow' : 'blue'; + + return ( + <Box flexDirection="column" paddingX={1}> + {/* Line 1: CWD (left) | Model name (right) */} + <Box flexDirection="row" justifyContent="space-between"> + <Box> + <Text color={pathColor}>{displayPath}</Text> + {branchName && <Text color="gray"> ({branchName})</Text>} + </Box> + <Box> + <Text color="cyan">{displayModelName}</Text> + {viaDexto && <Text color="gray"> via Dexto</Text>} + </Box> + </Box> + + {/* Line 2: Context left */} + {contextLeft && ( + <Box> + <Text color="gray">{contextLeft.percentLeft}% context left</Text> + </Box> + )} + + {/* Line 3: Mode indicators (left) */} + {/* Shift+Tab cycles: Normal → Plan Mode → Accept All Edits → Normal */} + {isShellMode && ( + <Box> + <Text color="yellow" bold> + ! + </Text> + <Text color="gray"> for shell mode</Text> + </Box> + )} + {planModeActive && !isShellMode && ( + <Box> + <Text color="magentaBright">plan mode</Text> + <Text color="gray"> (shift + tab to cycle)</Text> + </Box> + )} + {autoApproveEdits && !planModeActive && !isShellMode && ( + <Box> + <Text color="yellowBright">accept edits</Text> + <Text color="gray"> (shift + tab to cycle)</Text> + </Box> + )} + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/HistorySearchBar.tsx b/dexto/packages/cli/src/cli/ink-cli/components/HistorySearchBar.tsx new file mode 100644 index 00000000..d11fb936 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/HistorySearchBar.tsx @@ -0,0 +1,34 @@ +/** + * HistorySearchBar - UI for Ctrl+R reverse history search + * + * Displayed at the very bottom when search mode is active. + */ + +import React from 'react'; +import { Box, Text } from 'ink'; + +interface HistorySearchBarProps { + /** Current search query */ + query: string; + /** Whether there's a match for the current query */ + hasMatch: boolean; +} + +/** + * Search bar displayed during history search mode + */ +export function HistorySearchBar({ query, hasMatch }: HistorySearchBarProps) { + return ( + <Box flexDirection="column" paddingX={1}> + {/* Hints on separate line above */} + <Text color="gray">Ctrl+R: older, Ctrl+E: newer, Enter: accept, Esc: cancel</Text> + {/* Search query line */} + <Box> + <Text color="green">search history: </Text> + <Text color="cyan">{query}</Text> + <Text color="gray">_</Text> + {query && !hasMatch && <Text color="red"> (no match)</Text>} + </Box> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/MultiLineInput.tsx b/dexto/packages/cli/src/cli/ink-cli/components/MultiLineInput.tsx new file mode 100644 index 00000000..f57b429e --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/MultiLineInput.tsx @@ -0,0 +1,62 @@ +import React, { useMemo } from 'react'; +import { Box, Text } from 'ink'; + +interface MultiLineInputProps { + value: string; + placeholder?: string; + prompt?: string; +} + +/** + * Custom multi-line input display component + * Calculates height based on content and displays text with proper line breaks + */ +export default function MultiLineInput({ value, placeholder, prompt = '> ' }: MultiLineInputProps) { + // Calculate number of lines (split by newlines and count) + const lines = useMemo(() => { + if (!value) return []; + return value.split('\n'); + }, [value]); + + const lineCount = lines.length; + + // If empty, show placeholder + if (!value && placeholder) { + return ( + <Box flexDirection="row"> + <Text color="green" bold> + {prompt} + </Text> + <Text color="gray">{placeholder}</Text> + </Box> + ); + } + + // Display multi-line text - show last N lines if too many + const visibleLines = lineCount > 10 ? lines.slice(-10) : lines; + const startOffset = lineCount > 10 ? lineCount - 10 : 0; + + return ( + <Box flexDirection="column"> + {startOffset > 0 && ( + <Box> + <Text color="gray">... ({startOffset} more lines above)</Text> + </Box> + )} + {visibleLines.map((line, index) => { + const actualIndex = startOffset + index; + return ( + <Box key={actualIndex} flexDirection="row"> + {index === 0 && ( + <Text color="green" bold> + {prompt} + </Text> + )} + {index > 0 && <Text color="green">{' '.repeat(prompt.length)}</Text>} + <Text wrap="wrap">{line || ' '}</Text> + </Box> + ); + })} + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/ResourceAutocomplete.tsx b/dexto/packages/cli/src/cli/ink-cli/components/ResourceAutocomplete.tsx new file mode 100644 index 00000000..44d65406 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/ResourceAutocomplete.tsx @@ -0,0 +1,376 @@ +import React, { + useState, + useEffect, + useRef, + useMemo, + useCallback, + forwardRef, + useImperativeHandle, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../hooks/useInputOrchestrator.js'; +import type { ResourceMetadata } from '@dexto/core'; +import type { DextoAgent } from '@dexto/core'; + +export interface ResourceAutocompleteHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ResourceAutocompleteProps { + isVisible: boolean; + searchQuery: string; + onSelectResource: (resource: ResourceMetadata) => void; + onLoadIntoInput?: (text: string) => void; // New prop for Tab key + onClose: () => void; + agent: DextoAgent; +} + +/** + * Get match score for resource: 0 = no match, 1 = description/URI match, 2 = name includes, 3 = name starts with + * Prioritizes name matches over description/URI matches + */ +function getResourceMatchScore(resource: ResourceMetadata, query: string): number { + if (!query) return 3; // Show all when no query + const lowerQuery = query.toLowerCase(); + const name = (resource.name || '').toLowerCase(); + const uri = resource.uri.toLowerCase(); + const uriFilename = uri.split('/').pop()?.toLowerCase() || ''; + const description = (resource.description || '').toLowerCase(); + + // Highest priority: name starts with query + if (name.startsWith(lowerQuery)) { + return 4; + } + + // Second priority: name includes query + if (name.includes(lowerQuery)) { + return 3; + } + + // Third priority: URI filename starts with query + if (uriFilename.startsWith(lowerQuery)) { + return 2; + } + + // Fourth priority: URI filename includes query + if (uriFilename.includes(lowerQuery)) { + return 2; + } + + // Fifth priority: URI includes query + if (uri.includes(lowerQuery)) { + return 1; + } + + // Lowest priority: description includes query + if (description.includes(lowerQuery)) { + return 1; + } + + return 0; // No match +} + +/** + * Check if resource matches query (for filtering) + */ +function matchesQuery(resource: ResourceMetadata, query: string): boolean { + return getResourceMatchScore(resource, query) > 0; +} + +/** + * Sort resources by match score (highest first), then alphabetically + */ +function sortResources(resources: ResourceMetadata[], query: string): ResourceMetadata[] { + if (!query) return resources; + + const lowerQuery = query.toLowerCase(); + return [...resources].sort((a, b) => { + const scoreA = getResourceMatchScore(a, lowerQuery); + const scoreB = getResourceMatchScore(b, lowerQuery); + + // Sort by score first (higher score first) + if (scoreA !== scoreB) { + return scoreB - scoreA; + } + + // If scores are equal, sort alphabetically by name + const aName = (a.name || '').toLowerCase(); + const bName = (b.name || '').toLowerCase(); + return aName.localeCompare(bName); + }); +} + +/** + * Inner component - wrapped with React.memo below + */ +const ResourceAutocompleteInner = forwardRef<ResourceAutocompleteHandle, ResourceAutocompleteProps>( + function ResourceAutocomplete( + { isVisible, searchQuery, onSelectResource, onLoadIntoInput, onClose, agent }, + ref + ) { + const [resources, setResources] = useState<ResourceMetadata[]>([]); + const [isLoading, setIsLoading] = useState(false); + // Combined state to guarantee single render on navigation + const [selection, setSelection] = useState({ index: 0, offset: 0 }); + const selectedIndexRef = useRef(0); + const MAX_VISIBLE_ITEMS = 5; + + // Update selection AND scroll offset in a single state update + // This guarantees exactly one render per navigation action + const updateSelection = useCallback( + (indexUpdater: number | ((prev: number) => number)) => { + setSelection((prev) => { + const newIndex = + typeof indexUpdater === 'function' + ? indexUpdater(prev.index) + : indexUpdater; + selectedIndexRef.current = newIndex; + + // Calculate new scroll offset + let newOffset = prev.offset; + if (newIndex < prev.offset) { + newOffset = newIndex; + } else if (newIndex >= prev.offset + MAX_VISIBLE_ITEMS) { + newOffset = Math.max(0, newIndex - MAX_VISIBLE_ITEMS + 1); + } + + return { index: newIndex, offset: newOffset }; + }); + }, + [MAX_VISIBLE_ITEMS] + ); + + // Fetch resources from agent + useEffect(() => { + if (!isVisible) return; + + let cancelled = false; + setIsLoading(true); + + const fetchResources = async () => { + try { + const resourceSet = await agent.listResources(); + const resourceList: ResourceMetadata[] = Object.values(resourceSet); + if (!cancelled) { + setResources(resourceList); + setIsLoading(false); + } + } catch { + if (!cancelled) { + // Silently fail - don't use console.error as it interferes with Ink rendering + setResources([]); + setIsLoading(false); + } + } + }; + + void fetchResources(); + + return () => { + cancelled = true; + }; + }, [isVisible, agent]); + + // NOTE: Auto-close logic is handled synchronously in TextBufferInput.tsx + // (on backspace deleting @ and on space after @). We don't use useEffect here + // because React batches state updates, causing race conditions where isVisible + // and searchQuery update at different times. + + // Extract query from @mention (everything after @) + const mentionQuery = useMemo(() => { + // Find the last @ that's at start or after space + const atIndex = searchQuery.lastIndexOf('@'); + if (atIndex === -1) return ''; + + // Check if @ is at start or preceded by space + const prevChar = searchQuery[atIndex - 1]; + if (atIndex === 0 || (prevChar && /\s/.test(prevChar))) { + return searchQuery.slice(atIndex + 1).trim(); + } + return ''; + }, [searchQuery]); + + // Filter and sort resources (no limit - scrolling handles it) + const filteredResources = useMemo(() => { + const matched = resources.filter((r) => matchesQuery(r, mentionQuery)); + return sortResources(matched, mentionQuery); + }, [resources, mentionQuery]); + + // Track items length for reset detection + const prevItemsLengthRef = useRef(filteredResources.length); + const itemsChanged = filteredResources.length !== prevItemsLengthRef.current; + + // Derive clamped selection values during render (always valid, no setState needed) + // This prevents the double-render that was causing flickering + const selectedIndex = itemsChanged + ? 0 + : Math.min(selection.index, Math.max(0, filteredResources.length - 1)); + const scrollOffset = itemsChanged + ? 0 + : Math.min(selection.offset, Math.max(0, filteredResources.length - MAX_VISIBLE_ITEMS)); + + // Sync state only when items actually changed AND state differs + // This effect runs AFTER render, updating state for next user interaction + useEffect(() => { + if (itemsChanged) { + prevItemsLengthRef.current = filteredResources.length; + // Only setState if values actually differ (prevents unnecessary re-render) + if (selection.index !== 0 || selection.offset !== 0) { + selectedIndexRef.current = 0; + setSelection({ index: 0, offset: 0 }); + } else { + selectedIndexRef.current = 0; + } + } + }, [itemsChanged, filteredResources.length, selection.index, selection.offset]); + + // Calculate visible items based on scroll offset + const visibleResources = useMemo(() => { + return filteredResources.slice(scrollOffset, scrollOffset + MAX_VISIBLE_ITEMS); + }, [filteredResources, scrollOffset, MAX_VISIBLE_ITEMS]); + + // Expose handleInput method via ref + useImperativeHandle( + ref, + () => ({ + handleInput: (_input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape always closes, regardless of item count + if (key.escape) { + onClose(); + return true; + } + + const itemsLength = filteredResources.length; + if (itemsLength === 0) return false; + + if (key.upArrow) { + updateSelection((prev) => (prev - 1 + itemsLength) % itemsLength); + return true; + } + + if (key.downArrow) { + updateSelection((prev) => (prev + 1) % itemsLength); + return true; + } + + // Tab to load into input (for editing before selection) + if (key.tab) { + const resource = filteredResources[selectedIndexRef.current]; + if (!resource) return false; + + // Get the @ position and construct the text to load + const atIndex = searchQuery.lastIndexOf('@'); + if (atIndex >= 0) { + const before = searchQuery.slice(0, atIndex + 1); + const uriParts = resource.uri.split('/'); + const reference = + resource.name || uriParts[uriParts.length - 1] || resource.uri; + onLoadIntoInput?.(`${before}${reference}`); + } else { + // Fallback: just append @resource + const uriParts = resource.uri.split('/'); + const reference = + resource.name || uriParts[uriParts.length - 1] || resource.uri; + onLoadIntoInput?.(`${searchQuery}@${reference}`); + } + return true; + } + + // Enter to select + if (key.return) { + const resource = filteredResources[selectedIndexRef.current]; + if (resource) { + onSelectResource(resource); + return true; + } + } + + // Don't consume other keys (typing, backspace, etc.) + return false; + }, + }), + [ + isVisible, + filteredResources, + selectedIndexRef, + searchQuery, + onClose, + onLoadIntoInput, + onSelectResource, + updateSelection, + ] + ); + + if (!isVisible) return null; + + if (isLoading) { + return ( + <Box paddingX={0} paddingY={0}> + <Text color="gray">Loading resources...</Text> + </Box> + ); + } + + if (filteredResources.length === 0) { + return ( + <Box paddingX={0} paddingY={0}> + <Text color="gray"> + {mentionQuery + ? `No resources match "${mentionQuery}"` + : 'No resources available. Connect an MCP server or enable internal resources.'} + </Text> + </Box> + ); + } + + const totalItems = filteredResources.length; + + return ( + <Box flexDirection="column"> + <Box paddingX={0} paddingY={0}> + <Text color="yellowBright" bold> + Resources ({selectedIndex + 1}/{totalItems}) - ↑↓ navigate, Tab load, Enter + select, Esc close + </Text> + </Box> + {visibleResources.map((resource, visibleIndex) => { + const actualIndex = scrollOffset + visibleIndex; + const isSelected = actualIndex === selectedIndex; + const uriParts = resource.uri.split('/'); + const displayName = + resource.name || uriParts[uriParts.length - 1] || resource.uri; + const isImage = (resource.mimeType || '').startsWith('image/'); + + // Truncate URI for display (show last 40 chars with ellipsis) + const truncatedUri = + resource.uri.length > 50 ? '…' + resource.uri.slice(-49) : resource.uri; + + return ( + <Box key={resource.uri}> + {isImage && <Text color={isSelected ? 'cyan' : 'gray'}>🖼️ </Text>} + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {displayName} + </Text> + {resource.serverName && ( + <Text color="gray"> [{resource.serverName}]</Text> + )} + <Text color="gray"> {truncatedUri}</Text> + </Box> + ); + })} + </Box> + ); + } +); + +/** + * Export with React.memo to prevent unnecessary re-renders from parent + * Only re-renders when props actually change (shallow comparison) + */ +const ResourceAutocomplete = React.memo( + ResourceAutocompleteInner +) as typeof ResourceAutocompleteInner; + +export default ResourceAutocomplete; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/SlashCommandAutocomplete.tsx b/dexto/packages/cli/src/cli/ink-cli/components/SlashCommandAutocomplete.tsx new file mode 100644 index 00000000..04c89bdb --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/SlashCommandAutocomplete.tsx @@ -0,0 +1,589 @@ +import React, { + useState, + useEffect, + useRef, + useMemo, + useCallback, + forwardRef, + useImperativeHandle, +} from 'react'; +import { Box, Text, useStdout } from 'ink'; +import type { Key } from '../hooks/useInputOrchestrator.js'; +import type { PromptInfo } from '@dexto/core'; +import type { DextoAgent } from '@dexto/core'; +import { getAllCommands } from '../../commands/interactive-commands/commands.js'; +import type { CommandDefinition } from '../../commands/interactive-commands/command-parser.js'; + +export interface SlashCommandAutocompleteHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface SlashCommandAutocompleteProps { + isVisible: boolean; + searchQuery: string; + onSelectPrompt: (prompt: PromptInfo) => void; + onSelectSystemCommand?: (command: string) => void; + onLoadIntoInput?: (command: string) => void; // For Tab - loads command into input + onSubmitRaw?: ((text: string) => Promise<void> | void) | undefined; // For Enter with no matches - submit raw text + onClose: () => void; + agent: DextoAgent; +} + +interface PromptItem extends PromptInfo { + kind: 'prompt'; +} + +interface SystemCommandItem { + kind: 'system'; + name: string; + description: string; + category?: string; + aliases?: string[]; +} + +/** + * Get match score for prompt: 0 = no match, 1 = description/title match, 2 = name includes, 3 = name starts with + */ +function getPromptMatchScore(prompt: PromptInfo, query: string): number { + if (!query) return 3; // Show all when no query + const lowerQuery = query.toLowerCase(); + const name = prompt.name.toLowerCase(); + const description = (prompt.description || '').toLowerCase(); + const title = (prompt.title || '').toLowerCase(); + + // Highest priority: name starts with query + if (name.startsWith(lowerQuery)) { + return 3; + } + + // Second priority: name includes query + if (name.includes(lowerQuery)) { + return 2; + } + + // Lowest priority: description or title includes query + if (description.includes(lowerQuery) || title.includes(lowerQuery)) { + return 1; + } + + return 0; // No match +} + +/** + * Check if prompt matches query (for filtering) + */ +function matchesPromptQuery(prompt: PromptInfo, query: string): boolean { + return getPromptMatchScore(prompt, query) > 0; +} + +type CommandMatchCandidate = Pick<CommandDefinition, 'name' | 'description' | 'aliases'>; + +/** + * Simple fuzzy match - checks if query matches system command name or description + * Returns a score: 0 = no match, 1 = description match, 2 = alias match, 3 = name includes, 4 = name starts with + */ +function getSystemCommandMatchScore(cmd: CommandMatchCandidate, query: string): number { + if (!query) return 4; // Show all when no query + const lowerQuery = query.toLowerCase(); + const name = cmd.name.toLowerCase(); + const description = (cmd.description || '').toLowerCase(); + + // Highest priority: name starts with query + if (name.startsWith(lowerQuery)) { + return 4; + } + + // Second priority: name includes query + if (name.includes(lowerQuery)) { + return 3; + } + + // Third priority: aliases match + if (cmd.aliases) { + for (const alias of cmd.aliases) { + const lowerAlias = alias.toLowerCase(); + if (lowerAlias.startsWith(lowerQuery)) { + return 2; + } + if (lowerAlias.includes(lowerQuery)) { + return 2; + } + } + } + + // Lowest priority: description includes query + if (description.includes(lowerQuery)) { + return 1; + } + + return 0; // No match +} + +/** + * Check if command matches query (for filtering) + */ +function matchesSystemCommandQuery(cmd: CommandMatchCandidate, query: string): boolean { + return getSystemCommandMatchScore(cmd, query) > 0; +} + +/** + * Truncate text to fit within maxLength, adding ellipsis if truncated + */ +function truncateText(text: string, maxLength: number): string { + if (text.length <= maxLength) return text; + if (maxLength <= 3) return text.slice(0, maxLength); + return text.slice(0, maxLength - 1) + '…'; +} + +/** + * Inner component - wrapped with React.memo below + */ +const SlashCommandAutocompleteInner = forwardRef< + SlashCommandAutocompleteHandle, + SlashCommandAutocompleteProps +>(function SlashCommandAutocomplete( + { + isVisible, + searchQuery, + onSelectPrompt, + onSelectSystemCommand, + onLoadIntoInput, + onSubmitRaw, + onClose, + agent, + }, + ref +) { + const [prompts, setPrompts] = useState<PromptItem[]>([]); + const [systemCommands, setSystemCommands] = useState<SystemCommandItem[]>([]); + const [isLoading, setIsLoading] = useState(false); + // Combined state to guarantee single render on navigation + const [selection, setSelection] = useState({ index: 0, offset: 0 }); + const selectedIndexRef = useRef(0); + const { stdout } = useStdout(); + const terminalWidth = stdout?.columns || 80; + const MAX_VISIBLE_ITEMS = 8; + + // Update selection AND scroll offset in a single state update + // This guarantees exactly one render per navigation action + const updateSelection = useCallback( + (indexUpdater: number | ((prev: number) => number)) => { + setSelection((prev) => { + const newIndex = + typeof indexUpdater === 'function' ? indexUpdater(prev.index) : indexUpdater; + selectedIndexRef.current = newIndex; + + // Calculate new scroll offset + let newOffset = prev.offset; + if (newIndex < prev.offset) { + newOffset = newIndex; + } else if (newIndex >= prev.offset + MAX_VISIBLE_ITEMS) { + newOffset = Math.max(0, newIndex - MAX_VISIBLE_ITEMS + 1); + } + + return { index: newIndex, offset: newOffset }; + }); + }, + [MAX_VISIBLE_ITEMS] + ); + + // Fetch prompts and system commands from agent + useEffect(() => { + if (!isVisible) return; + + let cancelled = false; + setIsLoading(true); + + const fetchCommands = async () => { + try { + // Fetch prompts + const promptSet = await agent.listPrompts(); + const promptList: PromptItem[] = Object.values(promptSet).map((p) => ({ + ...p, + kind: 'prompt' as const, + })); + + // Fetch system commands + const allCommands = getAllCommands(); + const commandList: SystemCommandItem[] = allCommands.map((cmd) => ({ + kind: 'system' as const, + name: cmd.name, + description: cmd.description, + ...(cmd.category && { category: cmd.category }), + ...(cmd.aliases && { aliases: cmd.aliases }), + })); + + if (!cancelled) { + setPrompts(promptList); + setSystemCommands(commandList); + setIsLoading(false); + } + } catch { + if (!cancelled) { + // Silently fail - don't use console.error as it interferes with Ink rendering + setPrompts([]); + setSystemCommands([]); + setIsLoading(false); + } + } + }; + + void fetchCommands(); + + return () => { + cancelled = true; + }; + }, [isVisible, agent]); + + // Extract command name from search query (only the first word after /) + const commandQuery = useMemo(() => { + if (!searchQuery.startsWith('/')) return ''; + const afterSlash = searchQuery.slice(1).trim(); + // Only take the first word (command name), not the arguments + const spaceIndex = afterSlash.indexOf(' '); + return spaceIndex > 0 ? afterSlash.slice(0, spaceIndex) : afterSlash; + }, [searchQuery]); + + // Filter prompts and system commands based on query + const filteredPrompts = useMemo(() => { + if (!commandQuery) { + return prompts; + } + // Filter and sort by match score (highest first) + return prompts + .filter((p) => matchesPromptQuery(p, commandQuery)) + .sort((a, b) => { + const scoreA = getPromptMatchScore(a, commandQuery); + const scoreB = getPromptMatchScore(b, commandQuery); + return scoreB - scoreA; // Higher score first + }); + }, [prompts, commandQuery]); + + const filteredSystemCommands = useMemo(() => { + if (!commandQuery) { + return systemCommands; + } + // Filter and sort by match score (highest first) + return systemCommands + .filter((cmd) => matchesSystemCommandQuery(cmd, commandQuery)) + .sort((a, b) => { + const scoreA = getSystemCommandMatchScore(a, commandQuery); + const scoreB = getSystemCommandMatchScore(b, commandQuery); + return scoreB - scoreA; // Higher score first + }); + }, [systemCommands, commandQuery]); + + // Check if user has started typing arguments (hide autocomplete) + const hasArguments = useMemo(() => { + if (!searchQuery.startsWith('/')) return false; + const afterSlash = searchQuery.slice(1).trim(); + return afterSlash.includes(' '); + }, [searchQuery]); + + // Combine items: system commands first, then prompts + // When typing arguments, show only exact matches (for argument hints) + const combinedItems = useMemo(() => { + const items: Array< + { kind: 'system'; command: SystemCommandItem } | { kind: 'prompt'; prompt: PromptItem } + > = []; + + if (hasArguments) { + // When typing arguments, only show exact match for the command + const exactSystemCmd = systemCommands.find( + (cmd) => cmd.name.toLowerCase() === commandQuery.toLowerCase() + ); + if (exactSystemCmd) { + items.push({ kind: 'system', command: exactSystemCmd }); + } + const exactPrompt = prompts.find( + (p) => + p.name.toLowerCase() === commandQuery.toLowerCase() || + (p.displayName && p.displayName.toLowerCase() === commandQuery.toLowerCase()) + ); + if (exactPrompt) { + items.push({ kind: 'prompt', prompt: exactPrompt }); + } + return items; + } + + // System commands first (they're more commonly used) + filteredSystemCommands.forEach((cmd) => items.push({ kind: 'system', command: cmd })); + + // Then prompts + filteredPrompts.forEach((prompt) => items.push({ kind: 'prompt', prompt })); + + return items; + }, [ + hasArguments, + filteredPrompts, + filteredSystemCommands, + systemCommands, + prompts, + commandQuery, + ]); + + // Get stable identity for first item (used to detect content changes) + const getFirstItemId = (): string | null => { + const first = combinedItems[0]; + if (!first) return null; + if (first.kind === 'system') return `sys:${first.command.name}`; + return `prompt:${first.prompt.name}`; + }; + + // Track items for reset detection (length + first item identity) + const currentFirstId = getFirstItemId(); + const prevItemsRef = useRef({ length: combinedItems.length, firstId: currentFirstId }); + const itemsChanged = + combinedItems.length !== prevItemsRef.current.length || + currentFirstId !== prevItemsRef.current.firstId; + + // Derive clamped selection values during render (always valid, no setState needed) + // This prevents the double-render that was causing flickering + const selectedIndex = itemsChanged + ? 0 + : Math.min(selection.index, Math.max(0, combinedItems.length - 1)); + const scrollOffset = itemsChanged + ? 0 + : Math.min(selection.offset, Math.max(0, combinedItems.length - MAX_VISIBLE_ITEMS)); + + // Sync state only when items actually changed AND state differs + // This effect runs AFTER render, updating state for next user interaction + useEffect(() => { + if (itemsChanged) { + prevItemsRef.current = { length: combinedItems.length, firstId: currentFirstId }; + // Only setState if values actually differ (prevents unnecessary re-render) + if (selection.index !== 0 || selection.offset !== 0) { + selectedIndexRef.current = 0; + setSelection({ index: 0, offset: 0 }); + } else { + selectedIndexRef.current = 0; + } + } + }, [itemsChanged, combinedItems.length, currentFirstId, selection.index, selection.offset]); + + // Calculate visible items based on scroll offset + const visibleItems = useMemo(() => { + return combinedItems.slice(scrollOffset, scrollOffset + MAX_VISIBLE_ITEMS); + }, [combinedItems, scrollOffset, MAX_VISIBLE_ITEMS]); + + // Expose handleInput method via ref + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape always closes, regardless of item count + if (key.escape) { + onClose(); + return true; + } + + const itemsLength = combinedItems.length; + + // Handle Enter when no matches or typing arguments + // Submit raw text directly (main input won't handle it since overlay is active) + if (itemsLength === 0 || hasArguments) { + if (key.return) { + void Promise.resolve(onSubmitRaw?.(searchQuery)).catch((err) => { + const message = err instanceof Error ? err.message : String(err); + agent.logger.error( + `SlashCommandAutocomplete: Failed to submit raw command: ${message}` + ); + }); + onClose(); + return true; + } + // Let other keys (typing, backspace) fall through + return false; + } + + if (key.upArrow) { + updateSelection((prev) => (prev - 1 + itemsLength) % itemsLength); + return true; + } + + if (key.downArrow) { + updateSelection((prev) => (prev + 1) % itemsLength); + return true; + } + + // Tab: For interactive commands (model, resume, switch), execute them like Enter + // For other commands, load into input for editing + if (key.tab) { + const item = combinedItems[selectedIndexRef.current]; + if (!item) return false; + + // Check if this is an interactive command that should be executed + const interactiveCommands = ['model', 'resume', 'switch']; + const isInteractiveCommand = + item.kind === 'system' && interactiveCommands.includes(item.command.name); + + if (isInteractiveCommand && item.kind === 'system') { + // Execute interactive command (same as Enter) + onSelectSystemCommand?.(item.command.name); + } else if (item.kind === 'system') { + // Load system command into input + onLoadIntoInput?.(`/${item.command.name}`); + } else { + // Load prompt command into input using pre-computed commandName + // commandName is collision-resolved by PromptManager (e.g., "plan" or "config:plan") + const cmdName = + item.prompt.commandName || item.prompt.displayName || item.prompt.name; + const argsString = + item.prompt.arguments && item.prompt.arguments.length > 0 + ? ' ' + + item.prompt.arguments + .map((arg) => `<${arg.name}${arg.required ? '' : '?'}>`) + .join(' ') + : ''; + onLoadIntoInput?.(`/${cmdName}${argsString}`); + } + return true; + } + + // Enter: Execute the highlighted command/prompt and close overlay + if (key.return) { + const item = combinedItems[selectedIndexRef.current]; + if (!item) return false; + if (item.kind === 'system') { + onSelectSystemCommand?.(item.command.name); + } else { + onSelectPrompt(item.prompt); + } + onClose(); + return true; + } + + // Don't consume other keys (typing, backspace, etc.) + // Let them fall through to the input handler + return false; + }, + }), + [ + isVisible, + combinedItems, + hasArguments, + selectedIndexRef, + searchQuery, + onClose, + onLoadIntoInput, + onSubmitRaw, + onSelectPrompt, + onSelectSystemCommand, + updateSelection, + agent, + ] + ); + + if (!isVisible) return null; + + // Show loading state while fetching commands + if (isLoading) { + return ( + <Box width={terminalWidth} paddingX={0} paddingY={0}> + <Text color="gray">Loading commands...</Text> + </Box> + ); + } + + // If no items after loading, don't render + if (combinedItems.length === 0) { + return null; + } + + const totalItems = combinedItems.length; + + // Show simplified header when user is typing arguments + const headerText = hasArguments + ? 'Press Enter to execute' + : `Commands (${selectedIndex + 1}/${totalItems}) - ↑↓ navigate, Tab load, Enter execute, Esc close`; + + return ( + <Box flexDirection="column" width={terminalWidth}> + <Box paddingX={0} paddingY={0}> + <Text color="purple" bold> + {headerText} + </Text> + </Box> + {visibleItems.map((item, visibleIndex) => { + const actualIndex = scrollOffset + visibleIndex; + const isSelected = actualIndex === selectedIndex; + + if (item.kind === 'system') { + const cmd = item.command; + const nameText = `/${cmd.name}`; + const categoryText = cmd.category ? ` (${cmd.category})` : ''; + const descText = cmd.description || ''; + + // Two-line layout: + // Line 1: /command-name + // Line 2: Description text (category) + return ( + <Box key={`system-${cmd.name}`} flexDirection="column" paddingX={0}> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {nameText} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> + {' '} + {descText} + {categoryText} + </Text> + </Box> + ); + } + + // Prompt command (MCP prompts) + const prompt = item.prompt; + // Use displayName for user-friendly display, fall back to full name + const displayName = prompt.displayName || prompt.name; + // Check if there's a collision (commandName includes source prefix) + const hasCollision = prompt.commandName && prompt.commandName !== displayName; + const nameText = `/${displayName}`; + const argsString = + prompt.arguments && prompt.arguments.length > 0 + ? ' ' + + prompt.arguments + .map((arg) => `<${arg.name}${arg.required ? '' : '?'}>`) + .join(' ') + : ''; + const description = prompt.title || prompt.description || ''; + + // Two-line layout: + // Line 1: /command-name <args> + // Line 2: Description text (source) + const commandText = nameText + argsString; + // Show source as label, with collision indicator if needed + // For plugin skills, show namespace (plugin name) instead of "config" + const metadata = prompt.metadata as Record<string, unknown> | undefined; + const displaySource = metadata?.namespace + ? String(metadata.namespace) + : prompt.source || 'prompt'; + const sourceLabel = hasCollision + ? `${displaySource} - use /${prompt.commandName}` + : displaySource; + + return ( + <Box key={`prompt-${prompt.name}`} flexDirection="column" paddingX={0}> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {commandText} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> + {' '} + {description} + {` (${sourceLabel})`} + </Text> + </Box> + ); + })} + </Box> + ); +}); + +/** + * Export with React.memo to prevent unnecessary re-renders from parent + * Only re-renders when props actually change (shallow comparison) + */ +export const SlashCommandAutocomplete = React.memo( + SlashCommandAutocompleteInner +) as typeof SlashCommandAutocompleteInner; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/StatusBar.tsx b/dexto/packages/cli/src/cli/ink-cli/components/StatusBar.tsx new file mode 100644 index 00000000..0bc9fbd5 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/StatusBar.tsx @@ -0,0 +1,176 @@ +/** + * StatusBar Component + * Displays processing status and controls above the input area + * + * Layout: + * - Line 1: Spinner + phrase (+ queue count if any) + * - Line 2: Meta info (time, tokens, cancel hint) + * This 2-line layout prevents truncation on any terminal width. + */ + +import { Box, Text } from 'ink'; +import Spinner from 'ink-spinner'; +import type { DextoAgent } from '@dexto/core'; +import { usePhraseCycler } from '../hooks/usePhraseCycler.js'; +import { useElapsedTime } from '../hooks/useElapsedTime.js'; +import { useTokenCounter } from '../hooks/useTokenCounter.js'; + +interface StatusBarProps { + agent: DextoAgent; + isProcessing: boolean; + isThinking: boolean; + isCompacting: boolean; + approvalQueueCount: number; + copyModeEnabled?: boolean; + /** Whether an approval prompt is currently shown */ + isAwaitingApproval?: boolean; + /** Whether the todo list is expanded */ + todoExpanded?: boolean; + /** Whether there are todos to display */ + hasTodos?: boolean; + /** Whether plan mode is active */ + planModeActive?: boolean; + /** Whether accept all edits mode is active */ + autoApproveEdits?: boolean; +} + +/** + * Status bar that shows processing state above input area + * Provides clear feedback on whether the agent is running or idle + * + * Design decisions: + * - Hide spinner during approval wait (user is reviewing, not waiting) + * - Only show elapsed time after 30s (avoid visual noise for fast operations) + */ +export function StatusBar({ + agent, + isProcessing, + isThinking, + isCompacting, + approvalQueueCount, + copyModeEnabled = false, + isAwaitingApproval = false, + todoExpanded = true, + hasTodos = false, + planModeActive = false, + autoApproveEdits = false, +}: StatusBarProps) { + // Cycle through witty phrases while processing (not during compacting) + const { phrase } = usePhraseCycler({ isActive: isProcessing && !isCompacting }); + // Track elapsed time during processing + const { formatted: elapsedTime, elapsedMs } = useElapsedTime({ isActive: isProcessing }); + // Track token usage during processing + const { formatted: tokenCount } = useTokenCounter({ agent, isActive: isProcessing }); + // Only show time after 30 seconds + const showTime = elapsedMs >= 30000; + + // Show copy mode warning (highest priority) + if (copyModeEnabled) { + return ( + <Box paddingX={1} marginBottom={0}> + <Text color="yellowBright" bold> + 📋 Copy Mode - Select text with mouse. Press any key to exit. + </Text> + </Box> + ); + } + + if (!isProcessing) { + // Mode indicators (plan mode, accept edits) are shown in Footer, not here + return null; + } + + // Hide status bar during approval wait - user is reviewing, not waiting + if (isAwaitingApproval) { + return null; + } + + // Build the task toggle hint based on state + const todoHint = hasTodos + ? todoExpanded + ? 'ctrl+t to hide tasks' + : 'ctrl+t to show tasks' + : null; + + // Show compacting state - yellow/orange color to indicate context management + if (isCompacting) { + const metaParts: string[] = []; + if (showTime) metaParts.push(`(${elapsedTime})`); + metaParts.push('Esc to cancel'); + if (todoHint) metaParts.push(todoHint); + const metaContent = metaParts.join(' • '); + + return ( + <Box paddingX={1} marginTop={1} flexDirection="column"> + {/* Line 1: spinner + compacting message */} + <Box flexDirection="row" alignItems="center"> + <Text color="yellow"> + <Spinner type="dots" /> + </Text> + <Text color="yellow"> 📦 Compacting context...</Text> + </Box> + {/* Line 2: meta info */} + <Box marginLeft={2}> + <Text color="gray">{metaContent}</Text> + </Box> + </Box> + ); + } + + // Show initial processing state (before streaming starts) - green/teal color + // TODO: Rename this event/state to "reasoning" and associate it with actual reasoning tokens + // Currently "thinking" event fires before any response, not during reasoning token generation + if (isThinking) { + const metaParts: string[] = []; + if (showTime) metaParts.push(`(${elapsedTime})`); + if (tokenCount) metaParts.push(tokenCount); + metaParts.push('Esc to cancel'); + if (todoHint) metaParts.push(todoHint); + const metaContent = metaParts.join(' • '); + + return ( + <Box paddingX={1} marginTop={1} flexDirection="column"> + {/* Line 1: spinner + phrase */} + <Box flexDirection="row" alignItems="center"> + <Text color="green"> + <Spinner type="dots" /> + </Text> + <Text color="green"> {phrase}</Text> + </Box> + {/* Line 2: meta info */} + <Box marginLeft={2}> + <Text color="gray">{metaContent}</Text> + </Box> + </Box> + ); + } + + // Show active streaming state - green/teal color + // Always use 2-line layout: phrase on first line, meta on second + // This prevents truncation and messy wrapping on any terminal width + const metaParts: string[] = []; + if (showTime) metaParts.push(`(${elapsedTime})`); + if (tokenCount) metaParts.push(tokenCount); + metaParts.push('Esc to cancel'); + if (todoHint) metaParts.push(todoHint); + const metaContent = metaParts.join(' • '); + + return ( + <Box paddingX={1} marginTop={1} flexDirection="column"> + {/* Line 1: spinner + phrase + queue count */} + <Box flexDirection="row" alignItems="center"> + <Text color="green"> + <Spinner type="dots" /> + </Text> + <Text color="green"> {phrase}</Text> + {approvalQueueCount > 0 && ( + <Text color="yellowBright"> • {approvalQueueCount} queued</Text> + )} + </Box> + {/* Line 2: meta info (time, tokens, cancel hint) */} + <Box marginLeft={2}> + <Text color="gray">{metaContent}</Text> + </Box> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/TextBufferInput.tsx b/dexto/packages/cli/src/cli/ink-cli/components/TextBufferInput.tsx new file mode 100644 index 00000000..5032bcc9 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/TextBufferInput.tsx @@ -0,0 +1,719 @@ +/** + * TextBufferInput Component + * + * Buffer is passed as prop from parent. + * Uses direct useKeypress for input handling (no ref chain). + * Parent owns the buffer and can read values directly. + */ + +import React, { useCallback, useRef, useEffect } from 'react'; +import { Box, Text, useStdout } from 'ink'; +import { useKeypress, type Key } from '../hooks/useKeypress.js'; +import type { TextBuffer } from './shared/text-buffer.js'; +import type { PendingImage, PastedBlock } from '../state/types.js'; +import { readClipboardImage } from '../utils/clipboardUtils.js'; + +/** Overlay trigger types for event-driven overlay detection */ +export type OverlayTrigger = 'slash-autocomplete' | 'resource-autocomplete' | 'close'; + +/** Threshold for collapsing pasted content */ +const PASTE_COLLAPSE_LINE_THRESHOLD = 3; +const PASTE_COLLAPSE_CHAR_THRESHOLD = 150; + +/** Platform-aware keyboard shortcut labels */ +const isMac = process.platform === 'darwin'; +const KEY_LABELS = { + ctrlT: isMac ? '⌃T' : 'Ctrl+T', + altUp: isMac ? '⌥↑' : 'Alt+Up', + altDown: isMac ? '⌥↓' : 'Alt+Down', +}; + +interface TextBufferInputProps { + /** Text buffer (owned by parent) */ + buffer: TextBuffer; + /** Called when user presses Enter to submit */ + onSubmit: (value: string) => void; + /** Placeholder text when empty */ + placeholder?: string | undefined; + /** Whether input handling is disabled (e.g., during processing) */ + isDisabled?: boolean | undefined; + /** Called for history navigation (up/down at boundaries) */ + onHistoryNavigate?: ((direction: 'up' | 'down') => void) | undefined; + /** Called to trigger overlay (slash command, @mention) */ + onTriggerOverlay?: ((trigger: OverlayTrigger) => void) | undefined; + /** Maximum lines to show in viewport */ + maxViewportLines?: number | undefined; + /** Whether this input should handle keypresses */ + isActive: boolean; + /** Optional handler for keyboard scroll (PageUp/PageDown, Shift+arrows) */ + onKeyboardScroll?: ((direction: 'up' | 'down') => void) | undefined; + /** Current number of attached images (for placeholder numbering) */ + imageCount?: number | undefined; + /** Called when image is pasted from clipboard */ + onImagePaste?: ((image: PendingImage) => void) | undefined; + /** Current pending images (for placeholder removal detection) */ + images?: PendingImage[] | undefined; + /** Called when an image placeholder is removed from text */ + onImageRemove?: ((imageId: string) => void) | undefined; + /** Current pasted blocks for collapse/expand feature */ + pastedBlocks?: PastedBlock[] | undefined; + /** Called when a large paste is detected and should be collapsed */ + onPasteBlock?: ((block: PastedBlock) => void) | undefined; + /** Called to update a pasted block (e.g., toggle collapse) */ + onPasteBlockUpdate?: ((blockId: string, updates: Partial<PastedBlock>) => void) | undefined; + /** Called when a paste block placeholder is removed from text */ + onPasteBlockRemove?: ((blockId: string) => void) | undefined; + /** Query to highlight in input text (for history search) */ + highlightQuery?: string | undefined; +} + +function isBackspaceKey(key: Key): boolean { + return key.name === 'backspace' || key.sequence === '\x7f' || key.sequence === '\x08'; +} + +function isForwardDeleteKey(key: Key): boolean { + return key.name === 'delete'; +} + +/** Renders text with optional query highlighting in green */ +function HighlightedText({ text, query }: { text: string; query: string | undefined }) { + if (!query || !text) { + return <Text>{text}</Text>; + } + + const lowerText = text.toLowerCase(); + const lowerQuery = query.toLowerCase(); + const matchIndex = lowerText.indexOf(lowerQuery); + + if (matchIndex === -1) { + return <Text>{text}</Text>; + } + + const before = text.slice(0, matchIndex); + const match = text.slice(matchIndex, matchIndex + query.length); + const after = text.slice(matchIndex + query.length); + + return ( + <Text> + {before} + <Text color="green" bold> + {match} + </Text> + {after} + </Text> + ); +} + +export function TextBufferInput({ + buffer, + onSubmit, + placeholder, + isDisabled = false, + onHistoryNavigate, + onTriggerOverlay, + maxViewportLines = 10, + isActive, + onKeyboardScroll, + imageCount = 0, + onImagePaste, + images = [], + onImageRemove, + pastedBlocks = [], + onPasteBlock, + onPasteBlockUpdate, + onPasteBlockRemove, + highlightQuery, +}: TextBufferInputProps) { + const { stdout } = useStdout(); + const terminalWidth = stdout?.columns || 80; + + // Use ref to track imageCount to avoid stale closure in async paste handler + const imageCountRef = useRef(imageCount); + useEffect(() => { + imageCountRef.current = imageCount; + }, [imageCount]); + + // Use ref to track paste number for generating sequential IDs + const pasteCounterRef = useRef(pastedBlocks.length); + useEffect(() => { + // Update counter to be at least the current number of blocks + pasteCounterRef.current = Math.max(pasteCounterRef.current, pastedBlocks.length); + }, [pastedBlocks.length]); + + // Check for removed image placeholders after text changes + const checkRemovedImages = useCallback(() => { + if (!onImageRemove || images.length === 0) return; + const currentText = buffer.text; + for (const img of images) { + if (!currentText.includes(img.placeholder)) { + onImageRemove(img.id); + } + } + }, [buffer, images, onImageRemove]); + + // Check for removed paste block placeholders after text changes + const checkRemovedPasteBlocks = useCallback(() => { + if (!onPasteBlockRemove || pastedBlocks.length === 0) return; + const currentText = buffer.text; + for (const block of pastedBlocks) { + // Check if either the placeholder or the full text (when expanded) is present + const textToFind = block.isCollapsed ? block.placeholder : block.fullText; + if (!currentText.includes(textToFind)) { + onPasteBlockRemove(block.id); + } + } + }, [buffer, pastedBlocks, onPasteBlockRemove]); + + // Find the currently expanded paste block (only one can be expanded at a time) + const findExpandedBlock = useCallback((): PastedBlock | null => { + return pastedBlocks.find((block) => !block.isCollapsed) || null; + }, [pastedBlocks]); + + // Find which collapsed paste block the cursor is on (by placeholder) + const findCollapsedBlockAtCursor = useCallback((): PastedBlock | null => { + if (pastedBlocks.length === 0) return null; + const currentText = buffer.text; + const [cursorRow, cursorCol] = buffer.cursor; + const cursorOffset = getCursorPosition(buffer.lines, cursorRow, cursorCol); + + for (const block of pastedBlocks) { + if (!block.isCollapsed) continue; // Skip expanded blocks + const startIdx = currentText.indexOf(block.placeholder); + if (startIdx === -1) continue; + const endIdx = startIdx + block.placeholder.length; + if (cursorOffset >= startIdx && cursorOffset <= endIdx) { + return block; + } + } + return null; + }, [buffer, pastedBlocks]); + + // Handle Ctrl+T toggle: + // - If something is expanded: collapse it + // - If cursor is on a collapsed paste: expand it + const handlePasteToggle = useCallback(() => { + if (!onPasteBlockUpdate) return; + + const expandedBlock = findExpandedBlock(); + const currentText = buffer.text; + + // If something is expanded, collapse it + if (expandedBlock) { + // Normalize for comparison (buffer might have different line endings) + const normalizedCurrent = currentText.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + const normalizedFullText = expandedBlock.fullText + .replace(/\r\n/g, '\n') + .replace(/\r/g, '\n'); + + const startIdx = normalizedCurrent.indexOf(normalizedFullText); + if (startIdx === -1) { + // Fallback: just mark as collapsed without text replacement + // This handles edge cases where text was modified + onPasteBlockUpdate(expandedBlock.id, { isCollapsed: true }); + return; + } + + // Replace full text with placeholder + const before = currentText.slice(0, startIdx); + const after = currentText.slice(startIdx + normalizedFullText.length); + const newText = before + expandedBlock.placeholder + after; + + // Adjust cursor + const [cursorRow, cursorCol] = buffer.cursor; + const cursorOffset = getCursorPosition(buffer.lines, cursorRow, cursorCol); + let newCursorOffset = cursorOffset; + if (cursorOffset > startIdx) { + // Cursor is after the start of expanded block - adjust + const lengthDiff = expandedBlock.placeholder.length - normalizedFullText.length; + newCursorOffset = Math.max(startIdx, cursorOffset + lengthDiff); + } + + buffer.setText(newText); + buffer.moveToOffset(Math.min(newCursorOffset, newText.length)); + onPasteBlockUpdate(expandedBlock.id, { isCollapsed: true }); + return; + } + + // Otherwise, check if cursor is on a collapsed paste to expand + const collapsedBlock = findCollapsedBlockAtCursor(); + if (collapsedBlock) { + const startIdx = currentText.indexOf(collapsedBlock.placeholder); + if (startIdx === -1) return; + + // Replace placeholder with full text + const before = currentText.slice(0, startIdx); + const after = currentText.slice(startIdx + collapsedBlock.placeholder.length); + const newText = before + collapsedBlock.fullText + after; + + buffer.setText(newText); + // Move cursor to start of expanded content + buffer.moveToOffset(startIdx); + onPasteBlockUpdate(collapsedBlock.id, { isCollapsed: false }); + } + }, [buffer, findExpandedBlock, findCollapsedBlockAtCursor, onPasteBlockUpdate]); + + // Handle keyboard input directly - reads buffer state fresh each time + const handleKeypress = useCallback( + (key: Key) => { + if (isDisabled) return; + + // Read buffer state directly - always fresh, no stale closures + const currentText = buffer.text; + const cursorVisualRow = buffer.visualCursor[0]; + const visualLines = buffer.allVisualLines; + + // === KEYBOARD SCROLL (PageUp/PageDown, Shift+arrows) === + if (onKeyboardScroll) { + if (key.name === 'pageup' || (key.shift && key.name === 'up')) { + onKeyboardScroll('up'); + return; + } + if (key.name === 'pagedown' || (key.shift && key.name === 'down')) { + onKeyboardScroll('down'); + return; + } + } + + // === IMAGE PASTE (Ctrl+V) === + // Check clipboard for image before letting normal paste through + if (key.ctrl && key.name === 'v' && onImagePaste) { + // Async clipboard check - fire and forget, don't block input + void (async () => { + try { + const clipboardImage = await readClipboardImage(); + if (clipboardImage) { + // Use ref to get current count (avoids stale closure issue) + const currentCount = imageCountRef.current; + const imageNumber = currentCount + 1; + // Immediately increment ref to handle rapid pastes + imageCountRef.current = imageNumber; + + const placeholder = `[Image ${imageNumber}]`; + const pendingImage: PendingImage = { + id: `img-${Date.now()}-${imageNumber}`, + data: clipboardImage.data, + mimeType: clipboardImage.mimeType, + placeholder, + }; + onImagePaste(pendingImage); + buffer.insert(placeholder); + } + } catch { + // Clipboard read failed, ignore + } + })(); + return; + } + + // === NEWLINE DETECTION === + const isCtrlJ = key.sequence === '\n'; + const isShiftEnter = + key.sequence === '\\\r' || + (key.name === 'return' && key.shift) || + key.sequence === '\x1b[13;2u' || + key.sequence === '\x1bOM'; + const isPasteReturn = key.name === 'return' && key.paste; + const wantsNewline = + isCtrlJ || isShiftEnter || (key.name === 'return' && key.meta) || isPasteReturn; + + if (wantsNewline) { + buffer.newline(); + return; + } + + // === SUBMIT (Enter) === + if (key.name === 'return' && !key.paste) { + if (currentText.trim()) { + onSubmit(currentText); + } + return; + } + + // === UNDO/REDO === + if (key.ctrl && key.name === 'z' && !key.shift) { + buffer.undo(); + return; + } + if ((key.ctrl && key.name === 'y') || (key.ctrl && key.shift && key.name === 'z')) { + buffer.redo(); + return; + } + + // === PASTE BLOCK TOGGLE (Ctrl+T) === + if (key.ctrl && key.name === 't') { + handlePasteToggle(); + return; + } + + // === BACKSPACE === + if (isBackspaceKey(key) && !key.meta) { + const prevText = buffer.text; + const [cursorRow, cursorCol] = buffer.cursor; + const cursorPos = getCursorPosition(buffer.lines, cursorRow, cursorCol); + + buffer.backspace(); + checkRemovedImages(); + checkRemovedPasteBlocks(); + + // Check if we should close overlay after backspace + // NOTE: buffer.text is memoized and won't update until next render, + // so we calculate the expected new text ourselves + if (onTriggerOverlay && cursorPos > 0) { + const deletedChar = prevText[cursorPos - 1]; + // Calculate what the text will be after backspace + const expectedNewText = + prevText.slice(0, cursorPos - 1) + prevText.slice(cursorPos); + + if (deletedChar === '/' && cursorPos === 1) { + onTriggerOverlay('close'); + } else if (deletedChar === '@') { + // Close if no valid @ mention remains + // A valid @ is at start of text or after whitespace + const hasValidAt = /(^|[\s])@/.test(expectedNewText); + if (!hasValidAt) { + onTriggerOverlay('close'); + } + } + } + return; + } + + // === FORWARD DELETE === + if (isForwardDeleteKey(key)) { + buffer.del(); + checkRemovedImages(); + checkRemovedPasteBlocks(); + return; + } + + // === WORD DELETE === + if (key.ctrl && key.name === 'w') { + buffer.deleteWordLeft(); + checkRemovedImages(); + checkRemovedPasteBlocks(); + return; + } + if (key.meta && isBackspaceKey(key)) { + buffer.deleteWordLeft(); + checkRemovedImages(); + checkRemovedPasteBlocks(); + return; + } + + // === ARROW NAVIGATION === + if (key.name === 'left') { + buffer.move(key.meta || key.ctrl ? 'wordLeft' : 'left'); + return; + } + if (key.name === 'right') { + buffer.move(key.meta || key.ctrl ? 'wordRight' : 'right'); + return; + } + // Cmd+Up: Move to start of input + if (key.meta && key.name === 'up') { + buffer.moveToOffset(0); + return; + } + // Cmd+Down: Move to end of input + if (key.meta && key.name === 'down') { + buffer.moveToOffset(currentText.length); + return; + } + if (key.name === 'up') { + // Only trigger history navigation when at top visual line + if (cursorVisualRow === 0 && onHistoryNavigate) { + onHistoryNavigate('up'); + } else { + buffer.move('up'); + } + return; + } + if (key.name === 'down') { + // Only trigger history navigation when at bottom visual line + if (cursorVisualRow >= visualLines.length - 1 && onHistoryNavigate) { + onHistoryNavigate('down'); + } else { + buffer.move('down'); + } + return; + } + + // === LINE NAVIGATION === + if (key.ctrl && key.name === 'a') { + buffer.move('home'); + return; + } + if (key.ctrl && key.name === 'e') { + buffer.move('end'); + return; + } + if (key.ctrl && key.name === 'k') { + buffer.killLineRight(); + checkRemovedImages(); + checkRemovedPasteBlocks(); + return; + } + if (key.ctrl && key.name === 'u') { + buffer.killLineLeft(); + checkRemovedImages(); + checkRemovedPasteBlocks(); + return; + } + + // === WORD NAVIGATION === + if (key.meta && key.name === 'b') { + buffer.move('wordLeft'); + return; + } + if (key.meta && key.name === 'f') { + buffer.move('wordRight'); + return; + } + + // === CHARACTER INPUT === + if (key.insertable && !key.ctrl && !key.meta) { + const [cursorRow, cursorCol] = buffer.cursor; + const cursorPos = getCursorPosition(buffer.lines, cursorRow, cursorCol); + + // Check if this is a large paste that should be collapsed + if (key.paste && onPasteBlock) { + // Normalize line endings to \n for consistent handling + const pastedText = key.sequence.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + const lineCount = (pastedText.match(/\n/g)?.length ?? 0) + 1; + + if ( + lineCount >= PASTE_COLLAPSE_LINE_THRESHOLD || + pastedText.length > PASTE_COLLAPSE_CHAR_THRESHOLD + ) { + // Create collapsed paste block + pasteCounterRef.current += 1; + const pasteNumber = pasteCounterRef.current; + const placeholder = `[Paste ${pasteNumber}: ~${lineCount} lines]`; + + const pasteBlock: PastedBlock = { + id: `paste-${Date.now()}-${pasteNumber}`, + number: pasteNumber, + fullText: pastedText, + lineCount, + isCollapsed: true, + placeholder, + }; + + // Insert placeholder instead of full text + buffer.insert(placeholder); + onPasteBlock(pasteBlock); + return; + } + } + + buffer.insert(key.sequence, { paste: key.paste }); + + if (onTriggerOverlay) { + if (key.sequence === '/' && cursorPos === 0) { + onTriggerOverlay('slash-autocomplete'); + } else if (key.sequence === '@') { + onTriggerOverlay('resource-autocomplete'); + } else if (/\s/.test(key.sequence)) { + // Close resource autocomplete when user types whitespace + // Whitespace means user is done with the mention (either selected or abandoned) + onTriggerOverlay('close'); + } + } + } + }, + [ + buffer, + isDisabled, + onSubmit, + onHistoryNavigate, + onTriggerOverlay, + onKeyboardScroll, + // imageCount intentionally omitted - callback uses imageCountRef which is synced via useEffect + onImagePaste, + checkRemovedImages, + checkRemovedPasteBlocks, + handlePasteToggle, + onPasteBlock, + ] + ); + + // Subscribe to keypress events when active + useKeypress(handleKeypress, { isActive: isActive && !isDisabled }); + + // === RENDERING === + // Read buffer state for rendering + const bufferText = buffer.text; + const visualCursor = buffer.visualCursor; + const visualLines = buffer.allVisualLines; + const cursorVisualRow = visualCursor[0]; + const cursorVisualCol = visualCursor[1]; + + const separator = '─'.repeat(terminalWidth); + const totalLines = visualLines.length; + + // Detect shell command mode (input starts with "!") + const isShellMode = bufferText.startsWith('!'); + const promptPrefix = isShellMode ? '$ ' : '> '; + const promptColor = isShellMode ? 'yellow' : 'green'; + const separatorColor = isShellMode ? 'yellow' : 'gray'; + + // Calculate visible window + let startLine = 0; + let endLine = totalLines; + if (totalLines > maxViewportLines) { + const halfViewport = Math.floor(maxViewportLines / 2); + startLine = Math.max(0, cursorVisualRow - halfViewport); + endLine = Math.min(totalLines, startLine + maxViewportLines); + if (endLine === totalLines) { + startLine = Math.max(0, totalLines - maxViewportLines); + } + } + + const visibleLines = visualLines.slice(startLine, endLine); + + // Empty state + if (bufferText === '') { + return ( + <Box flexDirection="column" width={terminalWidth}> + <Text color="gray">{separator}</Text> + <Box width={terminalWidth}> + <Text color="green" bold> + {'> '} + </Text> + <Text inverse> </Text> + {placeholder && <Text color="gray">{placeholder}</Text>} + <Text> + {' '.repeat(Math.max(0, terminalWidth - 3 - (placeholder?.length || 0)))} + </Text> + </Box> + <Text color="gray">{separator}</Text> + </Box> + ); + } + + return ( + <Box flexDirection="column" width={terminalWidth}> + <Text color={separatorColor}>{separator}</Text> + {startLine > 0 && ( + <Text color="gray"> + {' '}↑ {startLine} more line{startLine > 1 ? 's' : ''} above ( + {KEY_LABELS.altUp} to jump) + </Text> + )} + {visibleLines.map((line: string, idx: number) => { + const absoluteRow = startLine + idx; + const isFirst = absoluteRow === 0; + const prefix = isFirst ? promptPrefix : ' '; + const isCursorLine = absoluteRow === cursorVisualRow; + + if (!isCursorLine) { + return ( + <Box key={absoluteRow} width={terminalWidth}> + <Text color={promptColor} bold={isFirst}> + {prefix} + </Text> + <HighlightedText text={line} query={highlightQuery} /> + <Text> + {' '.repeat( + Math.max(0, terminalWidth - prefix.length - line.length) + )} + </Text> + </Box> + ); + } + + const before = line.slice(0, cursorVisualCol); + const atCursor = line.charAt(cursorVisualCol) || ' '; + const after = line.slice(cursorVisualCol + 1); + + return ( + <Box key={absoluteRow} width={terminalWidth}> + <Text color={promptColor} bold={isFirst}> + {prefix} + </Text> + <HighlightedText text={before} query={highlightQuery} /> + <Text inverse>{atCursor}</Text> + <HighlightedText text={after} query={highlightQuery} /> + <Text> + {' '.repeat( + Math.max( + 0, + terminalWidth - prefix.length - before.length - 1 - after.length + ) + )} + </Text> + </Box> + ); + })} + {endLine < totalLines && ( + <Text color="gray"> + {' '}↓ {totalLines - endLine} more line{totalLines - endLine > 1 ? 's' : ''}{' '} + below ({KEY_LABELS.altDown} to jump) + </Text> + )} + {/* Paste block hints */} + {pastedBlocks.length > 0 && ( + <PasteBlockHint + pastedBlocks={pastedBlocks} + expandedBlock={findExpandedBlock()} + cursorOnCollapsed={findCollapsedBlockAtCursor()} + /> + )} + <Text color={separatorColor}>{separator}</Text> + </Box> + ); +} + +/** Hint component for paste blocks */ +function PasteBlockHint({ + pastedBlocks, + expandedBlock, + cursorOnCollapsed, +}: { + pastedBlocks: PastedBlock[]; + expandedBlock: PastedBlock | null; + cursorOnCollapsed: PastedBlock | null; +}) { + const collapsedCount = pastedBlocks.filter((b) => b.isCollapsed).length; + + // If something is expanded, always show collapse hint + if (expandedBlock) { + return ( + <Text color="cyan"> + {' '} + {KEY_LABELS.ctrlT} to collapse expanded paste + </Text> + ); + } + + // If cursor is on a collapsed paste, show expand hint + if (cursorOnCollapsed) { + return ( + <Text color="cyan"> + {' '} + {KEY_LABELS.ctrlT} to expand paste + </Text> + ); + } + + // Otherwise show count of collapsed pastes + if (collapsedCount > 0) { + return ( + <Text color="gray"> + {' '} + {collapsedCount} collapsed paste{collapsedCount > 1 ? 's' : ''} ({KEY_LABELS.ctrlT}{' '} + on placeholder to expand) + </Text> + ); + } + + return null; +} + +function getCursorPosition(lines: string[], cursorRow: number, cursorCol: number): number { + let pos = 0; + for (let i = 0; i < cursorRow; i++) { + pos += (lines[i]?.length ?? 0) + 1; + } + return pos + cursorCol; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/TodoPanel.tsx b/dexto/packages/cli/src/cli/ink-cli/components/TodoPanel.tsx new file mode 100644 index 00000000..07321f56 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/TodoPanel.tsx @@ -0,0 +1,158 @@ +/** + * TodoPanel Component + * + * Displays the current todo list for workflow tracking. + * Shows todos with their status indicators (pending, in progress, completed). + * + * Display modes: + * - Processing + Collapsed: Shows "Next:" with the next pending/in-progress task + * - Processing + Expanded: Shows simple checklist with ☐/☑ indicators below status bar + * - Idle + Expanded: Shows boxed format with header + * - Idle + Collapsed: Hidden + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { TodoItem, TodoStatus } from '../state/types.js'; + +interface TodoPanelProps { + todos: TodoItem[]; + /** Whether to show the full list or just the next task */ + isExpanded: boolean; + /** Whether the agent is currently processing (affects display style) */ + isProcessing?: boolean; +} + +/** + * Get status indicator for a todo item (used in boxed mode) + */ +function getStatusIndicator(status: TodoStatus): { icon: string; color: string } { + switch (status) { + case 'completed': + return { icon: '✓', color: 'green' }; + case 'in_progress': + return { icon: '●', color: 'yellow' }; + case 'pending': + default: + return { icon: '○', color: 'gray' }; + } +} + +/** + * TodoPanel - Shows current todos for workflow tracking + */ +export function TodoPanel({ todos, isExpanded, isProcessing = false }: TodoPanelProps) { + if (todos.length === 0) { + return null; + } + + // Sort todos by position + const sortedTodos = [...todos].sort((a, b) => a.position - b.position); + + // Find the next task to work on (in_progress first, then first pending) + const currentTask = sortedTodos.find((t) => t.status === 'in_progress'); + const nextPendingTask = sortedTodos.find((t) => t.status === 'pending'); + const nextTask = currentTask || nextPendingTask; + + // When idle (not processing) + if (!isProcessing) { + // Collapsed + idle = hidden + if (!isExpanded) { + return null; + } + + // Expanded + idle = boxed format + const completedCount = todos.filter((t) => t.status === 'completed').length; + const totalCount = todos.length; + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="gray" + paddingX={1} + marginX={1} + marginBottom={1} + > + {/* Header */} + <Box> + <Text bold color="cyan"> + 📋 Tasks{' '} + </Text> + <Text color="gray"> + ({completedCount}/{totalCount}) + </Text> + <Text color="gray" dimColor> + {' '} + · ctrl+t to hide tasks + </Text> + </Box> + + {/* Todo items */} + <Box flexDirection="column"> + {sortedTodos.map((todo) => { + const { icon, color } = getStatusIndicator(todo.status); + const isCompleted = todo.status === 'completed'; + const isInProgress = todo.status === 'in_progress'; + + return ( + <Box key={todo.id}> + <Text color={color}>{icon} </Text> + <Text + color={isCompleted ? 'gray' : isInProgress ? 'white' : 'gray'} + strikethrough={isCompleted} + dimColor={!isInProgress && !isCompleted} + > + {isInProgress ? todo.activeForm : todo.content} + </Text> + </Box> + ); + })} + </Box> + </Box> + ); + } + + // When processing - use minimal style + + // Collapsed: show current task being worked on + if (!isExpanded) { + if (!currentTask) { + return null; // No active task + } + + return ( + <Box paddingX={1} marginBottom={1}> + <Box marginLeft={2}> + <Text color="gray">⎿ </Text> + <Text color="gray">{currentTask.activeForm}</Text> + </Box> + </Box> + ); + } + + // Expanded: show simple checklist + return ( + <Box flexDirection="column" paddingX={1} marginBottom={1}> + {sortedTodos.map((todo, index) => { + const isFirst = index === 0; + const isCompleted = todo.status === 'completed'; + const isInProgress = todo.status === 'in_progress'; + const checkbox = isCompleted ? '☑' : '☐'; + + return ( + <Box key={todo.id} marginLeft={2}> + {/* Tree connector for first item, space for others */} + <Text color="gray">{isFirst ? '⎿ ' : ' '}</Text> + <Text color={isCompleted ? 'green' : isInProgress ? 'yellow' : 'white'}> + {checkbox}{' '} + </Text> + <Text color={isCompleted ? 'gray' : 'white'} dimColor={isCompleted}> + {todo.content} + </Text> + </Box> + ); + })} + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/base/BaseAutocomplete.tsx b/dexto/packages/cli/src/cli/ink-cli/components/base/BaseAutocomplete.tsx new file mode 100644 index 00000000..77c5a508 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/base/BaseAutocomplete.tsx @@ -0,0 +1,191 @@ +/** + * Base Autocomplete Component + * Reusable autocomplete with filtering, scoring, and keyboard navigation + * Used by SlashCommandAutocomplete and ResourceAutocomplete + */ + +import React, { useState, useEffect, useRef, useMemo, type ReactNode } from 'react'; +import { Box, Text, useInput } from 'ink'; + +export interface BaseAutocompleteProps<T> { + items: T[]; + query: string; + isVisible: boolean; + isLoading?: boolean; + onSelect: (item: T) => void; + onLoadIntoInput?: (text: string) => void; + onClose: () => void; + filterFn: (item: T, query: string) => boolean; + scoreFn: (item: T, query: string) => number; + formatItem: (item: T, isSelected: boolean) => ReactNode; + formatLoadText?: (item: T) => string; + title: string; + maxVisibleItems?: number; + loadingMessage?: string; + emptyMessage?: string; + borderColor?: string; +} + +/** + * Generic autocomplete component with filtering, scoring, and keyboard navigation + */ +export function BaseAutocomplete<T>({ + items, + query, + isVisible, + isLoading = false, + onSelect, + onLoadIntoInput, + onClose, + filterFn, + scoreFn, + formatItem, + formatLoadText, + title, + maxVisibleItems = 10, + loadingMessage = 'Loading...', + emptyMessage = 'No matches found', + borderColor = 'cyan', +}: BaseAutocompleteProps<T>) { + const [selectedIndex, setSelectedIndex] = useState(0); + const [scrollOffset, setScrollOffset] = useState(0); + const selectedIndexRef = useRef(0); + + // Keep ref in sync + useEffect(() => { + selectedIndexRef.current = selectedIndex; + }, [selectedIndex]); + + // Filter and sort items + const filteredItems = useMemo(() => { + if (!query) return items; + return items + .filter((item) => filterFn(item, query)) + .sort((a, b) => { + const scoreA = scoreFn(a, query); + const scoreB = scoreFn(b, query); + return scoreB - scoreA; // Higher score first + }); + }, [items, query, filterFn, scoreFn]); + + // Reset selection when items change + useEffect(() => { + setSelectedIndex(0); + setScrollOffset(0); + }, [filteredItems.length]); + + // Auto-scroll to keep selected item visible + useEffect(() => { + if (selectedIndex < scrollOffset) { + setScrollOffset(selectedIndex); + } else if (selectedIndex >= scrollOffset + maxVisibleItems) { + setScrollOffset(Math.max(0, selectedIndex - maxVisibleItems + 1)); + } + }, [selectedIndex, scrollOffset, maxVisibleItems]); + + // Calculate visible items + const visibleItems = useMemo(() => { + return filteredItems.slice(scrollOffset, scrollOffset + maxVisibleItems); + }, [filteredItems, scrollOffset, maxVisibleItems]); + + // Handle keyboard navigation + useInput( + (input, key) => { + if (!isVisible) return; + + const itemsLength = filteredItems.length; + if (itemsLength === 0) return; + + if (key.upArrow) { + setSelectedIndex((prev) => (prev - 1 + itemsLength) % itemsLength); + } + + if (key.downArrow) { + setSelectedIndex((prev) => (prev + 1) % itemsLength); + } + + if (key.escape) { + onClose(); + } + + // Tab: Load into input for editing + if (key.tab && onLoadIntoInput && formatLoadText && itemsLength > 0) { + const item = filteredItems[selectedIndexRef.current]; + if (item) { + onLoadIntoInput(formatLoadText(item)); + } + return; + } + + // Enter: Select item + if (key.return && itemsLength > 0) { + const item = filteredItems[selectedIndexRef.current]; + if (item) { + onSelect(item); + } + } + }, + { isActive: isVisible } + ); + + if (!isVisible) return null; + + if (isLoading) { + return ( + <Box borderStyle="single" borderColor="gray" paddingX={1} paddingY={1}> + <Text color="gray">{loadingMessage}</Text> + </Box> + ); + } + + if (filteredItems.length === 0) { + return ( + <Box borderStyle="single" borderColor="gray" paddingX={1} paddingY={1}> + <Text color="gray">{emptyMessage}</Text> + </Box> + ); + } + + const hasMoreAbove = scrollOffset > 0; + const hasMoreBelow = scrollOffset + maxVisibleItems < filteredItems.length; + const totalItems = filteredItems.length; + + return ( + <Box + borderStyle="single" + borderColor={borderColor} + flexDirection="column" + height={Math.min(maxVisibleItems + 3, totalItems + 3)} + > + <Box paddingX={1} paddingY={0}> + <Text color="gray"> + {title} ({selectedIndex + 1}/{totalItems}) - ↑↓ navigate + {onLoadIntoInput && ', Tab load'} + {', Enter select, Esc close'} + </Text> + </Box> + {hasMoreAbove && ( + <Box paddingX={1} paddingY={0}> + <Text color="gray">... ↑ ({scrollOffset} more above)</Text> + </Box> + )} + {visibleItems.map((item, visibleIndex) => { + const actualIndex = scrollOffset + visibleIndex; + const isSelected = actualIndex === selectedIndex; + + return ( + <Box key={actualIndex} paddingX={1} paddingY={0}> + {formatItem(item, isSelected)} + </Box> + ); + })} + {hasMoreBelow && ( + <Box paddingX={1} paddingY={0}> + <Text color="gray"> + ... ↓ ({totalItems - scrollOffset - maxVisibleItems} more below) + </Text> + </Box> + )} + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/base/BaseSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/base/BaseSelector.tsx new file mode 100644 index 00000000..3b970037 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/base/BaseSelector.tsx @@ -0,0 +1,221 @@ +/** + * Base Selector Component + * Reusable selector with keyboard navigation for lists + * Used by ModelSelector and SessionSelector + */ + +import { + useState, + useEffect, + useRef, + useMemo, + useCallback, + forwardRef, + useImperativeHandle, + type ReactNode, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; + +export interface BaseSelectorProps<T> { + items: T[]; + isVisible: boolean; + isLoading?: boolean; + selectedIndex: number; + onSelectIndex: (index: number) => void; + onSelect: (item: T) => void; + onClose: () => void; + formatItem: (item: T, isSelected: boolean) => ReactNode; + title: string; + maxVisibleItems?: number; + loadingMessage?: string; + emptyMessage?: string; + borderColor?: string; + onTab?: (item: T) => void; // Optional Tab key handler + supportsTab?: boolean; // Whether to show Tab in instructions +} + +export interface BaseSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * Generic selector component with keyboard navigation and scrolling + */ +function BaseSelectorInner<T>( + { + items, + isVisible, + isLoading = false, + selectedIndex, + onSelectIndex, + onSelect, + onClose, + formatItem, + title, + maxVisibleItems = 8, + loadingMessage = 'Loading...', + emptyMessage = 'No items found', + borderColor = 'cyan', + onTab, + supportsTab = false, + }: BaseSelectorProps<T>, + ref: React.Ref<BaseSelectorHandle> +) { + // Track scroll offset as state, but derive during render when needed + const [scrollOffsetState, setScrollOffset] = useState(0); + const selectedIndexRef = useRef(selectedIndex); + const prevItemsLengthRef = useRef(items.length); + + // Keep ref in sync + selectedIndexRef.current = selectedIndex; + + // Derive the correct scroll offset during render (no second render needed) + // This handles both selectedIndex changes from parent AND items length changes + const scrollOffset = useMemo(() => { + const itemsChanged = items.length !== prevItemsLengthRef.current; + + // Reset scroll if items changed significantly + if (itemsChanged && items.length <= maxVisibleItems) { + return 0; + } + + let offset = scrollOffsetState; + + // Adjust offset to keep selectedIndex visible + if (selectedIndex < offset) { + offset = selectedIndex; + } else if (selectedIndex >= offset + maxVisibleItems) { + offset = Math.max(0, selectedIndex - maxVisibleItems + 1); + } + + // Clamp to valid range + const maxOffset = Math.max(0, items.length - maxVisibleItems); + return Math.min(maxOffset, Math.max(0, offset)); + }, [selectedIndex, items.length, maxVisibleItems, scrollOffsetState]); + + // Update refs after render (not during useMemo which can run multiple times) + useEffect(() => { + prevItemsLengthRef.current = items.length; + }, [items.length]); + + // Sync scroll offset state after render if it changed + // This ensures the stored state is correct for next navigation + useEffect(() => { + if (scrollOffset !== scrollOffsetState) { + setScrollOffset(scrollOffset); + } + }, [scrollOffset, scrollOffsetState]); + + // Handle selection change - only updates parent state + const handleSelectIndex = useCallback( + (newIndex: number) => { + selectedIndexRef.current = newIndex; + onSelectIndex(newIndex); + }, + [onSelectIndex] + ); + + // Calculate visible items + const visibleItems = useMemo(() => { + return items.slice(scrollOffset, scrollOffset + maxVisibleItems); + }, [items, scrollOffset, maxVisibleItems]); + + // Expose handleInput method via ref + useImperativeHandle( + ref, + () => ({ + handleInput: (_input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape always works, regardless of item count + if (key.escape) { + onClose(); + return true; + } + + const itemsLength = items.length; + if (itemsLength === 0) return false; + + if (key.upArrow) { + const nextIndex = (selectedIndexRef.current - 1 + itemsLength) % itemsLength; + handleSelectIndex(nextIndex); + return true; + } + + if (key.downArrow) { + const nextIndex = (selectedIndexRef.current + 1) % itemsLength; + handleSelectIndex(nextIndex); + return true; + } + + if (key.tab && onTab) { + const item = items[selectedIndexRef.current]; + if (item !== undefined) { + onTab(item); + return true; + } + } + + if (key.return && itemsLength > 0) { + const item = items[selectedIndexRef.current]; + if (item !== undefined) { + onSelect(item); + return true; + } + } + + return false; + }, + }), + [isVisible, items, handleSelectIndex, onClose, onSelect, onTab] + ); + + if (!isVisible) return null; + + if (isLoading) { + return ( + <Box paddingX={0} paddingY={0}> + <Text color="gray">{loadingMessage}</Text> + </Box> + ); + } + + if (items.length === 0) { + return ( + <Box paddingX={0} paddingY={0}> + <Text color="gray">{emptyMessage}</Text> + </Box> + ); + } + + // Build instruction text based on features + const instructions = supportsTab + ? '↑↓ navigate, Tab load, Enter select, Esc close' + : '↑↓ navigate, Enter select, Esc close'; + + return ( + <Box flexDirection="column"> + <Box paddingX={0} paddingY={0}> + <Text color={borderColor} bold> + {title} ({selectedIndex + 1}/{items.length}) - {instructions} + </Text> + </Box> + {visibleItems.map((item, visibleIndex) => { + const actualIndex = scrollOffset + visibleIndex; + const isSelected = actualIndex === selectedIndex; + + return ( + <Box key={actualIndex} paddingX={0} paddingY={0}> + {formatItem(item, isSelected)} + </Box> + ); + })} + </Box> + ); +} + +// Export with proper generic type support +export const BaseSelector = forwardRef(BaseSelectorInner) as <T>( + props: BaseSelectorProps<T> & { ref?: React.Ref<BaseSelectorHandle> } +) => ReturnType<typeof BaseSelectorInner>; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/base/index.ts b/dexto/packages/cli/src/cli/ink-cli/components/base/index.ts new file mode 100644 index 00000000..682e2a48 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/base/index.ts @@ -0,0 +1,6 @@ +/** + * Base components module exports + */ + +export { BaseSelector, type BaseSelectorProps } from './BaseSelector.js'; +export { BaseAutocomplete, type BaseAutocompleteProps } from './BaseAutocomplete.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/ChatView.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/ChatView.tsx new file mode 100644 index 00000000..2dbfdbac --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/ChatView.tsx @@ -0,0 +1,42 @@ +/** + * ChatView Component + * Main chat display area combining header and messages + */ + +import React from 'react'; +import { Box } from 'ink'; +import { Header } from './Header.js'; +import { MessageList } from './MessageList.js'; +import type { Message, StartupInfo } from '../../state/types.js'; + +interface ChatViewProps { + messages: Message[]; + modelName: string; + sessionId?: string | undefined; + hasActiveSession: boolean; + startupInfo: StartupInfo; +} + +/** + * Pure presentational component for chat area + * Combines header and message list + */ +export function ChatView({ + messages, + modelName, + sessionId, + hasActiveSession, + startupInfo, +}: ChatViewProps) { + return ( + <Box flexDirection="column" flexGrow={1}> + <Header + modelName={modelName} + sessionId={sessionId} + hasActiveSession={hasActiveSession} + startupInfo={startupInfo} + /> + <MessageList messages={messages} /> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/Footer.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/Footer.tsx new file mode 100644 index 00000000..54403060 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/Footer.tsx @@ -0,0 +1,20 @@ +/** + * Footer Component + * Displays keyboard shortcuts and help information + */ + +import React from 'react'; +import { Box, Text } from 'ink'; + +/** + * Pure presentational component for CLI footer + */ +export function Footer() { + return ( + <Box borderStyle="single" borderColor="gray" paddingX={1}> + <Text color="gray"> + Shift+Enter/Ctrl+J: newline • Ctrl+W: del word • Ctrl+U: del line • Ctrl+C: exit + </Text> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/Header.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/Header.tsx new file mode 100644 index 00000000..bbc4d809 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/Header.tsx @@ -0,0 +1,106 @@ +/** + * Header Component + * Displays CLI branding and session information + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { StartupInfo } from '../../state/types.js'; +import { useTerminalSize } from '../../hooks/useTerminalSize.js'; + +interface HeaderProps { + modelName: string; + sessionId?: string | undefined; + hasActiveSession: boolean; + startupInfo: StartupInfo; +} + +/** + * Pure presentational component for CLI header + * Automatically adjusts width to terminal size + */ +export function Header({ modelName, sessionId, hasActiveSession, startupInfo }: HeaderProps) { + const { columns } = useTerminalSize(); + + return ( + <Box + borderStyle="single" + borderColor="gray" + paddingX={1} + flexDirection="column" + flexShrink={0} + width={columns} + > + <Box marginTop={1}> + <Text color="greenBright"> + {`██████╗ ███████╗██╗ ██╗████████╗ ██████╗ +██╔══██╗██╔════╝╚██╗██╔╝╚══██╔══╝██╔═══██╗ +██║ ██║█████╗ ╚███╔╝ ██║ ██║ ██║ +██║ ██║██╔══╝ ██╔██╗ ██║ ██║ ██║ +██████╔╝███████╗██╔╝ ██╗ ██║ ╚██████╔╝ +╚═════╝ ╚══════╝╚═╝ ╚═╝ ╚═╝ ╚═════╝`} + </Text> + </Box> + + {/* Model and Session */} + <Box marginTop={1} flexDirection="row"> + <Text color="gray">Model: </Text> + <Text color="white">{modelName}</Text> + {hasActiveSession && sessionId && ( + <> + <Text color="gray"> • Session: </Text> + <Text color="white">{sessionId.slice(0, 8)}</Text> + </> + )} + </Box> + + {/* MCP Servers and Tools */} + <Box flexDirection="row"> + <Text color="gray">Servers: </Text> + <Text color="white">{startupInfo.connectedServers.count}</Text> + <Text color="gray"> • Tools: </Text> + <Text color="white">{startupInfo.toolCount}</Text> + </Box> + + {/* Failed connections warning */} + {startupInfo.failedConnections.length > 0 && ( + <Box flexDirection="row"> + <Text color="yellowBright"> + ⚠️ Failed: {startupInfo.failedConnections.join(', ')} + </Text> + </Box> + )} + + {/* Log file (only shown in dev mode) */} + {startupInfo.logFile && process.env.DEXTO_DEV_MODE === 'true' && ( + <Box flexDirection="row"> + <Text color="gray">Logs: {startupInfo.logFile}</Text> + </Box> + )} + + {/* Update available notification */} + {startupInfo.updateInfo && ( + <Box marginTop={1} flexDirection="row"> + <Text color="yellow"> + ⬆️ Update available: {startupInfo.updateInfo.current} →{' '} + {startupInfo.updateInfo.latest} + </Text> + <Text color="gray"> • Run: </Text> + <Text color="cyan">{startupInfo.updateInfo.updateCommand}</Text> + </Box> + )} + + {/* Agent sync notification */} + {startupInfo.needsAgentSync && ( + <Box marginTop={startupInfo.updateInfo ? 0 : 1} flexDirection="row"> + <Text color="yellow">🔄 Agent configs have updates available. Run: </Text> + <Text color="cyan">dexto sync-agents</Text> + </Box> + )} + + <Box marginBottom={1}> + <Text> </Text> + </Box> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/MessageItem.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/MessageItem.tsx new file mode 100644 index 00000000..8608662f --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/MessageItem.tsx @@ -0,0 +1,311 @@ +/** + * MessageItem Component + * Displays a single message with visual hierarchy + * Uses colors and spacing instead of explicit labels + */ + +import { memo } from 'react'; +import { Box, Text } from 'ink'; +import wrapAnsi from 'wrap-ansi'; +import type { + Message, + ConfigStyledData, + StatsStyledData, + HelpStyledData, + SessionListStyledData, + SessionHistoryStyledData, + LogConfigStyledData, + RunSummaryStyledData, + ShortcutsStyledData, + SysPromptStyledData, +} from '../../state/types.js'; +import { + ConfigBox, + StatsBox, + HelpBox, + SessionListBox, + SessionHistoryBox, + LogConfigBox, + ShortcutsBox, + SyspromptBox, +} from './styled-boxes/index.js'; +import { ToolResultRenderer } from '../renderers/index.js'; +import { MarkdownText } from '../shared/MarkdownText.js'; +import { ToolIcon } from './ToolIcon.js'; + +/** + * Strip <plan-mode>...</plan-mode> tags from content. + * Plan mode instructions are injected for the LLM but should not be shown in the UI. + * Only trims when a tag was actually removed to preserve user-intended formatting. + */ +function stripPlanModeTags(content: string): string { + // Remove <plan-mode>...</plan-mode> including any trailing whitespace + const stripped = content.replace(/<plan-mode>[\s\S]*?<\/plan-mode>\s*/g, ''); + // Only trim if a tag was actually removed + return stripped === content ? content : stripped.trim(); +} + +/** + * Format milliseconds into a compact human-readable string + * Examples: "1.2s", "1m 23s", "1h 2m" + */ +function formatDuration(ms: number): string { + const seconds = Math.floor(ms / 1000); + const tenths = Math.floor((ms % 1000) / 100); + + if (seconds < 60) { + return `${seconds}.${tenths}s`; + } + + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + + if (minutes < 60) { + return `${minutes}m ${remainingSeconds}s`; + } + + const hours = Math.floor(minutes / 60); + const remainingMinutes = minutes % 60; + return `${hours}h ${remainingMinutes}m`; +} + +interface MessageItemProps { + message: Message; + /** Terminal width for proper text wrapping calculations */ + terminalWidth?: number; +} + +/** + * Pure presentational component for a single message + * Visual hierarchy through colors and spacing only (no borders for easy text copying) + * + * Memoization with custom comparator prevents re-renders when message array changes + * but individual message content hasn't changed. + */ +export const MessageItem = memo( + ({ message, terminalWidth = 80 }: MessageItemProps) => { + // Check for styled message first + if (message.styledType && message.styledData) { + switch (message.styledType) { + case 'config': + return <ConfigBox data={message.styledData as ConfigStyledData} />; + case 'stats': + return <StatsBox data={message.styledData as StatsStyledData} />; + case 'help': + return <HelpBox data={message.styledData as HelpStyledData} />; + case 'session-list': + return <SessionListBox data={message.styledData as SessionListStyledData} />; + case 'session-history': + return ( + <SessionHistoryBox data={message.styledData as SessionHistoryStyledData} /> + ); + case 'log-config': + return <LogConfigBox data={message.styledData as LogConfigStyledData} />; + case 'run-summary': { + const data = message.styledData as RunSummaryStyledData; + const durationStr = formatDuration(data.durationMs); + // Only show tokens when >= 1000, using K notation + const tokensStr = + data.totalTokens >= 1000 + ? `, Used ${(data.totalTokens / 1000).toFixed(1)}K tokens` + : ''; + return ( + <Box marginTop={1} marginBottom={1} width={terminalWidth}> + <Text color="gray"> + ─ Worked for {durationStr} + {tokensStr} ─ + </Text> + </Box> + ); + } + case 'shortcuts': + return <ShortcutsBox data={message.styledData as ShortcutsStyledData} />; + case 'sysprompt': + return <SyspromptBox data={message.styledData as SysPromptStyledData} />; + } + } + + // User message: '>' prefix with gray background + // Strip plan-mode tags before display (plan instructions are for LLM, not user) + // Properly wrap text accounting for prefix "> " (2 chars) and paddingX={1} (2 chars total) + if (message.role === 'user') { + const prefix = '> '; + const paddingChars = 2; // paddingX={1} = 1 char on each side + const availableWidth = Math.max(20, terminalWidth - prefix.length - paddingChars); + const displayContent = stripPlanModeTags(message.content); + const wrappedContent = wrapAnsi(displayContent, availableWidth, { + hard: true, + wordWrap: true, + trim: false, + }); + const lines = wrappedContent.split('\n'); + + return ( + <Box flexDirection="column" marginTop={2} marginBottom={1} width={terminalWidth}> + <Box flexDirection="column" paddingX={1} backgroundColor="gray"> + {lines.map((line, i) => ( + <Box key={i} flexDirection="row"> + <Text color="green">{i === 0 ? prefix : ' '}</Text> + <Text color="white">{line}</Text> + </Box> + ))} + </Box> + </Box> + ); + } + + // Assistant message: Gray circle indicator (unless continuation) + // IMPORTANT: width="100%" is required to prevent Ink layout failures on large content. + // Without width constraints, streaming content causes terminal blackout at ~50+ lines. + // marginTop={1} for consistent spacing with tool messages + if (message.role === 'assistant') { + // Continuation messages: no indicator, just content + if (message.isContinuation) { + return ( + <Box flexDirection="column" width={terminalWidth}> + <MarkdownText>{message.content || ''}</MarkdownText> + </Box> + ); + } + + // Regular assistant message: bullet prefix inline with first line + // Text wraps at terminal width - wrapped lines may start at column 0 + // This is simpler and avoids mid-word splitting issues with Ink's wrap + return ( + <Box flexDirection="column" marginTop={1} width={terminalWidth}> + <MarkdownText bulletPrefix="⏺ ">{message.content || ''}</MarkdownText> + </Box> + ); + } + + // Tool message: Animated icon based on status + // - Running: green spinner + "Running..." + // - Finished (success): green dot + // - Finished (error): red dot + if (message.role === 'tool') { + // Use structured renderers if display data is available + const hasStructuredDisplay = message.toolDisplayData && message.toolContent; + const isRunning = message.toolStatus === 'running'; + const isPending = + message.toolStatus === 'pending' || message.toolStatus === 'pending_approval'; + + // Check for sub-agent progress data + const subAgentProgress = message.subAgentProgress; + + // Parse tool name and args for bold formatting: "ToolName(args)" → bold name + normal args + const parenIndex = message.content.indexOf('('); + const toolName = + parenIndex > 0 ? message.content.slice(0, parenIndex) : message.content; + const toolArgs = parenIndex > 0 ? message.content.slice(parenIndex) : ''; + + // Build the full tool header text for wrapping + // Don't include status suffix if we have sub-agent progress (it shows its own status) + const statusSuffix = subAgentProgress + ? '' + : isRunning + ? ' Running...' + : isPending + ? ' Waiting...' + : ''; + const fullToolText = `${toolName}${toolArgs}${statusSuffix}`; + + // ToolIcon takes 2 chars ("● "), so available width is terminalWidth - 2 + const iconWidth = 2; + const availableWidth = Math.max(20, terminalWidth - iconWidth); + const wrappedToolText = wrapAnsi(fullToolText, availableWidth, { + hard: true, + wordWrap: true, + trim: false, + }); + const toolLines = wrappedToolText.split('\n'); + + return ( + <Box flexDirection="column" marginTop={1} width={terminalWidth}> + {/* Tool header: icon + name + args + status text */} + {toolLines.map((line, i) => ( + <Box key={i} flexDirection="row"> + {i === 0 ? ( + <ToolIcon + status={message.toolStatus || 'finished'} + isError={message.isError ?? false} + /> + ) : ( + <Text>{' '}</Text> + )} + <Text> + {i === 0 ? ( + <> + <Text bold>{line.slice(0, toolName.length)}</Text> + <Text>{line.slice(toolName.length)}</Text> + </> + ) : ( + line + )} + </Text> + </Box> + ))} + {/* Sub-agent progress line - show when we have progress data */} + {subAgentProgress && isRunning && ( + <Box marginLeft={2}> + <Text color="gray"> + └─ {subAgentProgress.toolsCalled} tool + {subAgentProgress.toolsCalled !== 1 ? 's' : ''} called | Current:{' '} + {subAgentProgress.currentTool} + {subAgentProgress.tokenUsage && + subAgentProgress.tokenUsage.total > 0 + ? ` | ${subAgentProgress.tokenUsage.total.toLocaleString()} tokens` + : ''} + </Text> + </Box> + )} + {/* Tool result - only show when finished */} + {hasStructuredDisplay ? ( + <ToolResultRenderer + display={message.toolDisplayData!} + content={message.toolContent!} + /> + ) : ( + message.toolResult && ( + <Box flexDirection="column"> + <Text color="gray"> ⎿ {message.toolResult}</Text> + </Box> + ) + )} + </Box> + ); + } + + // System message: Compact gray text + return ( + <Box flexDirection="column" marginBottom={1} width={terminalWidth}> + <Text color="gray">{message.content}</Text> + </Box> + ); + }, + // Custom comparator: only re-render if message content actually changed + (prev, next) => { + return ( + prev.message.id === next.message.id && + prev.message.content === next.message.content && + prev.message.role === next.message.role && + prev.message.toolStatus === next.message.toolStatus && + prev.message.toolResult === next.message.toolResult && + prev.message.isStreaming === next.message.isStreaming && + prev.message.styledType === next.message.styledType && + prev.message.styledData === next.message.styledData && + prev.message.isContinuation === next.message.isContinuation && + prev.message.isError === next.message.isError && + prev.message.toolDisplayData === next.message.toolDisplayData && + prev.message.toolContent === next.message.toolContent && + prev.terminalWidth === next.terminalWidth && + prev.message.subAgentProgress?.toolsCalled === + next.message.subAgentProgress?.toolsCalled && + prev.message.subAgentProgress?.currentTool === + next.message.subAgentProgress?.currentTool && + prev.message.subAgentProgress?.tokenUsage?.total === + next.message.subAgentProgress?.tokenUsage?.total + ); + } +); + +MessageItem.displayName = 'MessageItem'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/MessageList.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/MessageList.tsx new file mode 100644 index 00000000..6ddaa691 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/MessageList.tsx @@ -0,0 +1,49 @@ +/** + * MessageList Component + * Displays a list of messages with optional welcome message + */ + +import React, { useMemo } from 'react'; +import { Box, Text } from 'ink'; +import { MessageItem } from './MessageItem.js'; +import type { Message } from '../../state/types.js'; + +interface MessageListProps { + messages: Message[]; + maxVisible?: number; +} + +/** + * Pure presentational component for message list + * Shows only recent messages for performance + */ +export function MessageList({ messages, maxVisible = 50 }: MessageListProps) { + // Only render recent messages for performance + const visibleMessages = useMemo(() => { + return messages.slice(-maxVisible); + }, [messages, maxVisible]); + + const hasMoreMessages = messages.length > maxVisible; + + return ( + <Box flexDirection="column" flexGrow={1} paddingX={1} paddingY={1}> + {hasMoreMessages && ( + <Box marginBottom={1}> + <Text color="gray"> + ... ({messages.length - maxVisible} earlier messages hidden) + </Text> + </Box> + )} + {visibleMessages.length === 0 && ( + <Box marginY={2}> + <Text color="greenBright"> + Welcome to Dexto CLI! Type your message below or use /help for commands. + </Text> + </Box> + )} + {visibleMessages.map((msg) => ( + <MessageItem key={msg.id} message={msg} /> + ))} + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/QueuedMessagesDisplay.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/QueuedMessagesDisplay.tsx new file mode 100644 index 00000000..cf7ccc69 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/QueuedMessagesDisplay.tsx @@ -0,0 +1,68 @@ +/** + * QueuedMessagesDisplay - Shows queued messages waiting to be processed + * + * Similar to webui's QueuedMessagesDisplay.tsx but for Ink/terminal. + * Shows: + * - Count of queued messages + * - Keyboard hint (↑ to edit) + * - Truncated preview of each queued message + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { QueuedMessage, ContentPart } from '@dexto/core'; + +interface QueuedMessagesDisplayProps { + messages: QueuedMessage[]; +} + +/** + * Extract text content from ContentPart[] + */ +function getMessageText(content: ContentPart[]): string { + const textParts = content + .filter((part): part is { type: 'text'; text: string } => part.type === 'text') + .map((part) => part.text); + return textParts.join(' ') || '[attachment]'; +} + +/** + * Truncate text to fit terminal width + */ +function truncateText(text: string, maxLength: number = 60): string { + // Replace newlines with spaces for single-line display + const singleLine = text.replace(/\n/g, ' ').trim(); + if (singleLine.length <= maxLength) { + return singleLine; + } + return singleLine.slice(0, maxLength - 3) + '...'; +} + +export function QueuedMessagesDisplay({ messages }: QueuedMessagesDisplayProps) { + if (messages.length === 0) return null; + + return ( + <Box flexDirection="column" marginBottom={1}> + {/* Header with count and keyboard hint */} + <Box> + <Text color="gray"> + {messages.length} message{messages.length !== 1 ? 's' : ''} queued + </Text> + <Text color="gray"> • </Text> + <Text color="gray">↑ to edit</Text> + </Box> + + {/* Messages list */} + {messages.map((message, index) => ( + <Box key={message.id} flexDirection="row"> + {/* Arrow indicator - last message gets special indicator */} + <Text color="gray">{index === messages.length - 1 ? '↳ ' : '│ '}</Text> + {/* Message preview */} + <Text color="gray" italic> + {truncateText(getMessageText(message.content))} + </Text> + </Box> + ))} + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/ToolIcon.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/ToolIcon.tsx new file mode 100644 index 00000000..ae4dcf72 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/ToolIcon.tsx @@ -0,0 +1,77 @@ +/** + * ToolIcon Component + * Animated icon for tool calls with status-based visual feedback + */ + +import { useState, useEffect } from 'react'; +import { Text } from 'ink'; +import type { ToolStatus } from '../../state/types.js'; + +interface ToolIconProps { + status: ToolStatus; + isError?: boolean; +} + +// Spinner frames for running animation +const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; + +/** + * Animated tool icon that changes based on execution status + * - Running: Animated spinner (green/teal) + * - Finished (success): Green dot + * - Finished (error): Red dot + */ +export function ToolIcon({ status, isError }: ToolIconProps) { + const [frame, setFrame] = useState(0); + + // Animate spinner only when actually running (not during approval) + useEffect(() => { + if (status !== 'running') { + return; + } + + const interval = setInterval(() => { + setFrame((prev) => (prev + 1) % SPINNER_FRAMES.length); + }, 80); // 80ms per frame for smooth animation + + return () => clearInterval(interval); + }, [status]); + + // Pending: static gray dot (tool call received, checking approval) + if (status === 'pending') { + return <Text color="gray">● </Text>; + } + + // Pending approval: static yellowBright dot (waiting for user) + if (status === 'pending_approval') { + return ( + <Text color="yellowBright" bold> + ●{' '} + </Text> + ); + } + + if (status === 'finished') { + // Error state: red dot + if (isError) { + return ( + <Text color="red" bold> + ●{' '} + </Text> + ); + } + // Success state: green dot + return ( + <Text color="green" bold> + ●{' '} + </Text> + ); + } + + // Running state with spinner + return ( + <Text color="green" bold> + {SPINNER_FRAMES[frame]}{' '} + </Text> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/index.ts b/dexto/packages/cli/src/cli/ink-cli/components/chat/index.ts new file mode 100644 index 00000000..fb2f4126 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/index.ts @@ -0,0 +1,10 @@ +/** + * Chat components module exports + */ + +export { Header } from './Header.js'; +export { Footer } from './Footer.js'; +export { MessageItem } from './MessageItem.js'; +export { MessageList } from './MessageList.js'; +export { ChatView } from './ChatView.js'; +export { ToolIcon } from './ToolIcon.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/ConfigBox.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/ConfigBox.tsx new file mode 100644 index 00000000..923d7ec4 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/ConfigBox.tsx @@ -0,0 +1,81 @@ +/** + * ConfigBox - Styled output for /config command + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { ConfigStyledData } from '../../../state/types.js'; +import { StyledBox, StyledSection, StyledRow } from './StyledBox.js'; + +interface ConfigBoxProps { + data: ConfigStyledData; +} + +export function ConfigBox({ data }: ConfigBoxProps) { + return ( + <StyledBox title="Runtime Configuration" titleColor="cyan"> + {/* Config file path at the top */} + {data.configFilePath && ( + <Box> + <Text color="gray">Agent config: </Text> + <Text color="blue">{data.configFilePath}</Text> + </Box> + )} + + <StyledSection title="LLM"> + <StyledRow label="Provider" value={data.provider} /> + <StyledRow label="Model" value={data.model} /> + {data.maxTokens !== null && ( + <StyledRow label="Max Tokens" value={data.maxTokens.toString()} /> + )} + {data.temperature !== null && ( + <StyledRow label="Temperature" value={data.temperature.toString()} /> + )} + </StyledSection> + + <StyledSection title="Tool Confirmation"> + <StyledRow label="Mode" value={data.toolConfirmationMode} /> + </StyledSection> + + <StyledSection title="Sessions"> + <StyledRow label="Max Sessions" value={data.maxSessions} /> + <StyledRow label="Session TTL" value={data.sessionTTL} /> + </StyledSection> + + <StyledSection title="MCP Servers"> + {data.mcpServers.length > 0 ? ( + data.mcpServers.map((server) => ( + <Box key={server}> + <Text color="cyan">{server}</Text> + </Box> + )) + ) : ( + <Text color="gray">No MCP servers configured</Text> + )} + </StyledSection> + + {data.promptsCount > 0 && ( + <StyledSection title="Prompts"> + <Text color="gray">{data.promptsCount} prompt(s) configured</Text> + </StyledSection> + )} + + {data.pluginsEnabled.length > 0 && ( + <StyledSection title="Plugins"> + {data.pluginsEnabled.map((plugin) => ( + <Box key={plugin}> + <Text color="green">{plugin}</Text> + </Box> + ))} + </StyledSection> + )} + + {/* Footer note about CLI-populated fields */} + <Box marginTop={1}> + <Text color="gray" italic> + Note: Some fields (logs, database paths) are auto-populated by the CLI. + </Text> + </Box> + </StyledBox> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/HelpBox.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/HelpBox.tsx new file mode 100644 index 00000000..18146758 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/HelpBox.tsx @@ -0,0 +1,47 @@ +/** + * HelpBox - Styled output for /help command + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { HelpStyledData } from '../../../state/types.js'; +import { StyledBox } from './StyledBox.js'; + +interface HelpBoxProps { + data: HelpStyledData; +} + +export function HelpBox({ data }: HelpBoxProps) { + // Group commands by category + const categories = data.commands.reduce( + (acc, cmd) => { + const cat = cmd.category || 'Other'; + if (!acc[cat]) { + acc[cat] = []; + } + acc[cat].push(cmd); + return acc; + }, + {} as Record<string, typeof data.commands> + ); + + return ( + <StyledBox title="Available Commands"> + {Object.entries(categories).map(([category, commands]) => ( + <Box key={category} flexDirection="column" marginTop={1}> + <Text bold color="gray"> + {category} + </Text> + {commands.map((cmd) => ( + <Box key={cmd.name} marginLeft={2}> + <Box width={16}> + <Text color="cyan">/{cmd.name}</Text> + </Box> + <Text color="gray">{cmd.description}</Text> + </Box> + ))} + </Box> + ))} + </StyledBox> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/LogConfigBox.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/LogConfigBox.tsx new file mode 100644 index 00000000..277d8b2c --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/LogConfigBox.tsx @@ -0,0 +1,44 @@ +/** + * LogConfigBox - Styled output for /log command (no args) + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { LogConfigStyledData } from '../../../state/types.js'; +import { StyledBox, StyledRow, StyledListItem } from './StyledBox.js'; + +interface LogConfigBoxProps { + data: LogConfigStyledData; +} + +export function LogConfigBox({ data }: LogConfigBoxProps) { + return ( + <StyledBox title="Logging Configuration"> + <Box marginTop={1} flexDirection="column"> + <StyledRow label="Current level" value={data.currentLevel} valueColor="green" /> + {data.logFile && process.env.DEXTO_DEV_MODE === 'true' && ( + <StyledRow label="Log file" value={data.logFile} /> + )} + </Box> + + <Box marginTop={1} flexDirection="column"> + <Text color="gray">Available levels (least to most verbose):</Text> + {data.availableLevels.map((level) => { + const isCurrent = level === data.currentLevel; + return ( + <StyledListItem + key={level} + icon={isCurrent ? '>' : ' '} + text={level} + isActive={isCurrent} + /> + ); + })} + </Box> + + <Box marginTop={1}> + <Text color="gray">Use /log <level> to change level</Text> + </Box> + </StyledBox> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/SessionHistoryBox.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/SessionHistoryBox.tsx new file mode 100644 index 00000000..3bb78883 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/SessionHistoryBox.tsx @@ -0,0 +1,75 @@ +/** + * SessionHistoryBox - Styled output for /session history command + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { SessionHistoryStyledData } from '../../../state/types.js'; +import { StyledBox } from './StyledBox.js'; + +interface SessionHistoryBoxProps { + data: SessionHistoryStyledData; +} + +/** + * Truncate content to a reasonable preview length + */ +function truncateContent(content: string, maxLength: number = 100): string { + if (content.length <= maxLength) return content; + return content.slice(0, maxLength) + '...'; +} + +/** + * Get role color and icon + */ +function getRoleStyle(role: string): { color: string; icon: string } { + switch (role) { + case 'user': + return { color: 'blue', icon: '>' }; + case 'assistant': + return { color: 'green', icon: '|' }; + case 'system': + return { color: 'orange', icon: '#' }; + case 'tool': + return { color: 'green', icon: '*' }; + default: + return { color: 'white', icon: '-' }; + } +} + +export function SessionHistoryBox({ data }: SessionHistoryBoxProps) { + if (data.messages.length === 0) { + return ( + <StyledBox title={`Session History: ${data.sessionId.slice(0, 8)}`}> + <Box marginTop={1}> + <Text color="gray">No messages in this session yet.</Text> + </Box> + </StyledBox> + ); + } + + return ( + <StyledBox title={`Session History: ${data.sessionId.slice(0, 8)}`}> + {data.messages.map((msg, index) => { + const style = getRoleStyle(msg.role); + return ( + <Box key={index} flexDirection="column" marginTop={index === 0 ? 1 : 0}> + <Box> + <Text color={style.color} bold> + {style.icon}{' '} + </Text> + <Text color={style.color}>[{msg.role}]</Text> + <Text color="gray"> {msg.timestamp}</Text> + </Box> + <Box marginLeft={2}> + <Text>{truncateContent(msg.content)}</Text> + </Box> + </Box> + ); + })} + <Box marginTop={1}> + <Text color="gray">Total: {data.total} messages</Text> + </Box> + </StyledBox> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/SessionListBox.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/SessionListBox.tsx new file mode 100644 index 00000000..d3a6a413 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/SessionListBox.tsx @@ -0,0 +1,51 @@ +/** + * SessionListBox - Styled output for /session list command + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { SessionListStyledData } from '../../../state/types.js'; +import { StyledBox } from './StyledBox.js'; + +interface SessionListBoxProps { + data: SessionListStyledData; +} + +export function SessionListBox({ data }: SessionListBoxProps) { + if (data.sessions.length === 0) { + return ( + <StyledBox title="Sessions"> + <Box marginTop={1}> + <Text color="gray">No sessions found.</Text> + </Box> + <Box marginTop={1}> + <Text color="gray">Run `dexto` to start a new session.</Text> + </Box> + </StyledBox> + ); + } + + return ( + <StyledBox title="Sessions"> + {data.sessions.map((session) => ( + <Box key={session.id} marginTop={1}> + <Box width={12}> + <Text color={session.isCurrent ? 'green' : 'cyan'} bold={session.isCurrent}> + {session.isCurrent ? '>' : ' '} {session.id.slice(0, 8)} + </Text> + </Box> + <Box width={14}> + <Text color="gray">{session.messageCount} messages</Text> + </Box> + <Text color="gray">{session.lastActive}</Text> + </Box> + ))} + <Box marginTop={1}> + <Text color="gray">Total: {data.total} sessions</Text> + </Box> + <Box> + <Text color="gray">Use /resume to switch sessions</Text> + </Box> + </StyledBox> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/ShortcutsBox.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/ShortcutsBox.tsx new file mode 100644 index 00000000..d3a9a9bd --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/ShortcutsBox.tsx @@ -0,0 +1,34 @@ +/** + * ShortcutsBox - Styled output for /shortcuts command + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { ShortcutsStyledData } from '../../../state/types.js'; +import { StyledBox } from './StyledBox.js'; + +interface ShortcutsBoxProps { + data: ShortcutsStyledData; +} + +export function ShortcutsBox({ data }: ShortcutsBoxProps) { + return ( + <StyledBox title="Keyboard Shortcuts"> + {data.categories.map((category, catIndex) => ( + <Box key={category.name} flexDirection="column" marginTop={catIndex === 0 ? 0 : 1}> + <Text bold color="cyan"> + {category.name} + </Text> + {category.shortcuts.map((shortcut) => ( + <Box key={shortcut.keys} marginLeft={2}> + <Box width={16}> + <Text color="cyan">{shortcut.keys}</Text> + </Box> + <Text color="gray">{shortcut.description}</Text> + </Box> + ))} + </Box> + ))} + </StyledBox> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/StatsBox.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/StatsBox.tsx new file mode 100644 index 00000000..4c2e0bed --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/StatsBox.tsx @@ -0,0 +1,106 @@ +/** + * StatsBox - Styled output for /stats command + */ + +import React from 'react'; +import type { StatsStyledData } from '../../../state/types.js'; +import { StyledBox, StyledSection, StyledRow } from './StyledBox.js'; + +interface StatsBoxProps { + data: StatsStyledData; +} + +/** + * Format a number with K/M suffixes for compact display + */ +function formatTokenCount(count: number): string { + if (count >= 1_000_000) { + return `${(count / 1_000_000).toFixed(1)}M`; + } + if (count >= 1_000) { + return `${(count / 1_000).toFixed(1)}K`; + } + return count.toString(); +} + +/** + * Format cost in USD with appropriate precision + */ +function formatCost(cost: number): string { + if (cost < 0.01) { + return `$${cost.toFixed(4)}`; + } + if (cost < 1) { + return `$${cost.toFixed(3)}`; + } + return `$${cost.toFixed(2)}`; +} + +export function StatsBox({ data }: StatsBoxProps) { + return ( + <StyledBox title="System Statistics"> + <StyledSection title="Sessions"> + <StyledRow label="Total Sessions" value={data.sessions.total.toString()} /> + <StyledRow label="In Memory" value={data.sessions.inMemory.toString()} /> + <StyledRow label="Max Allowed" value={data.sessions.maxAllowed.toString()} /> + </StyledSection> + + <StyledSection title="MCP Servers"> + <StyledRow + label="Connected" + value={data.mcp.connected.toString()} + valueColor="green" + /> + {data.mcp.failed > 0 && ( + <StyledRow label="Failed" value={data.mcp.failed.toString()} valueColor="red" /> + )} + <StyledRow label="Available Tools" value={data.mcp.toolCount.toString()} /> + </StyledSection> + + {data.tokenUsage && ( + <StyledSection title="Token Usage (This Session)"> + <StyledRow + label="Input" + value={formatTokenCount(data.tokenUsage.inputTokens)} + /> + <StyledRow + label="Output" + value={formatTokenCount(data.tokenUsage.outputTokens)} + /> + {data.tokenUsage.reasoningTokens > 0 && ( + <StyledRow + label="Reasoning" + value={formatTokenCount(data.tokenUsage.reasoningTokens)} + /> + )} + {data.tokenUsage.cacheReadTokens > 0 && ( + <StyledRow + label="Cache Read" + value={formatTokenCount(data.tokenUsage.cacheReadTokens)} + valueColor="cyan" + /> + )} + {data.tokenUsage.cacheWriteTokens > 0 && ( + <StyledRow + label="Cache Write" + value={formatTokenCount(data.tokenUsage.cacheWriteTokens)} + valueColor="orange" + /> + )} + <StyledRow + label="Total" + value={formatTokenCount(data.tokenUsage.totalTokens)} + valueColor="blue" + /> + {data.estimatedCost !== undefined && ( + <StyledRow + label="Est. Cost" + value={formatCost(data.estimatedCost)} + valueColor="green" + /> + )} + </StyledSection> + )} + </StyledBox> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/StyledBox.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/StyledBox.tsx new file mode 100644 index 00000000..e6fc96a1 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/StyledBox.tsx @@ -0,0 +1,114 @@ +/** + * StyledBox - Base component for styled command output + * Provides consistent box styling for structured output + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import { useTerminalSize } from '../../../hooks/useTerminalSize.js'; + +interface StyledBoxProps { + title: string; + titleColor?: string; + children: React.ReactNode; +} + +/** + * Base styled box component with rounded border and title + * Automatically adjusts width to terminal size + */ +export function StyledBox({ title, titleColor = 'cyan', children }: StyledBoxProps) { + const { columns } = useTerminalSize(); + + return ( + <Box flexDirection="column" marginBottom={1} width={columns}> + <Box + flexDirection="column" + borderStyle="round" + borderColor="gray" + paddingX={1} + paddingY={0} + > + {/* Header */} + <Box marginBottom={0}> + <Text bold color={titleColor}> + {title} + </Text> + </Box> + + {/* Content */} + {children} + </Box> + </Box> + ); +} + +interface StyledSectionProps { + title: string; + icon?: string; + children: React.ReactNode; +} + +/** + * Section within a styled box + */ +export function StyledSection({ title, icon, children }: StyledSectionProps) { + return ( + <Box flexDirection="column" marginTop={1}> + <Text bold> + {icon && `${icon} `} + {title} + </Text> + <Box flexDirection="column" marginLeft={2}> + {children} + </Box> + </Box> + ); +} + +interface StyledRowProps { + label: string; + value: string; + valueColor?: string; +} + +/** + * Key-value row within a section + */ +export function StyledRow({ label, value, valueColor = 'cyan' }: StyledRowProps) { + return ( + <Box> + <Text color="gray">{label}: </Text> + <Text color={valueColor}>{value}</Text> + </Box> + ); +} + +interface StyledListItemProps { + icon?: string; + text: string; + isActive?: boolean; + dimmed?: boolean; +} + +/** + * List item with optional icon and active state + */ +export function StyledListItem({ icon, text, isActive, dimmed }: StyledListItemProps) { + // Build props object conditionally to avoid undefined with exactOptionalPropertyTypes + const textProps: Record<string, unknown> = {}; + if (isActive) { + textProps.color = 'green'; + textProps.bold = true; + } + if (dimmed) { + textProps.color = 'gray'; + } + + return ( + <Box> + {icon && <Text {...textProps}>{icon} </Text>} + <Text {...textProps}>{text}</Text> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/SyspromptBox.tsx b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/SyspromptBox.tsx new file mode 100644 index 00000000..bbdcb201 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/SyspromptBox.tsx @@ -0,0 +1,22 @@ +/** + * SyspromptBox - Styled output for /sysprompt command + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { SysPromptStyledData } from '../../../state/types.js'; +import { StyledBox } from './StyledBox.js'; + +interface SyspromptBoxProps { + data: SysPromptStyledData; +} + +export function SyspromptBox({ data }: SyspromptBoxProps) { + return ( + <StyledBox title="System Prompt" titleColor="green"> + <Box marginTop={1} flexDirection="column"> + <Text>{data.content}</Text> + </Box> + </StyledBox> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/index.ts b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/index.ts new file mode 100644 index 00000000..c4a9a266 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/chat/styled-boxes/index.ts @@ -0,0 +1,13 @@ +/** + * Styled boxes for command output + */ + +export { StyledBox, StyledSection, StyledRow, StyledListItem } from './StyledBox.js'; +export { ConfigBox } from './ConfigBox.js'; +export { StatsBox } from './StatsBox.js'; +export { HelpBox } from './HelpBox.js'; +export { SessionListBox } from './SessionListBox.js'; +export { SessionHistoryBox } from './SessionHistoryBox.js'; +export { LogConfigBox } from './LogConfigBox.js'; +export { ShortcutsBox } from './ShortcutsBox.js'; +export { SyspromptBox } from './SyspromptBox.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/input/InputArea.tsx b/dexto/packages/cli/src/cli/ink-cli/components/input/InputArea.tsx new file mode 100644 index 00000000..d8d1d9c4 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/input/InputArea.tsx @@ -0,0 +1,93 @@ +/** + * InputArea Component + * Wrapper around TextBufferInput - accepts buffer from parent + */ + +import React from 'react'; +import { Box } from 'ink'; +import { TextBufferInput, type OverlayTrigger } from '../TextBufferInput.js'; +import type { TextBuffer } from '../shared/text-buffer.js'; +import type { PendingImage, PastedBlock } from '../../state/types.js'; + +export type { OverlayTrigger }; + +interface InputAreaProps { + /** Text buffer (owned by parent) */ + buffer: TextBuffer; + /** Called when user submits */ + onSubmit: (value: string) => void; + /** Whether input is currently disabled */ + isDisabled: boolean; + /** Whether input should handle keypresses */ + isActive: boolean; + /** Placeholder text */ + placeholder?: string | undefined; + /** History navigation callback */ + onHistoryNavigate?: ((direction: 'up' | 'down') => void) | undefined; + /** Overlay trigger callback */ + onTriggerOverlay?: ((trigger: OverlayTrigger) => void) | undefined; + /** Keyboard scroll callback (for alternate buffer mode) */ + onKeyboardScroll?: ((direction: 'up' | 'down') => void) | undefined; + /** Current number of attached images (for placeholder numbering) */ + imageCount?: number | undefined; + /** Called when image is pasted from clipboard */ + onImagePaste?: ((image: PendingImage) => void) | undefined; + /** Current pending images (for placeholder removal detection) */ + images?: PendingImage[] | undefined; + /** Called when an image placeholder is removed from text */ + onImageRemove?: ((imageId: string) => void) | undefined; + /** Current pasted blocks for collapse/expand feature */ + pastedBlocks?: PastedBlock[] | undefined; + /** Called when a large paste is detected and should be collapsed */ + onPasteBlock?: ((block: PastedBlock) => void) | undefined; + /** Called to update a pasted block (e.g., toggle collapse) */ + onPasteBlockUpdate?: ((blockId: string, updates: Partial<PastedBlock>) => void) | undefined; + /** Called when a paste block placeholder is removed from text */ + onPasteBlockRemove?: ((blockId: string) => void) | undefined; + /** Query to highlight in input text (for history search) */ + highlightQuery?: string | undefined; +} + +export function InputArea({ + buffer, + onSubmit, + isDisabled, + isActive, + placeholder, + onHistoryNavigate, + onTriggerOverlay, + onKeyboardScroll, + imageCount, + onImagePaste, + images, + onImageRemove, + pastedBlocks, + onPasteBlock, + onPasteBlockUpdate, + onPasteBlockRemove, + highlightQuery, +}: InputAreaProps) { + return ( + <Box flexDirection="column"> + <TextBufferInput + buffer={buffer} + onSubmit={onSubmit} + placeholder={placeholder} + isDisabled={isDisabled} + isActive={isActive} + onHistoryNavigate={onHistoryNavigate} + onTriggerOverlay={onTriggerOverlay} + onKeyboardScroll={onKeyboardScroll} + imageCount={imageCount} + onImagePaste={onImagePaste} + images={images} + onImageRemove={onImageRemove} + pastedBlocks={pastedBlocks} + onPasteBlock={onPasteBlock} + onPasteBlockUpdate={onPasteBlockUpdate} + onPasteBlockRemove={onPasteBlockRemove} + highlightQuery={highlightQuery} + /> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/input/index.ts b/dexto/packages/cli/src/cli/ink-cli/components/input/index.ts new file mode 100644 index 00000000..de9c8a0f --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/input/index.ts @@ -0,0 +1,5 @@ +/** + * Input components module exports + */ + +export { InputArea } from './InputArea.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/modes/AlternateBufferCLI.tsx b/dexto/packages/cli/src/cli/ink-cli/components/modes/AlternateBufferCLI.tsx new file mode 100644 index 00000000..b1628ee5 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/modes/AlternateBufferCLI.tsx @@ -0,0 +1,395 @@ +/** + * AlternateBufferCLI - VirtualizedList rendering mode + * + * Uses the terminal's alternate buffer for a fullscreen, scrollable UI. + * Features: + * - VirtualizedList for efficient message rendering + * - Mouse scroll support via ScrollProvider + * - Keyboard scroll (PageUp/PageDown, Shift+Arrow) + * - Copy mode toggle (Ctrl+S) + * - Selection hint when user tries to drag without Option key + */ + +import React, { useMemo, useCallback, useRef, useState, useEffect } from 'react'; +import { Box, Text, type DOMElement } from 'ink'; +import type { DextoAgent } from '@dexto/core'; + +// Types +import type { Message, StartupInfo } from '../../state/types.js'; + +// Hooks +import { useTerminalSize } from '../../hooks/index.js'; +import { useCLIState } from '../../hooks/useCLIState.js'; +import { useScrollable } from '../../contexts/index.js'; + +// Components +import { Header } from '../chat/Header.js'; +import { MessageItem } from '../chat/MessageItem.js'; +import { QueuedMessagesDisplay } from '../chat/QueuedMessagesDisplay.js'; +import { StatusBar } from '../StatusBar.js'; +import { HistorySearchBar } from '../HistorySearchBar.js'; +import { Footer } from '../Footer.js'; +import { TodoPanel } from '../TodoPanel.js'; +import { + VirtualizedList, + SCROLL_TO_ITEM_END, + type VirtualizedListRef, +} from '../shared/VirtualizedList.js'; + +// Containers +import { InputContainer, type InputContainerHandle } from '../../containers/InputContainer.js'; +import { OverlayContainer } from '../../containers/OverlayContainer.js'; + +// Union type for virtualized list items: header or message +type ListItem = { type: 'header' } | { type: 'message'; message: Message }; + +interface AlternateBufferCLIProps { + agent: DextoAgent; + initialSessionId: string | null; + startupInfo: StartupInfo; + /** Callback when user attempts to select text (drag without Option key) */ + onSelectionAttempt?: () => void; + /** Whether to stream chunks or wait for complete response */ + useStreaming?: boolean; +} + +export function AlternateBufferCLI({ + agent, + initialSessionId, + startupInfo, + onSelectionAttempt, + useStreaming = true, +}: AlternateBufferCLIProps) { + // Refs for VirtualizedList + const listRef = useRef<VirtualizedListRef<ListItem>>(null); + const listContainerRef = useRef<DOMElement>(null); + + // Ref to InputContainer for programmatic submit + const inputContainerRef = useRef<InputContainerHandle>(null); + + // Selection hint state + const [selectionHintVisible, setSelectionHintVisible] = useState(false); + + // Keyboard scroll handler for VirtualizedList + const handleKeyboardScroll = useCallback((direction: 'up' | 'down') => { + const delta = direction === 'up' ? -10 : 10; + listRef.current?.scrollBy(delta); + }, []); + + // Use shared CLI state with keyboard scroll handler + const { + messages, + setMessages, + pendingMessages, + setPendingMessages, + dequeuedBuffer, + setDequeuedBuffer, + queuedMessages, + setQueuedMessages, + todos, + setTodos, + ui, + setUi, + input, + setInput, + session, + setSession, + approval, + setApproval, + approvalQueue, + setApprovalQueue, + inputService, + buffer, + overlayContainerRef, + visibleMessages, + } = useCLIState({ + agent, + initialSessionId, + startupInfo, + onKeyboardScroll: handleKeyboardScroll, + }); + + // Register the VirtualizedList as scrollable so ScrollProvider can handle mouse scroll + const getScrollState = useCallback(() => { + const scrollState = listRef.current?.getScrollState(); + return scrollState ?? { scrollTop: 0, scrollHeight: 0, innerHeight: 0 }; + }, []); + + const scrollBy = useCallback((delta: number) => { + listRef.current?.scrollBy(delta); + }, []); + + const hasFocus = useCallback(() => true, []); // List always has focus for scroll + + // Compute whether history search has a match (for HistorySearchBar indicator) + const historySearchHasMatch = useMemo(() => { + if (!ui.historySearch.isActive || !ui.historySearch.query) return false; + const query = ui.historySearch.query.toLowerCase(); + return input.history.some((item) => item.toLowerCase().includes(query)); + }, [ui.historySearch.isActive, ui.historySearch.query, input.history]); + + // Callback for OverlayContainer to submit prompt commands through InputContainer + const handleSubmitPromptCommand = useCallback( + async (commandText: string) => { + try { + await inputContainerRef.current?.submit(commandText); + } catch (error) { + agent.logger.error( + `AlternateBufferCLI.handleSubmitPromptCommand failed: ${error instanceof Error ? error.message : String(error)}` + ); + } + }, + [agent] + ); + + useScrollable( + { + ref: listContainerRef, + getScrollState, + scrollBy, + hasFocus, + }, + true // Always active in alternate buffer mode + ); + + // Handle selection attempt - show hint + const handleSelectionAttempt = useCallback(() => { + setSelectionHintVisible(true); + onSelectionAttempt?.(); + }, [onSelectionAttempt]); + + // Auto-hide selection hint after 3 seconds + useEffect(() => { + if (!selectionHintVisible) return; + const timer = setTimeout(() => { + setSelectionHintVisible(false); + }, 3000); + return () => clearTimeout(timer); + }, [selectionHintVisible]); + + // Get terminal dimensions - updates on resize + const { rows: terminalHeight, columns: terminalWidth } = useTerminalSize(); + + // Build list data: header as first item, then finalized + pending + dequeued buffer + // In alternate buffer mode, everything is re-rendered anyway, so we combine all + // Order: finalized messages → pending/streaming → dequeued user messages (guarantees order) + // IMPORTANT: Deduplicate by ID to prevent race condition where a message appears in both + // finalized (messages) and pending during the brief window between setState calls + const listData = useMemo<ListItem[]>(() => { + const items: ListItem[] = [{ type: 'header' }]; + const seenIds = new Set<string>(); + + for (const msg of visibleMessages) { + items.push({ type: 'message', message: msg }); + seenIds.add(msg.id); + } + // Add pending/streaming messages (skip if already in finalized - race condition guard) + for (const msg of pendingMessages) { + if (!seenIds.has(msg.id)) { + items.push({ type: 'message', message: msg }); + seenIds.add(msg.id); + } + } + // Add dequeued buffer (user messages waiting to be flushed to finalized) + // These render AFTER pending to guarantee correct visual order + for (const msg of dequeuedBuffer) { + if (!seenIds.has(msg.id)) { + items.push({ type: 'message', message: msg }); + } + } + return items; + }, [visibleMessages, pendingMessages, dequeuedBuffer]); + + // Render callback for VirtualizedList items + const renderListItem = useCallback( + ({ item }: { item: ListItem }) => { + if (item.type === 'header') { + return ( + <Header + modelName={session.modelName} + sessionId={session.id || undefined} + hasActiveSession={session.hasActiveSession} + startupInfo={startupInfo} + /> + ); + } + return <MessageItem message={item.message} terminalWidth={terminalWidth} />; + }, + [session.modelName, session.id, session.hasActiveSession, startupInfo, terminalWidth] + ); + + // Smart height estimation based on item type and content + const estimateItemHeight = useCallback( + (index: number) => { + const item = listData[index]; + if (!item) return 3; + + // Header is approximately 10 lines (logo + info) + if (item.type === 'header') { + return 10; + } + + const msg = item.message; + + // Tool messages with results are taller + if (msg.role === 'tool') { + if (msg.toolResult) { + const resultLines = Math.ceil(msg.toolResult.length / 80); + return Math.min(2 + resultLines, 10); + } + return 2; + } + + // User messages have margin and background + if (msg.role === 'user') { + const contentLines = Math.ceil(msg.content.length / 80); + return Math.max(3, contentLines + 2); + } + + // Assistant messages + if (msg.role === 'assistant') { + if (msg.isStreaming) return 5; + const contentLines = Math.ceil(msg.content.length / 80); + return Math.max(2, contentLines + 1); + } + + // System/styled messages + if (msg.styledType) { + return 8; + } + + return 3; + }, + [listData] + ); + + const getItemKey = useCallback((item: ListItem) => { + if (item.type === 'header') return 'header'; + return item.message.id; + }, []); + + return ( + <Box flexDirection="column" height={terminalHeight}> + {/* Content area - VirtualizedList */} + <Box ref={listContainerRef} flexGrow={1} flexShrink={1} minHeight={0}> + <VirtualizedList + ref={listRef} + data={listData} + renderItem={renderListItem} + estimatedItemHeight={estimateItemHeight} + keyExtractor={getItemKey} + initialScrollIndex={SCROLL_TO_ITEM_END} + initialScrollOffsetInIndex={SCROLL_TO_ITEM_END} + /> + </Box> + + {/* Controls area - fixed at bottom */} + <Box flexDirection="column" flexShrink={0}> + <StatusBar + agent={agent} + isProcessing={ui.isProcessing} + isThinking={ui.isThinking} + isCompacting={ui.isCompacting} + approvalQueueCount={approvalQueue.length} + copyModeEnabled={ui.copyModeEnabled} + isAwaitingApproval={approval !== null} + todoExpanded={ui.todoExpanded} + hasTodos={todos.some((t) => t.status !== 'completed')} + planModeActive={ui.planModeActive} + autoApproveEdits={ui.autoApproveEdits} + /> + + {/* Todo panel - shown below status bar */} + <TodoPanel + todos={todos} + isExpanded={ui.todoExpanded} + isProcessing={ui.isProcessing} + /> + + {/* Selection hint when user tries to select without Option key */} + {selectionHintVisible && ( + <Box paddingX={1}> + <Text color="yellowBright"> + 💡 Tip: Hold Option (⌥) and click to select text, or press Ctrl+S to + toggle copy mode + </Text> + </Box> + )} + + {/* Queued messages display (shows when messages are pending) */} + <QueuedMessagesDisplay messages={queuedMessages} /> + + <InputContainer + ref={inputContainerRef} + buffer={buffer} + input={input} + ui={ui} + session={session} + approval={approval} + queuedMessages={queuedMessages} + setInput={setInput} + setUi={setUi} + setSession={setSession} + setMessages={setMessages} + setPendingMessages={setPendingMessages} + setDequeuedBuffer={setDequeuedBuffer} + setQueuedMessages={setQueuedMessages} + setApproval={setApproval} + setApprovalQueue={setApprovalQueue} + setTodos={setTodos} + agent={agent} + inputService={inputService} + onKeyboardScroll={handleKeyboardScroll} + useStreaming={useStreaming} + /> + + <OverlayContainer + ref={overlayContainerRef} + ui={ui} + input={input} + session={session} + approval={approval} + setInput={setInput} + setUi={setUi} + setSession={setSession} + setMessages={setMessages} + setApproval={setApproval} + setApprovalQueue={setApprovalQueue} + agent={agent} + inputService={inputService} + buffer={buffer} + onSubmitPromptCommand={handleSubmitPromptCommand} + /> + + {/* Exit warning (Ctrl+C pressed once) - shown above footer */} + {ui.exitWarningShown && ( + <Box paddingX={1}> + <Text color="yellowBright" bold> + ⚠ Press Ctrl+C again to exit + </Text> + <Text color="gray"> (or press any key to cancel)</Text> + </Box> + )} + + {/* Footer status line */} + <Footer + agent={agent} + sessionId={session.id} + modelName={session.modelName} + cwd={process.cwd()} + autoApproveEdits={ui.autoApproveEdits} + planModeActive={ui.planModeActive} + isShellMode={buffer.text.startsWith('!')} + /> + + {/* History search bar (Ctrl+R) - shown at very bottom */} + {ui.historySearch.isActive && ( + <HistorySearchBar + query={ui.historySearch.query} + hasMatch={historySearchHasMatch} + /> + )} + </Box> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/modes/StaticCLI.tsx b/dexto/packages/cli/src/cli/ink-cli/components/modes/StaticCLI.tsx new file mode 100644 index 00000000..4836682c --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/modes/StaticCLI.tsx @@ -0,0 +1,288 @@ +/** + * StaticCLI - Static pattern rendering mode + * + * Uses Ink's Static component for copy-friendly terminal output. + * Features: + * - Static component for finalized messages (rendered to terminal scrollback) + * - Native terminal scrolling and text selection + * - No mouse event interception + * - Simpler, more compatible with traditional terminal workflows + * + * Architecture: + * - `messages` = finalized messages → rendered in <Static> (permanent output) + * - `pendingMessages` = streaming/in-progress → rendered dynamically (redrawn) + * This prevents duplicate output when streaming completes. + */ + +import React, { useMemo, useState, useEffect, useRef, useCallback } from 'react'; +import { Box, Static, Text, useStdout } from 'ink'; +import type { DextoAgent } from '@dexto/core'; + +// ANSI escape sequence to clear terminal (equivalent to ansiEscapes.clearTerminal) +const CLEAR_TERMINAL = '\x1B[2J\x1B[3J\x1B[H'; + +// Types +import type { StartupInfo } from '../../state/types.js'; + +// Hooks +import { useCLIState } from '../../hooks/useCLIState.js'; +import { useTerminalSize } from '../../hooks/useTerminalSize.js'; + +// Components +import { Header } from '../chat/Header.js'; +import { MessageItem } from '../chat/MessageItem.js'; +import { QueuedMessagesDisplay } from '../chat/QueuedMessagesDisplay.js'; +import { StatusBar } from '../StatusBar.js'; +import { HistorySearchBar } from '../HistorySearchBar.js'; +import { Footer } from '../Footer.js'; +import { TodoPanel } from '../TodoPanel.js'; + +// Containers +import { InputContainer, type InputContainerHandle } from '../../containers/InputContainer.js'; +import { OverlayContainer } from '../../containers/OverlayContainer.js'; + +interface StaticCLIProps { + agent: DextoAgent; + initialSessionId: string | null; + startupInfo: StartupInfo; + /** Whether to stream chunks or wait for complete response */ + useStreaming?: boolean; +} + +export function StaticCLI({ + agent, + initialSessionId, + startupInfo, + useStreaming = true, +}: StaticCLIProps) { + // Use shared CLI state (no keyboard scroll in Static mode) + const { + messages, + setMessages, + pendingMessages, + setPendingMessages, + dequeuedBuffer, + setDequeuedBuffer, + queuedMessages, + setQueuedMessages, + todos, + setTodos, + ui, + setUi, + input, + setInput, + session, + setSession, + approval, + setApproval, + approvalQueue, + setApprovalQueue, + inputService, + buffer, + overlayContainerRef, + visibleMessages, + } = useCLIState({ + agent, + initialSessionId, + startupInfo, + // No keyboard scroll handler - let terminal handle scrollback + }); + + // Terminal resize handling - clear and re-render Static content + const { write: stdoutWrite } = useStdout(); + const { columns: terminalWidth } = useTerminalSize(); + const [staticRemountKey, setStaticRemountKey] = useState(0); + const isInitialMount = useRef(true); + + // Ref to InputContainer for programmatic submit + const inputContainerRef = useRef<InputContainerHandle>(null); + + // Compute whether history search has a match (for HistorySearchBar indicator) + const historySearchHasMatch = useMemo(() => { + if (!ui.historySearch.isActive || !ui.historySearch.query) return false; + const query = ui.historySearch.query.toLowerCase(); + return input.history.some((item) => item.toLowerCase().includes(query)); + }, [ui.historySearch.isActive, ui.historySearch.query, input.history]); + + // Callback for OverlayContainer to submit prompt commands through InputContainer + const handleSubmitPromptCommand = useCallback( + async (commandText: string) => { + try { + await inputContainerRef.current?.submit(commandText); + } catch (error) { + agent.logger.error( + `StaticCLI.handleSubmitPromptCommand failed: ${error instanceof Error ? error.message : String(error)}` + ); + } + }, + [agent] + ); + + // Function to refresh static content (clear terminal and force re-render) + const refreshStatic = useCallback(() => { + stdoutWrite(CLEAR_TERMINAL); + setStaticRemountKey((prev) => prev + 1); + }, [stdoutWrite]); + + // Handle terminal resize - debounced refresh of static content + useEffect(() => { + // Skip initial mount to avoid unnecessary clear on startup + if (isInitialMount.current) { + isInitialMount.current = false; + return; + } + + // Debounce resize handling (300ms) + const handler = setTimeout(() => { + refreshStatic(); + }, 300); + + return () => { + clearTimeout(handler); + }; + }, [terminalWidth, refreshStatic]); + + // Pre-render static items as JSX elements (Gemini pattern) + // Header + finalized messages go in <Static> (rendered once, permanent) + const staticItems = useMemo(() => { + const items: React.ReactElement[] = [ + <Header + key="header" + modelName={session.modelName} + sessionId={session.id || undefined} + hasActiveSession={session.hasActiveSession} + startupInfo={startupInfo} + />, + ...visibleMessages.map((msg) => ( + <MessageItem key={msg.id} message={msg} terminalWidth={terminalWidth} /> + )), + ]; + return items; + }, [ + visibleMessages, + session.modelName, + session.id, + session.hasActiveSession, + startupInfo, + terminalWidth, + ]); + + return ( + <Box flexDirection="column" width={terminalWidth}> + {/* Static: header + finalized messages - rendered once to terminal scrollback */} + {/* Key changes on resize to force full re-render after terminal clear */} + <Static key={staticRemountKey} items={staticItems}> + {(item) => item} + </Static> + + {/* Dynamic: pending/streaming messages - re-rendered on updates */} + {pendingMessages.map((message) => ( + <MessageItem key={message.id} message={message} terminalWidth={terminalWidth} /> + ))} + + {/* Dequeued buffer: user messages waiting to be flushed to finalized */} + {/* Rendered AFTER pending to guarantee correct visual order */} + {dequeuedBuffer.map((message) => ( + <MessageItem key={message.id} message={message} terminalWidth={terminalWidth} /> + ))} + + {/* Controls area */} + <Box flexDirection="column" flexShrink={0}> + <StatusBar + agent={agent} + isProcessing={ui.isProcessing} + isThinking={ui.isThinking} + isCompacting={ui.isCompacting} + approvalQueueCount={approvalQueue.length} + copyModeEnabled={ui.copyModeEnabled} + isAwaitingApproval={approval !== null} + todoExpanded={ui.todoExpanded} + hasTodos={todos.some((t) => t.status !== 'completed')} + planModeActive={ui.planModeActive} + autoApproveEdits={ui.autoApproveEdits} + /> + + {/* Todo panel - shown below status bar */} + <TodoPanel + todos={todos} + isExpanded={ui.todoExpanded} + isProcessing={ui.isProcessing} + /> + + {/* Queued messages display (shows when messages are pending) */} + <QueuedMessagesDisplay messages={queuedMessages} /> + + <InputContainer + ref={inputContainerRef} + buffer={buffer} + input={input} + ui={ui} + session={session} + approval={approval} + queuedMessages={queuedMessages} + setInput={setInput} + setUi={setUi} + setSession={setSession} + setMessages={setMessages} + setPendingMessages={setPendingMessages} + setDequeuedBuffer={setDequeuedBuffer} + setQueuedMessages={setQueuedMessages} + setApproval={setApproval} + setApprovalQueue={setApprovalQueue} + setTodos={setTodos} + agent={agent} + inputService={inputService} + useStreaming={useStreaming} + /> + + <OverlayContainer + ref={overlayContainerRef} + ui={ui} + input={input} + session={session} + approval={approval} + setInput={setInput} + setUi={setUi} + setSession={setSession} + setMessages={setMessages} + setApproval={setApproval} + setApprovalQueue={setApprovalQueue} + agent={agent} + inputService={inputService} + buffer={buffer} + refreshStatic={refreshStatic} + onSubmitPromptCommand={handleSubmitPromptCommand} + /> + + {/* Exit warning (Ctrl+C pressed once) - shown above footer */} + {ui.exitWarningShown && ( + <Box paddingX={1}> + <Text color="yellowBright" bold> + ⚠ Press Ctrl+C again to exit + </Text> + <Text color="gray"> (or press any key to cancel)</Text> + </Box> + )} + + {/* Footer status line */} + <Footer + agent={agent} + sessionId={session.id} + modelName={session.modelName} + cwd={process.cwd()} + autoApproveEdits={ui.autoApproveEdits} + planModeActive={ui.planModeActive} + isShellMode={buffer.text.startsWith('!')} + /> + + {/* History search bar (Ctrl+R) - shown at very bottom */} + {ui.historySearch.isActive && ( + <HistorySearchBar + query={ui.historySearch.query} + hasMatch={historySearchHasMatch} + /> + )} + </Box> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/modes/index.ts b/dexto/packages/cli/src/cli/ink-cli/components/modes/index.ts new file mode 100644 index 00000000..40c7a31c --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/modes/index.ts @@ -0,0 +1,10 @@ +/** + * CLI Rendering Modes + * + * Two rendering modes are available: + * - AlternateBufferCLI: VirtualizedList with mouse scroll, keyboard scroll, copy mode + * - StaticCLI: Static pattern with native terminal scrollback and selection + */ + +export { AlternateBufferCLI } from './AlternateBufferCLI.js'; +export { StaticCLI } from './StaticCLI.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/ApiKeyInput.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ApiKeyInput.tsx new file mode 100644 index 00000000..5e7b64c8 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ApiKeyInput.tsx @@ -0,0 +1,194 @@ +/** + * ApiKeyInput Component + * Interactive overlay for entering API keys when switching to a provider without one + */ + +import React, { useState, useEffect, forwardRef, useImperativeHandle, useCallback } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { LLMProvider } from '@dexto/core'; +import { getPrimaryApiKeyEnvVar, saveProviderApiKey } from '@dexto/agent-management'; +import { applyLayeredEnvironmentLoading } from '../../../../utils/env.js'; +import { + getProviderDisplayName, + isValidApiKeyFormat, + getProviderInstructions, +} from '../../../utils/provider-setup.js'; + +export interface ApiKeyInputProps { + isVisible: boolean; + provider: LLMProvider; + onSaved: (meta: { provider: LLMProvider; envVar: string }) => void; + onClose: () => void; +} + +export interface ApiKeyInputHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * API key input overlay - prompts user for API key when switching to a provider + * that doesn't have a configured key + */ +const ApiKeyInput = forwardRef<ApiKeyInputHandle, ApiKeyInputProps>(function ApiKeyInput( + { isVisible, provider, onSaved, onClose }, + ref +) { + const [apiKey, setApiKey] = useState(''); + const [error, setError] = useState<string | null>(null); + const [isSaving, setIsSaving] = useState(false); + + // Reset when becoming visible or provider changes + useEffect(() => { + if (isVisible) { + setApiKey(''); + setError(null); + setIsSaving(false); + } + }, [isVisible, provider]); + + const handleSubmit = useCallback(async () => { + const trimmedKey = apiKey.trim(); + + // Validate + if (!trimmedKey) { + setError('API key is required'); + return; + } + + if (!isValidApiKeyFormat(trimmedKey, provider)) { + setError(`Invalid ${getProviderDisplayName(provider)} API key format`); + return; + } + + setError(null); + setIsSaving(true); + + try { + const meta = await saveProviderApiKey(provider, trimmedKey, process.cwd()); + + // Reload environment variables so the key is available + await applyLayeredEnvironmentLoading(); + + onSaved({ provider, envVar: meta.envVar }); + } catch (err) { + setError(`Failed to save: ${err instanceof Error ? err.message : String(err)}`); + setIsSaving(false); + } + }, [apiKey, provider, onSaved]); + + // Handle keyboard input + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible || isSaving) return false; + + // Escape to close + if (key.escape) { + onClose(); + return true; + } + + // Enter to submit + if (key.return) { + void handleSubmit(); + return true; + } + + // Backspace + if (key.backspace || key.delete) { + setApiKey((prev) => prev.slice(0, -1)); + setError(null); + return true; + } + + // Regular character input + if (input && !key.ctrl && !key.meta) { + setApiKey((prev) => prev + input); + setError(null); + return true; + } + + return false; + }, + }), + [isVisible, isSaving, onClose, handleSubmit] + ); + + if (!isVisible) return null; + + const providerName = getProviderDisplayName(provider); + const envVar = getPrimaryApiKeyEnvVar(provider); + const instructions = getProviderInstructions(provider); + + // Mask the API key for display (show first 4 and last 4 chars) + const maskedKey = + apiKey.length > 8 + ? `${apiKey.slice(0, 4)}${'*'.repeat(Math.min(apiKey.length - 8, 20))}${apiKey.slice(-4)}` + : '*'.repeat(apiKey.length); + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="cyan" + paddingX={1} + marginTop={1} + > + {/* Header */} + <Box marginBottom={1}> + <Text bold color="cyan"> + API Key Required for {providerName} + </Text> + </Box> + + {/* Instructions */} + {instructions && ( + <Box flexDirection="column" marginBottom={1}> + <Text color="gray">{instructions.content}</Text> + </Box> + )} + + {/* Env var hint */} + <Box marginBottom={1}> + <Text color="gray">This key will be saved to </Text> + <Text color="yellowBright">{envVar}</Text> + <Text color="gray"> in your .env file</Text> + </Box> + + {/* Input prompt */} + <Box flexDirection="column"> + <Text bold>Enter your {providerName} API key:</Text> + </Box> + + {/* Input field (masked) */} + <Box marginTop={1}> + <Text color="cyan">> </Text> + <Text>{maskedKey}</Text> + {!isSaving && <Text color="cyan">_</Text>} + </Box> + + {/* Saving indicator */} + {isSaving && ( + <Box marginTop={1}> + <Text color="yellowBright">Saving API key...</Text> + </Box> + )} + + {/* Error message */} + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + {/* Help text */} + <Box marginTop={1}> + <Text color="gray">Enter to save • Esc to cancel</Text> + </Box> + </Box> + ); +}); + +export default ApiKeyInput; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/ContextStatsOverlay.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ContextStatsOverlay.tsx new file mode 100644 index 00000000..f126eb79 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ContextStatsOverlay.tsx @@ -0,0 +1,542 @@ +/** + * ContextStatsOverlay Component + * Interactive overlay for viewing context window usage statistics + * Features: + * - Stacked colored progress bar showing breakdown + * - Navigate with arrow keys to highlight items + * - Press Enter to expand/collapse sections (e.g., Tools) + */ + +import React, { useState, useEffect, forwardRef, useImperativeHandle } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent } from '@dexto/core'; + +interface ContextStatsOverlayProps { + isVisible: boolean; + onClose: () => void; + agent: DextoAgent; + sessionId: string; +} + +export interface ContextStatsOverlayHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ContextStats { + estimatedTokens: number; + actualTokens: number | null; + /** Effective max context tokens (after applying maxContextTokens override and thresholdPercent) */ + maxContextTokens: number; + /** The model's raw context window before any config overrides */ + modelContextWindow: number; + /** Configured threshold percent (0.0-1.0), defaults to 1.0 */ + thresholdPercent: number; + usagePercent: number; + messageCount: number; + filteredMessageCount: number; + prunedToolCount: number; + hasSummary: boolean; + model: string; + modelDisplayName: string; + breakdown: { + systemPrompt: number; + tools: { + total: number; + perTool: Array<{ name: string; tokens: number }>; + }; + messages: number; + }; + /** Calculation basis showing how the estimate was computed */ + calculationBasis?: { + method: 'actuals' | 'estimate'; + lastInputTokens?: number; + lastOutputTokens?: number; + newMessagesEstimate?: number; + }; +} + +// Breakdown items that can be selected +type BreakdownItem = 'systemPrompt' | 'tools' | 'messages' | 'freeSpace' | 'autoCompactBuffer'; +const BREAKDOWN_ITEMS: BreakdownItem[] = [ + 'systemPrompt', + 'tools', + 'messages', + 'freeSpace', + 'autoCompactBuffer', +]; + +// Colors for each breakdown category +const ITEM_COLORS: Record<BreakdownItem, string> = { + systemPrompt: 'cyan', + tools: 'yellow', + messages: 'blue', + freeSpace: 'gray', + autoCompactBuffer: 'magenta', +}; + +/** + * Format token count for display (e.g., 1500 -> "1.5k") + */ +function formatTokens(tokens: number): string { + if (tokens >= 1000) { + return `${(tokens / 1000).toFixed(1)}k`; + } + return tokens.toLocaleString(); +} + +/** + * Create a stacked colored progress bar + * Returns an array of {char, color, item} segments to render + * + * The bar represents the full model context window: + * - Used segments (systemPrompt, tools, messages) - solid blocks + * - Free space - light blocks (available for use before threshold) + * - Threshold buffer - hatched blocks (reserved margin before compaction triggers) + */ +interface BarSegment { + char: string; + color: string; + width: number; + item: BreakdownItem; +} + +function createStackedBar( + breakdown: ContextStats['breakdown'], + maxContextTokens: number, + thresholdPercent: number, + totalWidth: number = 40 +): BarSegment[] { + const segments: BarSegment[] = []; + + // Calculate auto compact buffer (the reserved margin for early compaction) + // maxContextTokens already has thresholdPercent applied, so derive buffer as: + // buffer = maxContextTokens * (1 - thresholdPercent) / thresholdPercent + const autoCompactBuffer = + thresholdPercent > 0 && thresholdPercent < 1.0 + ? Math.floor((maxContextTokens * (1 - thresholdPercent)) / thresholdPercent) + : 0; + // Total space = effective limit + buffer + const totalTokenSpace = maxContextTokens + autoCompactBuffer; + + // Calculate widths for each segment (proportional to token count) + const usedTokens = breakdown.systemPrompt + breakdown.tools.total + breakdown.messages; + const freeTokens = Math.max(0, maxContextTokens - usedTokens); + + // Helper to calculate width (minimum 1 char if tokens > 0, proportional otherwise) + const getWidth = (tokens: number): number => { + if (tokens <= 0) return 0; + const proportional = Math.round((tokens / totalTokenSpace) * totalWidth); + return Math.max(1, proportional); + }; + + // Add used segments + const sysWidth = getWidth(breakdown.systemPrompt); + const toolsWidth = getWidth(breakdown.tools.total); + const msgsWidth = getWidth(breakdown.messages); + const freeWidth = getWidth(freeTokens); + const reservedWidth = getWidth(autoCompactBuffer); + + // Adjust to fit total width (take from free space) + const totalUsed = sysWidth + toolsWidth + msgsWidth + freeWidth + reservedWidth; + const adjustment = totalUsed - totalWidth; + + // Apply adjustment to free space (it's the most flexible) + const adjustedFreeWidth = Math.max(0, freeWidth - adjustment); + + if (sysWidth > 0) { + segments.push({ + char: '█', + color: ITEM_COLORS.systemPrompt, + width: sysWidth, + item: 'systemPrompt', + }); + } + if (toolsWidth > 0) { + segments.push({ char: '█', color: ITEM_COLORS.tools, width: toolsWidth, item: 'tools' }); + } + if (msgsWidth > 0) { + segments.push({ + char: '█', + color: ITEM_COLORS.messages, + width: msgsWidth, + item: 'messages', + }); + } + if (adjustedFreeWidth > 0) { + segments.push({ char: '░', color: 'gray', width: adjustedFreeWidth, item: 'freeSpace' }); + } + if (reservedWidth > 0) { + segments.push({ + char: '▒', + color: ITEM_COLORS.autoCompactBuffer, + width: reservedWidth, + item: 'autoCompactBuffer', + }); + } + + return segments; +} + +/** + * Context stats overlay with selectable breakdown items + */ +const ContextStatsOverlay = forwardRef<ContextStatsOverlayHandle, ContextStatsOverlayProps>( + function ContextStatsOverlay({ isVisible, onClose, agent, sessionId }, ref) { + const [stats, setStats] = useState<ContextStats | null>(null); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState<string | null>(null); + const [selectedIndex, setSelectedIndex] = useState(0); + const [expandedSections, setExpandedSections] = useState<Set<BreakdownItem>>(new Set()); + + // Fetch stats when overlay becomes visible + useEffect(() => { + if (!isVisible) { + setStats(null); + setError(null); + setSelectedIndex(0); + setExpandedSections(new Set()); + return; + } + + let cancelled = false; + setIsLoading(true); + + const fetchStats = async () => { + try { + const contextStats = await agent.getContextStats(sessionId); + if (!cancelled) { + setStats(contextStats); + setIsLoading(false); + } + } catch (err) { + if (!cancelled) { + setError(err instanceof Error ? err.message : String(err)); + setIsLoading(false); + } + } + }; + + fetchStats(); + + return () => { + cancelled = true; + }; + }, [isVisible, agent, sessionId]); + + // Handle keyboard input + useImperativeHandle( + ref, + () => ({ + handleInput: (_input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape or 'q' to close + if (key.escape || _input === 'q') { + onClose(); + return true; + } + + // Arrow keys for navigation + if (key.upArrow) { + setSelectedIndex((prev) => Math.max(0, prev - 1)); + return true; + } + if (key.downArrow) { + setSelectedIndex((prev) => Math.min(BREAKDOWN_ITEMS.length - 1, prev + 1)); + return true; + } + + // Enter to expand/collapse + if (key.return) { + const item = BREAKDOWN_ITEMS[selectedIndex]; + // Only tools is expandable for now + if (item === 'tools') { + setExpandedSections((prev) => { + const next = new Set(prev); + if (next.has(item)) { + next.delete(item); + } else { + next.add(item); + } + return next; + }); + return true; + } + } + + return false; + }, + }), + [isVisible, onClose, selectedIndex] + ); + + if (!isVisible) return null; + + // Loading state + if (isLoading) { + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="cyan" + paddingX={2} + paddingY={1} + > + <Text color="cyan" bold> + 📊 Context Usage + </Text> + <Text color="gray">Loading...</Text> + </Box> + ); + } + + // Error state + if (error) { + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="red" + paddingX={2} + paddingY={1} + > + <Text color="red" bold> + ❌ Error + </Text> + <Text color="gray">{error}</Text> + <Box marginTop={1}> + <Text color="gray" dimColor> + Press Esc to close + </Text> + </Box> + </Box> + ); + } + + if (!stats) return null; + + // Calculate auto compact buffer early so it's available for pct() + // maxContextTokens already has thresholdPercent applied, so we need to derive + // the buffer as: maxContextTokens * (1 - thresholdPercent) / thresholdPercent + const autoCompactBuffer = + stats.thresholdPercent > 0 && stats.thresholdPercent < 1.0 + ? Math.floor( + (stats.maxContextTokens * (1 - stats.thresholdPercent)) / + stats.thresholdPercent + ) + : 0; + + // Total token space = effective limit + buffer (matches the visual bar) + const totalTokenSpace = stats.maxContextTokens + autoCompactBuffer; + + // Calculate percentage helper (relative to total token space for bar consistency) + const pct = (tokens: number): string => { + const percent = + totalTokenSpace > 0 ? ((tokens / totalTokenSpace) * 100).toFixed(1) : '0.0'; + return `${percent}%`; + }; + + const usedTokens = stats.estimatedTokens + autoCompactBuffer; + const tokenDisplay = `~${formatTokens(usedTokens)}`; + + const isToolsExpanded = expandedSections.has('tools'); + + // Create stacked bar segments + // Uses maxContextTokens (effective limit) + autoCompactBuffer as the full bar + const barSegments = createStackedBar( + stats.breakdown, + stats.maxContextTokens, + stats.thresholdPercent + ); + + // Helper to render a breakdown row with colored indicator + const renderRow = ( + index: number, + item: BreakdownItem, + label: string, + tokens: number, + isLast: boolean, + expandable?: boolean + ) => { + const isSelected = selectedIndex === index; + const prefix = isLast ? '└─' : '├─'; + const expandIcon = expandable ? (isToolsExpanded ? '▼' : '▶') : ' '; + const itemColor = ITEM_COLORS[item]; + // Use different characters for different types + const indicator = item === 'freeSpace' ? '░' : item === 'autoCompactBuffer' ? '▒' : '█'; + + return ( + <Box key={item}> + <Text color={isSelected ? 'white' : 'gray'}>{prefix} </Text> + <Text color={itemColor} bold={isSelected}> + {indicator} + </Text> + <Text color={isSelected ? 'white' : 'gray'} bold={isSelected}> + {' '} + {expandIcon} {label}: {formatTokens(tokens)} ({pct(tokens)}) + </Text> + {isSelected && expandable && ( + <Text color="gray" dimColor> + {' '} + (Enter to {isToolsExpanded ? 'collapse' : 'expand'}) + </Text> + )} + </Box> + ); + }; + + // Calculate free space using the actual/estimated tokens + // maxContextTokens is already the effective limit (with threshold applied) + const freeTokens = Math.max(0, stats.maxContextTokens - stats.estimatedTokens); + + // Buffer percent for display (autoCompactBuffer already calculated above) + const bufferPercent = Math.round((1 - stats.thresholdPercent) * 100); + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="cyan" + paddingX={2} + paddingY={1} + > + {/* Header */} + <Box marginBottom={1}> + <Text color="cyan" bold> + 📊 Context Usage + </Text> + <Text color="gray"> - {stats.modelDisplayName}</Text> + </Box> + + {/* Arrow indicator above the bar */} + <Box> + {barSegments.map((segment, idx) => { + const isHighlighted = BREAKDOWN_ITEMS[selectedIndex] === segment.item; + return ( + <Text key={idx} color="white"> + {isHighlighted + ? '▼'.repeat(segment.width) + : ' '.repeat(segment.width)} + </Text> + ); + })} + </Box> + + {/* Stacked progress bar */} + <Box> + {barSegments.map((segment, idx) => ( + <Text key={idx} color={segment.color}> + {segment.char.repeat(segment.width)} + </Text> + ))} + </Box> + + {/* Token summary */} + <Box marginBottom={1}> + <Text color="gray"> + {tokenDisplay} / {formatTokens(totalTokenSpace)} tokens + </Text> + <Text color="gray"> • </Text> + <Text + color={ + stats.usagePercent > 80 + ? 'red' + : stats.usagePercent > 60 + ? 'yellow' + : 'green' + } + > + {stats.usagePercent}% used + </Text> + </Box> + + {/* Breakdown */} + <Box flexDirection="column"> + <Text color="white">Breakdown:</Text> + {renderRow( + 0, + 'systemPrompt', + 'System prompt', + stats.breakdown.systemPrompt, + false + )} + {renderRow( + 1, + 'tools', + `Tools (${stats.breakdown.tools.perTool.length})`, + stats.breakdown.tools.total, + false, + true + )} + + {/* Expanded tools list */} + {isToolsExpanded && ( + <Box flexDirection="column" marginLeft={4}> + {stats.breakdown.tools.perTool.length === 0 ? ( + <Text color="gray" dimColor> + No tools registered + </Text> + ) : ( + [...stats.breakdown.tools.perTool] + .sort((a, b) => b.tokens - a.tokens) + .map((tool, idx, arr) => ( + <Text key={tool.name} color="gray" dimColor> + {idx === arr.length - 1 ? '└─' : '├─'}{' '} + <Text color="yellow" dimColor> + {tool.name} + </Text> + : {formatTokens(tool.tokens)} ({pct(tool.tokens)}) + </Text> + )) + )} + </Box> + )} + + {renderRow(2, 'messages', 'Messages', stats.breakdown.messages, false)} + {renderRow(3, 'freeSpace', 'Free space', freeTokens, false)} + {renderRow( + 4, + 'autoCompactBuffer', + bufferPercent > 0 + ? `Auto compact buffer (${bufferPercent}%)` + : 'Auto compact buffer', + autoCompactBuffer, + true + )} + </Box> + + {/* Additional stats */} + <Box marginTop={1} flexDirection="column"> + <Text color="gray"> + Messages: {stats.filteredMessageCount} visible ({stats.messageCount} total) + </Text> + + {stats.prunedToolCount > 0 && ( + <Text color="yellow">🗑️ {stats.prunedToolCount} tool output(s) pruned</Text> + )} + + {stats.hasSummary && ( + <Text color="blue">📦 Context has been compacted (summary present)</Text> + )} + + {stats.usagePercent > 100 && ( + <Text color="yellow"> + 💡 Use /compact to manually compact, or send a message to trigger + auto-compaction + </Text> + )} + </Box> + + {/* Footer with controls */} + <Box marginTop={1}> + <Text color="gray" dimColor> + ↑↓: navigate | Enter: expand/collapse | Esc: close + </Text> + </Box> + </Box> + ); + } +); + +export default ContextStatsOverlay; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/CustomModelWizard.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/CustomModelWizard.tsx new file mode 100644 index 00000000..4ebcc937 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/CustomModelWizard.tsx @@ -0,0 +1,460 @@ +/** + * CustomModelWizard Component + * Multi-step wizard for adding custom models (openai-compatible, openrouter, litellm, glama, bedrock) + * + * Architecture: + * - Provider configs centralized in ./custom-model-wizard/provider-config.ts + * - Shared UI components in ./custom-model-wizard/shared/ + * - This file is the orchestrator - handles state, navigation, and keyboard input + */ + +import React, { + useState, + useEffect, + forwardRef, + useImperativeHandle, + useCallback, + useRef, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { + saveCustomModel, + deleteCustomModel, + CUSTOM_MODEL_PROVIDERS, + type CustomModel, + type CustomModelProvider, + saveProviderApiKey, + getProviderKeyStatus, + resolveApiKeyForProvider, + determineApiKeyStorage, +} from '@dexto/agent-management'; +import { logger, type LLMProvider } from '@dexto/core'; + +// Import from new modular architecture +import { + getProviderConfig, + getAvailableProviders, + runAsyncValidation, +} from './custom-model-wizard/provider-config.js'; +import { + ProviderSelector, + WizardStepInput, + SetupInfoBanner, + ApiKeyStep, +} from './custom-model-wizard/shared/index.js'; +import LocalModelWizard, { + type LocalModelWizardHandle, +} from './custom-model-wizard/LocalModelWizard.js'; + +interface CustomModelWizardProps { + isVisible: boolean; + onComplete: (model: CustomModel) => void; + onClose: () => void; + /** Optional model to edit - if provided, form will be pre-populated */ + initialModel?: CustomModel | null; +} + +export interface CustomModelWizardHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * Multi-step wizard for custom model configuration. + * Uses data-driven provider configs instead of scattered conditionals. + */ +const CustomModelWizard = forwardRef<CustomModelWizardHandle, CustomModelWizardProps>( + function CustomModelWizard({ isVisible, onComplete, onClose, initialModel }, ref) { + // Provider selection (step 0) then wizard steps + const [selectedProvider, setSelectedProvider] = useState<CustomModelProvider | null>(null); + const [providerIndex, setProviderIndex] = useState(0); + const [currentStep, setCurrentStep] = useState(0); + const [values, setValues] = useState<Record<string, string>>({}); + const [currentInput, setCurrentInput] = useState(''); + const [error, setError] = useState<string | null>(null); + const [isSaving, setIsSaving] = useState(false); + const [isValidating, setIsValidating] = useState(false); + // Track original name when editing (to handle renames) + const [originalName, setOriginalName] = useState<string | null>(null); + const isEditing = initialModel !== null && initialModel !== undefined; + + // Ref for LocalModelWizard (specialized wizard for 'local' provider) + const localModelWizardRef = useRef<LocalModelWizardHandle>(null); + + // Get provider config (data-driven, no conditionals) + const providerConfig = selectedProvider ? getProviderConfig(selectedProvider) : null; + const allWizardSteps = providerConfig?.steps ?? []; + + /** + * Get visible steps based on current values. + * Steps with a condition function are only shown if the condition returns true. + */ + const getVisibleSteps = useCallback( + (currentValues: Record<string, string>) => { + return allWizardSteps.filter( + (step) => !step.condition || step.condition(currentValues) + ); + }, + [allWizardSteps] + ); + + // Current visible steps based on accumulated values + const visibleSteps = getVisibleSteps(values); + const currentStepConfig = visibleSteps[currentStep]; + + // Reset when becoming visible + useEffect(() => { + if (isVisible) { + if (initialModel) { + // Editing mode - pre-populate from initialModel + const provider = initialModel.provider ?? 'openai-compatible'; + setSelectedProvider(provider); + setOriginalName(initialModel.name); + setValues({ + name: initialModel.name, + baseURL: initialModel.baseURL ?? '', + displayName: initialModel.displayName ?? '', + maxInputTokens: initialModel.maxInputTokens?.toString() ?? '', + apiKey: initialModel.apiKey ?? '', + }); + setCurrentStep(0); + setCurrentInput(initialModel.name); + setProviderIndex(CUSTOM_MODEL_PROVIDERS.indexOf(provider)); + } else { + // Adding mode - reset everything + setSelectedProvider(null); + setOriginalName(null); + setProviderIndex(0); + setCurrentStep(0); + setValues({}); + setCurrentInput(''); + } + setError(null); + setIsSaving(false); + setIsValidating(false); + } + }, [isVisible, initialModel]); + + const handleProviderSelect = useCallback(() => { + const provider = CUSTOM_MODEL_PROVIDERS[providerIndex]; + if (provider) { + setSelectedProvider(provider); + setCurrentStep(0); + setCurrentInput(''); + setError(null); + } + }, [providerIndex]); + + const handleNext = useCallback(async () => { + if (!currentStepConfig || !selectedProvider || isSaving || isValidating) return; + + const value = currentInput.trim(); + + // Sync validation + if (currentStepConfig.validate) { + const validationError = currentStepConfig.validate(value); + if (validationError) { + setError(validationError); + return; + } + } else if (currentStepConfig.required && !value) { + setError(`${currentStepConfig.label} is required`); + return; + } + + // Async validation (data-driven - no provider-specific conditionals) + const asyncError = await (async () => { + setIsValidating(true); + setError(null); + try { + return await runAsyncValidation( + selectedProvider, + currentStepConfig.field, + value + ); + } finally { + setIsValidating(false); + } + })(); + + if (asyncError) { + setError(asyncError); + return; + } + + // Save value + const newValues = { ...values, [currentStepConfig.field]: value }; + setValues(newValues); + setError(null); + setCurrentInput(''); + + // Get updated visible steps with new values + const updatedVisibleSteps = getVisibleSteps(newValues); + + // Check if we're done + if (currentStep >= updatedVisibleSteps.length - 1) { + await saveModel(newValues); + } else { + const nextStep = currentStep + 1; + setCurrentStep(nextStep); + // Pre-populate next step from stored values (for edit mode) + const nextStepConfig = updatedVisibleSteps[nextStep]; + const nextValue = nextStepConfig ? newValues[nextStepConfig.field] : undefined; + setCurrentInput(nextValue ?? ''); + } + }, [ + currentInput, + currentStep, + currentStepConfig, + getVisibleSteps, + isSaving, + isValidating, + selectedProvider, + values, + ]); + + /** + * Build and save the model using provider config's buildModel function. + */ + const saveModel = useCallback( + async (finalValues: Record<string, string>) => { + if (!selectedProvider || !providerConfig) return; + + // Build model using provider config (no conditionals!) + const model = providerConfig.buildModel(finalValues, selectedProvider); + + // Handle API key storage + const userEnteredKey = finalValues.apiKey?.trim(); + const providerKeyStatus = getProviderKeyStatus(selectedProvider as LLMProvider); + const existingProviderKey = resolveApiKeyForProvider( + selectedProvider as LLMProvider + ); + + const keyStorage = determineApiKeyStorage( + selectedProvider, + userEnteredKey, + providerKeyStatus.hasApiKey, + existingProviderKey + ); + + if (keyStorage.saveToProviderEnvVar && userEnteredKey) { + try { + await saveProviderApiKey( + selectedProvider as LLMProvider, + userEnteredKey, + process.cwd() + ); + } catch (err) { + logger.warn( + `Failed to save provider API key: ${err instanceof Error ? err.message : 'Unknown error'}` + ); + // Fall back to per-model storage + keyStorage.saveAsPerModel = true; + } + } + + if (keyStorage.saveAsPerModel && userEnteredKey) { + model.apiKey = userEnteredKey; + } + + // Save to storage + setIsSaving(true); + try { + // If editing and name changed, delete the old model first + if (originalName && originalName !== model.name) { + try { + await deleteCustomModel(originalName); + } catch (err) { + // Log but continue - old model might already be deleted + logger.warn( + `Failed to delete old model "${originalName}" during rename: ${err instanceof Error ? err.message : String(err)}` + ); + } + } + await saveCustomModel(model); + onComplete(model); + } catch (err) { + logger.error( + `Failed to save custom model: ${err instanceof Error ? err.message : 'Unknown error'}` + ); + setError( + `Failed to save: ${err instanceof Error ? err.message : 'Unknown error'}` + ); + setIsSaving(false); + } + }, + [selectedProvider, providerConfig, originalName, onComplete] + ); + + const handleBack = useCallback(() => { + if (currentStep > 0) { + setCurrentStep(currentStep - 1); + // Restore previous value + const prevStep = visibleSteps[currentStep - 1]; + if (prevStep) { + setCurrentInput(values[prevStep.field] || ''); + } + setError(null); + } else if (selectedProvider) { + // Go back to provider selection + setSelectedProvider(null); + setError(null); + } else { + onClose(); + } + }, [currentStep, onClose, selectedProvider, values, visibleSteps]); + + // Handle keyboard input + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible || isSaving || isValidating) return false; + + // Delegate to LocalModelWizard when local provider is selected + if (selectedProvider === 'local' && localModelWizardRef.current) { + return localModelWizardRef.current.handleInput(input, key); + } + + // Escape to go back/close + if (key.escape) { + handleBack(); + return true; + } + + // Provider selection mode + if (!selectedProvider) { + const providers = getAvailableProviders(); + if (key.upArrow) { + setProviderIndex((prev) => + prev > 0 ? prev - 1 : providers.length - 1 + ); + return true; + } + if (key.downArrow) { + setProviderIndex((prev) => + prev < providers.length - 1 ? prev + 1 : 0 + ); + return true; + } + if (key.return) { + handleProviderSelect(); + return true; + } + return true; // Consume all input during provider selection + } + + // Wizard step mode + if (key.return) { + void handleNext(); + return true; + } + + // Backspace + if (key.backspace || key.delete) { + setCurrentInput((prev) => prev.slice(0, -1)); + setError(null); + return true; + } + + // Regular character input + if (input && !key.ctrl && !key.meta) { + setCurrentInput((prev) => prev + input); + setError(null); + return true; + } + + return false; + }, + }), + [ + isVisible, + isSaving, + isValidating, + handleBack, + handleNext, + handleProviderSelect, + selectedProvider, + ] + ); + + if (!isVisible) return null; + + // Provider selection screen (using shared component) + if (!selectedProvider) { + return <ProviderSelector selectedIndex={providerIndex} isEditing={isEditing} />; + } + + // Local provider uses specialized wizard with download support + if (selectedProvider === 'local') { + return ( + <LocalModelWizard + ref={localModelWizardRef} + isVisible={isVisible} + onComplete={onComplete} + onClose={() => { + // Go back to provider selection instead of closing completely + setSelectedProvider(null); + }} + /> + ); + } + + // Wizard steps screen for other providers + if (!currentStepConfig || !providerConfig) return null; + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="green" + paddingX={1} + marginTop={1} + > + {/* Header */} + <Box marginBottom={1}> + <Text bold color="green"> + {isEditing ? 'Edit Custom Model' : 'Add Custom Model'} + </Text> + <Text color="gray"> + {' '} + ({providerConfig.displayName}) Step {currentStep + 1}/{visibleSteps.length} + </Text> + </Box> + + {/* Setup info banner - data-driven, shown on first step only */} + {providerConfig.setupInfo && currentStep === 0 && ( + <SetupInfoBanner + title={providerConfig.setupInfo.title} + description={providerConfig.setupInfo.description} + docsUrl={providerConfig.setupInfo.docsUrl} + /> + )} + + {/* Step input with optional API key status */} + <WizardStepInput + step={currentStepConfig} + currentInput={currentInput} + error={error} + isValidating={isValidating} + isSaving={isSaving} + additionalContent={ + currentStepConfig.field === 'apiKey' ? ( + <ApiKeyStep provider={selectedProvider} /> + ) : undefined + } + /> + + {/* Help text */} + <Box marginTop={1}> + <Text color="gray"> + Enter to continue • Esc to{' '} + {currentStep > 0 ? 'go back' : 'back to provider'} + </Text> + </Box> + </Box> + ); + } +); + +export default CustomModelWizard; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/ExportWizard.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ExportWizard.tsx new file mode 100644 index 00000000..913e57e1 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ExportWizard.tsx @@ -0,0 +1,528 @@ +/** + * ExportWizard Component + * Interactive wizard for exporting conversation to markdown or JSON + */ + +import React, { useState, useEffect, forwardRef, useImperativeHandle, useCallback } from 'react'; +import { Box, Text } from 'ink'; +import fs from 'fs/promises'; +import path from 'path'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent } from '@dexto/core'; + +type ExportFormat = 'markdown' | 'json'; + +interface ExportOptions { + format: ExportFormat; + includeToolCalls: boolean; + filename: string; +} + +type WizardStep = 'format' | 'toolCalls' | 'filename' | 'confirm' | 'exporting' | 'done' | 'error'; + +interface ExportWizardProps { + isVisible: boolean; + agent: DextoAgent; + sessionId: string | null; + onClose: () => void; +} + +export interface ExportWizardHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * Generate default filename based on date and session ID + */ +function generateDefaultFilename(sessionId: string | null, format: ExportFormat): string { + const date = new Date().toISOString().split('T')[0]; // YYYY-MM-DD + const shortId = sessionId ? sessionId.slice(0, 6) : 'unknown'; + const ext = format === 'markdown' ? 'md' : 'json'; + return `conversation-${date}-${shortId}.${ext}`; +} + +interface FormattedMessage { + role: string; + content: unknown; + timestamp?: string | undefined; +} + +interface ExportMetadata { + sessionId: string; + title?: string | undefined; + createdAt?: string | undefined; +} + +/** + * Format conversation history as Markdown + */ +function formatAsMarkdown( + messages: FormattedMessage[], + metadata: ExportMetadata, + includeToolCalls: boolean +): string { + const lines: string[] = []; + + // Header + lines.push('# Conversation Export'); + lines.push(''); + lines.push(`- **Session**: ${metadata.sessionId}`); + if (metadata.title) { + lines.push(`- **Title**: ${metadata.title}`); + } + if (metadata.createdAt) { + lines.push(`- **Created**: ${metadata.createdAt}`); + } + lines.push(`- **Exported**: ${new Date().toISOString()}`); + lines.push(''); + lines.push('---'); + lines.push(''); + + // Messages + for (const msg of messages) { + const role = msg.role.charAt(0).toUpperCase() + msg.role.slice(1); + + // Skip tool messages if not including tool calls + if (!includeToolCalls && msg.role === 'tool') { + continue; + } + + lines.push(`## ${role}`); + if (msg.timestamp) { + lines.push(`*${msg.timestamp}*`); + } + lines.push(''); + + // Handle different content types + if (typeof msg.content === 'string') { + lines.push(msg.content); + } else if (Array.isArray(msg.content)) { + // Handle content parts (text, tool calls, tool results) + for (const part of msg.content) { + if (typeof part === 'string') { + lines.push(part); + } else if (part && typeof part === 'object') { + if ('text' in part) { + lines.push(String(part.text)); + } else if ('type' in part && part.type === 'tool-call' && includeToolCalls) { + const toolCall = part as { toolName?: string; args?: unknown }; + lines.push(''); + lines.push(`### Tool: ${toolCall.toolName || 'unknown'}`); + lines.push('```json'); + lines.push(JSON.stringify(toolCall.args || {}, null, 2)); + lines.push('```'); + } else if ('type' in part && part.type === 'tool-result' && includeToolCalls) { + const toolResult = part as { toolName?: string; result?: unknown }; + lines.push(''); + lines.push('<details>'); + lines.push( + `<summary>Result: ${toolResult.toolName || 'unknown'}</summary>` + ); + lines.push(''); + lines.push('```'); + const resultStr = + typeof toolResult.result === 'string' + ? toolResult.result + : JSON.stringify(toolResult.result, null, 2); + // Truncate very long results + if (resultStr.length > 2000) { + lines.push(resultStr.slice(0, 2000) + '\n... (truncated)'); + } else { + lines.push(resultStr); + } + lines.push('```'); + lines.push('</details>'); + } + } + } + } else if (msg.content && typeof msg.content === 'object') { + lines.push('```json'); + lines.push(JSON.stringify(msg.content, null, 2)); + lines.push('```'); + } + + lines.push(''); + lines.push('---'); + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Format conversation history as JSON + */ +function formatAsJson( + messages: FormattedMessage[], + metadata: ExportMetadata, + includeToolCalls: boolean +): string { + const filteredMessages = includeToolCalls + ? messages + : messages + .filter((m) => m.role !== 'tool') + .map((m) => ({ + ...m, + content: Array.isArray(m.content) + ? m.content.filter((part) => { + if (!part || typeof part !== 'object') return true; + return !( + 'type' in part && + (part.type === 'tool-call' || part.type === 'tool-result') + ); + }) + : m.content, + })); + + return JSON.stringify( + { + exportedAt: new Date().toISOString(), + session: metadata, + messages: filteredMessages, + }, + null, + 2 + ); +} + +/** + * Interactive wizard for exporting conversation + */ +const ExportWizard = forwardRef<ExportWizardHandle, ExportWizardProps>(function ExportWizard( + { isVisible, agent, sessionId, onClose }, + ref +) { + const [step, setStep] = useState<WizardStep>('format'); + const [options, setOptions] = useState<ExportOptions>({ + format: 'markdown', + includeToolCalls: true, + filename: '', + }); + const [selectedIndex, setSelectedIndex] = useState(0); + const [filenameInput, setFilenameInput] = useState(''); + const [exportResult, setExportResult] = useState<{ + success: boolean; + path?: string; + error?: string; + } | null>(null); + + // Reset when becoming visible + useEffect(() => { + if (isVisible) { + setStep('format'); + setOptions({ + format: 'markdown', + includeToolCalls: true, + filename: '', + }); + setSelectedIndex(0); + setFilenameInput(''); + setExportResult(null); + } + }, [isVisible]); + + // Update default filename when format changes + useEffect(() => { + if (step === 'filename' && !filenameInput) { + setFilenameInput(generateDefaultFilename(sessionId, options.format)); + } + }, [step, sessionId, options.format, filenameInput]); + + const doExport = useCallback(async () => { + if (!sessionId) { + setExportResult({ success: false, error: 'No active session' }); + setStep('error'); + return; + } + + setStep('exporting'); + + try { + // Get session history and metadata + const history = await agent.getSessionHistory(sessionId); + const metadata = await agent.getSessionMetadata(sessionId); + + const exportMetadata = { + sessionId, + title: metadata?.title, + createdAt: metadata?.createdAt + ? new Date(metadata.createdAt).toISOString() + : undefined, + }; + + // Format content - cast history to expected type + const formattedHistory = history.map((msg) => ({ + role: msg.role, + content: msg.content, + timestamp: + 'timestamp' in msg && typeof msg.timestamp === 'number' + ? new Date(msg.timestamp).toISOString() + : undefined, + })); + + const content = + options.format === 'markdown' + ? formatAsMarkdown(formattedHistory, exportMetadata, options.includeToolCalls) + : formatAsJson(formattedHistory, exportMetadata, options.includeToolCalls); + + // Write file - use path.basename to prevent path traversal attacks + const rawFilename = options.filename || filenameInput; + const safeFilename = path.basename(rawFilename); + const outputPath = path.resolve(process.cwd(), safeFilename); + await fs.writeFile(outputPath, content, 'utf-8'); + + setExportResult({ success: true, path: outputPath }); + setStep('done'); + } catch (error) { + setExportResult({ + success: false, + error: error instanceof Error ? error.message : String(error), + }); + setStep('error'); + } + }, [agent, sessionId, options, filenameInput]); + + // Handle keyboard input + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape to close + if (key.escape) { + onClose(); + return true; + } + + // Done/error state - Enter/Esc closes, consume all other input + if (step === 'done' || step === 'error') { + if (key.return || key.escape) { + onClose(); + } + return true; // Consume all input in terminal states + } + + // Format selection step + if (step === 'format') { + if (key.upArrow || key.downArrow) { + setSelectedIndex((prev) => (prev === 0 ? 1 : 0)); + return true; + } + if (key.return) { + setOptions((prev) => ({ + ...prev, + format: selectedIndex === 0 ? 'markdown' : 'json', + })); + setSelectedIndex(0); + setStep('toolCalls'); + return true; + } + return false; + } + + // Tool calls selection step + if (step === 'toolCalls') { + if (key.upArrow || key.downArrow) { + setSelectedIndex((prev) => (prev === 0 ? 1 : 0)); + return true; + } + if (key.return) { + setOptions((prev) => ({ + ...prev, + includeToolCalls: selectedIndex === 0, + })); + setFilenameInput(generateDefaultFilename(sessionId, options.format)); + setStep('filename'); + return true; + } + return false; + } + + // Filename input step + if (step === 'filename') { + if (key.return) { + const finalFilename = filenameInput.trim(); + if (!finalFilename) { + // Don't proceed with empty filename + return true; + } + setOptions((prev) => ({ ...prev, filename: finalFilename })); + setSelectedIndex(0); // Reset to "Export" option + setStep('confirm'); + return true; + } + if (key.backspace || key.delete) { + setFilenameInput((prev) => prev.slice(0, -1)); + return true; + } + if (input && !key.ctrl && !key.meta) { + setFilenameInput((prev) => prev + input); + return true; + } + return false; + } + + // Confirm step + if (step === 'confirm') { + if (key.upArrow || key.downArrow) { + setSelectedIndex((prev) => (prev === 0 ? 1 : 0)); + return true; + } + if (key.return) { + if (selectedIndex === 0) { + doExport(); + } else { + onClose(); + } + return true; + } + return false; + } + + return false; + }, + }), + [isVisible, step, selectedIndex, options, filenameInput, onClose, doExport, sessionId] + ); + + if (!isVisible) return null; + + // Render based on current step + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="cyan" + paddingX={1} + marginTop={1} + > + {/* Header */} + <Box marginBottom={1}> + <Text bold color="cyan"> + 📤 Export Conversation + </Text> + </Box> + + {/* Format selection */} + {step === 'format' && ( + <Box flexDirection="column"> + <Text>Select export format:</Text> + <Box marginTop={1} flexDirection="column"> + <Text {...(selectedIndex === 0 ? { color: 'cyan' } : {})}> + {selectedIndex === 0 ? '❯ ' : ' '}Markdown (.md) - Human readable + </Text> + <Text {...(selectedIndex === 1 ? { color: 'cyan' } : {})}> + {selectedIndex === 1 ? '❯ ' : ' '}JSON (.json) - Structured data + </Text> + </Box> + <Box marginTop={1}> + <Text color="gray">↑↓ to select • Enter to continue • Esc to cancel</Text> + </Box> + </Box> + )} + + {/* Tool calls selection */} + {step === 'toolCalls' && ( + <Box flexDirection="column"> + <Text>Include tool calls and results?</Text> + <Box marginTop={1} flexDirection="column"> + <Text {...(selectedIndex === 0 ? { color: 'cyan' } : {})}> + {selectedIndex === 0 ? '❯ ' : ' '}Yes - Include all tool interactions + </Text> + <Text {...(selectedIndex === 1 ? { color: 'cyan' } : {})}> + {selectedIndex === 1 ? '❯ ' : ' '}No - Only user and assistant messages + </Text> + </Box> + <Box marginTop={1}> + <Text color="gray">↑↓ to select • Enter to continue • Esc to cancel</Text> + </Box> + </Box> + )} + + {/* Filename input */} + {step === 'filename' && ( + <Box flexDirection="column"> + <Text>Filename:</Text> + <Box marginTop={1}> + <Text color="cyan">> </Text> + <Text>{filenameInput}</Text> + <Text color="cyan">_</Text> + </Box> + <Box marginTop={1}> + <Text color="gray">Enter to continue • Esc to cancel</Text> + </Box> + </Box> + )} + + {/* Confirm step */} + {step === 'confirm' && ( + <Box flexDirection="column"> + <Text>Export with these settings?</Text> + <Box marginTop={1} flexDirection="column" marginLeft={2}> + <Text color="gray"> + Format:{' '} + <Text color="white"> + {options.format === 'markdown' ? 'Markdown' : 'JSON'} + </Text> + </Text> + <Text color="gray"> + Tool calls:{' '} + <Text color="white">{options.includeToolCalls ? 'Yes' : 'No'}</Text> + </Text> + <Text color="gray"> + File: <Text color="white">{filenameInput}</Text> + </Text> + </Box> + <Box marginTop={1} flexDirection="column"> + <Text {...(selectedIndex === 0 ? { color: 'green' } : {})}> + {selectedIndex === 0 ? '❯ ' : ' '}Export + </Text> + <Text {...(selectedIndex === 1 ? { color: 'red' } : {})}> + {selectedIndex === 1 ? '❯ ' : ' '}Cancel + </Text> + </Box> + <Box marginTop={1}> + <Text color="gray">↑↓ to select • Enter to confirm</Text> + </Box> + </Box> + )} + + {/* Exporting state */} + {step === 'exporting' && ( + <Box flexDirection="column"> + <Text color="yellow">Exporting...</Text> + </Box> + )} + + {/* Done state */} + {step === 'done' && exportResult?.success && ( + <Box flexDirection="column"> + <Text color="green">✓ Exported successfully!</Text> + <Box marginTop={1}> + <Text color="gray">Saved to: </Text> + <Text>{exportResult.path}</Text> + </Box> + <Box marginTop={1}> + <Text color="gray">Press Enter or Esc to close</Text> + </Box> + </Box> + )} + + {/* Error state */} + {step === 'error' && ( + <Box flexDirection="column"> + <Text color="red">✗ Export failed</Text> + <Box marginTop={1}> + <Text color="red">{exportResult?.error}</Text> + </Box> + <Box marginTop={1}> + <Text color="gray">Press Enter or Esc to close</Text> + </Box> + </Box> + )} + </Box> + ); +}); + +export default ExportWizard; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/LogLevelSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/LogLevelSelector.tsx new file mode 100644 index 00000000..a68b75d0 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/LogLevelSelector.tsx @@ -0,0 +1,133 @@ +/** + * LogLevelSelector Component + * Interactive selector for changing log level + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent } from '@dexto/core'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +interface LogLevelSelectorProps { + isVisible: boolean; + onSelect: (level: string) => void; + onClose: () => void; + agent: DextoAgent; +} + +export interface LogLevelSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface LogLevelOption { + level: string; + description: string; + icon: string; + isCurrent: boolean; +} + +// Log levels matching DextoLogger's supported levels +const LOG_LEVELS: { level: string; description: string; icon: string }[] = [ + { level: 'error', description: 'Errors only', icon: '❌' }, + { level: 'warn', description: 'Warnings and above', icon: '⚠️' }, + { level: 'info', description: 'Info and above (default)', icon: 'ℹ️' }, + { level: 'debug', description: 'Debug information', icon: '🔍' }, + { level: 'silly', description: 'Everything (most verbose)', icon: '🔬' }, +]; + +/** + * Log level selector - thin wrapper around BaseSelector + */ +const LogLevelSelector = forwardRef<LogLevelSelectorHandle, LogLevelSelectorProps>( + function LogLevelSelector({ isVisible, onSelect, onClose, agent }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [levels, setLevels] = useState<LogLevelOption[]>([]); + const [selectedIndex, setSelectedIndex] = useState(0); + const [logFilePath, setLogFilePath] = useState<string | null>(null); + + // Build levels list with current indicator + useEffect(() => { + if (!isVisible) { + setLogFilePath(null); + return; + } + + // Get current level from agent's logger (shared across all child loggers) + const currentLevel = agent.logger.getLevel(); + const levelList = LOG_LEVELS.map((l) => ({ + ...l, + isCurrent: l.level === currentLevel, + })); + + setLevels(levelList); + setLogFilePath(agent.logger.getLogFilePath()); + + // Set initial selection to current level + const currentIndex = levelList.findIndex((l) => l.isCurrent); + if (currentIndex >= 0) { + setSelectedIndex(currentIndex); + } + }, [isVisible, agent]); + + // Format level item for display + const formatItem = (option: LogLevelOption, isSelected: boolean) => ( + <> + <Text>{option.icon} </Text> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {option.level} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> - {option.description}</Text> + {option.isCurrent && ( + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {' '} + ← Current + </Text> + )} + </> + ); + + // Handle selection + const handleSelect = (option: LogLevelOption) => { + onSelect(option.level); + }; + + return ( + <Box flexDirection="column"> + <BaseSelector + ref={baseSelectorRef} + items={levels} + isVisible={isVisible} + isLoading={false} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Select Log Level" + borderColor="yellowBright" + emptyMessage="No log levels available" + /> + {logFilePath && process.env.DEXTO_DEV_MODE === 'true' && ( + <Box marginTop={1}> + <Text color="gray">📁 Log file: {logFilePath}</Text> + </Box> + )} + </Box> + ); + } +); + +export default LogLevelSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/MarketplaceAddPrompt.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/MarketplaceAddPrompt.tsx new file mode 100644 index 00000000..51640148 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/MarketplaceAddPrompt.tsx @@ -0,0 +1,161 @@ +/** + * MarketplaceAddPrompt Component + * Prompts user to enter a marketplace source to add + */ + +import React, { useState, useEffect, forwardRef, useImperativeHandle, useCallback } from 'react'; +import { Box, Text } from 'ink'; +import { addMarketplace } from '@dexto/agent-management'; +import { logger } from '@dexto/core'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; + +interface MarketplaceAddPromptProps { + isVisible: boolean; + onComplete: (name: string, pluginCount: number) => void; + onClose: () => void; +} + +export interface MarketplaceAddPromptHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * Marketplace add prompt - single input for source + */ +const MarketplaceAddPrompt = forwardRef<MarketplaceAddPromptHandle, MarketplaceAddPromptProps>( + function MarketplaceAddPrompt({ isVisible, onComplete, onClose }, ref) { + const [input, setInput] = useState(''); + const [error, setError] = useState<string | null>(null); + const [isAdding, setIsAdding] = useState(false); + + // Reset when becoming visible + useEffect(() => { + if (isVisible) { + setInput(''); + setError(null); + setIsAdding(false); + } + }, [isVisible]); + + // Handle adding marketplace + const handleAdd = useCallback(async () => { + const source = input.trim(); + if (!source) { + setError('Please enter a marketplace source'); + return; + } + + setError(null); + setIsAdding(true); + + try { + const result = await addMarketplace(source); + onComplete(result.name, result.pluginCount); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : String(err); + setError(errorMessage); + logger.error(`MarketplaceAddPrompt.handleAdd failed: ${errorMessage}`); + } finally { + setIsAdding(false); + } + }, [input, onComplete]); + + // Handle input + useImperativeHandle( + ref, + () => ({ + handleInput: (inputStr: string, key: Key): boolean => { + // Escape to close + if (key.escape) { + onClose(); + return true; + } + + // Enter to submit + if (key.return) { + if (!isAdding) { + handleAdd(); + } + return true; + } + + // Backspace + if (key.backspace || key.delete) { + setInput((prev) => prev.slice(0, -1)); + setError(null); + return true; + } + + // Regular character input + if (inputStr && !key.ctrl && !key.meta) { + setInput((prev) => prev + inputStr); + setError(null); + return true; + } + + return false; + }, + }), + [handleAdd, isAdding, onClose] + ); + + if (!isVisible) return null; + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="green" + paddingX={1} + marginTop={1} + > + <Box marginBottom={1}> + <Text bold color="green"> + Add Marketplace + </Text> + </Box> + + <Box marginBottom={1}> + <Text color="gray">Enter marketplace source:</Text> + </Box> + <Box marginBottom={1} flexDirection="column"> + <Text color="gray" dimColor> + - GitHub: owner/repo (e.g., anthropics/claude-plugins-official) + </Text> + <Text color="gray" dimColor> + - Git URL: https://github.com/user/repo.git + </Text> + <Text color="gray" dimColor> + - Local: /path/to/marketplace or ~/marketplace + </Text> + </Box> + + <Box> + <Text color="cyan">{'> '}</Text> + <Text>{input}</Text> + <Text color="cyan">_</Text> + </Box> + + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + {isAdding && ( + <Box marginTop={1}> + <Text color="yellow">Adding marketplace...</Text> + </Box> + )} + + <Box marginTop={1}> + <Text color="gray" dimColor> + Press Enter to add, Escape to cancel + </Text> + </Box> + </Box> + ); + } +); + +export default MarketplaceAddPrompt; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/MarketplaceBrowser.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/MarketplaceBrowser.tsx new file mode 100644 index 00000000..7eb7f29d --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/MarketplaceBrowser.tsx @@ -0,0 +1,557 @@ +/** + * MarketplaceBrowser Component + * Clean, intuitive marketplace browser with two-level navigation + */ + +import React, { + useState, + useEffect, + forwardRef, + useRef, + useImperativeHandle, + useMemo, +} from 'react'; +import { Box, Text } from 'ink'; +import { + listMarketplaces, + listAllMarketplacePlugins, + installPluginFromMarketplace, + getUninstalledDefaults, + addMarketplace, + listInstalledPlugins, + type MarketplaceEntry, + type MarketplacePlugin, +} from '@dexto/agent-management'; +import { logger } from '@dexto/core'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type MarketplaceBrowserAction = + | { type: 'add-marketplace' } + | { type: 'install-plugin'; plugin: MarketplacePlugin } + | { type: 'plugin-installed'; pluginName: string; marketplace: string } + | { type: 'marketplace-added'; marketplaceName: string }; + +interface MarketplaceBrowserProps { + isVisible: boolean; + onAction: (action: MarketplaceBrowserAction) => void; + onClose: () => void; +} + +export interface MarketplaceBrowserHandle { + handleInput: (input: string, key: Key) => boolean; +} + +type BrowserView = 'marketplaces' | 'plugins' | 'scope-select'; + +type InstallScope = 'user' | 'project'; + +// List item types +interface BackItem { + type: 'back'; +} + +interface MarketplaceItem { + type: 'marketplace'; + marketplace: MarketplaceEntry; + pluginCount: number; +} + +interface DefaultMarketplaceItem { + type: 'default-marketplace'; + name: string; + sourceValue: string; + sourceType: 'github' | 'git' | 'local'; +} + +interface AddMarketplaceItem { + type: 'add-new'; +} + +interface PluginItem { + type: 'plugin'; + plugin: MarketplacePlugin; + isInstalled: boolean; +} + +interface ScopeItem { + type: 'scope'; + scope: InstallScope; + label: string; + description: string; + icon: string; +} + +type ListItem = + | BackItem + | MarketplaceItem + | DefaultMarketplaceItem + | AddMarketplaceItem + | PluginItem + | ScopeItem; + +/** + * Get source type icon + */ +function getSourceIcon(type: string): string { + switch (type) { + case 'github': + return '🐙'; + case 'git': + return '📦'; + case 'local': + return '📁'; + default: + return '📦'; + } +} + +/** + * Marketplace browser overlay - clean two-level navigation + */ +interface UninstalledDefault { + name: string; + source: { type: 'github' | 'git' | 'local'; value: string }; +} + +const MarketplaceBrowser = forwardRef<MarketplaceBrowserHandle, MarketplaceBrowserProps>( + function MarketplaceBrowser({ isVisible, onAction, onClose }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + const [selectedIndex, setSelectedIndex] = useState(0); + const [isLoading, setIsLoading] = useState(true); + const [loadError, setLoadError] = useState<string | null>(null); + const [view, setView] = useState<BrowserView>('marketplaces'); + const [marketplaces, setMarketplaces] = useState<MarketplaceEntry[]>([]); + const [plugins, setPlugins] = useState<MarketplacePlugin[]>([]); + const [selectedMarketplace, setSelectedMarketplace] = useState<string | null>(null); + const [isInstalling, setIsInstalling] = useState(false); + const [uninstalledDefaults, setUninstalledDefaults] = useState<UninstalledDefault[]>([]); + const [pendingPlugin, setPendingPlugin] = useState<MarketplacePlugin | null>(null); + const [installedPluginNames, setInstalledPluginNames] = useState<Set<string>>(new Set()); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + // Load data when visible + useEffect(() => { + if (isVisible) { + loadMarketplaces(); + } + }, [isVisible]); + + // Load marketplaces + const loadMarketplaces = () => { + setIsLoading(true); + setLoadError(null); + setView('marketplaces'); + setSelectedMarketplace(null); + setSelectedIndex(0); + + try { + const mktplaces = listMarketplaces(); + setMarketplaces(mktplaces); + + const allPlugins = listAllMarketplacePlugins(); + setPlugins(allPlugins); + + const defaults = getUninstalledDefaults(); + setUninstalledDefaults(defaults); + + const installed = listInstalledPlugins(); + setInstalledPluginNames(new Set(installed.map((p) => p.name.toLowerCase()))); + } catch (error) { + setLoadError( + `Failed to load: ${error instanceof Error ? error.message : String(error)}` + ); + setMarketplaces([]); + setPlugins([]); + setUninstalledDefaults([]); + setInstalledPluginNames(new Set()); + } finally { + setIsLoading(false); + } + }; + + // Show plugins for a marketplace + const showMarketplacePlugins = (marketplaceName: string) => { + setSelectedMarketplace(marketplaceName); + setView('plugins'); + setSelectedIndex(0); + }; + + // Go back to marketplace list + const goBackToMarketplaces = () => { + setView('marketplaces'); + setSelectedMarketplace(null); + setSelectedIndex(0); + }; + + // Show scope selection for a plugin + const showScopeSelection = (plugin: MarketplacePlugin) => { + setPendingPlugin(plugin); + setView('scope-select'); + setSelectedIndex(0); + }; + + // Go back to plugins from scope selection + const goBackToPlugins = () => { + setPendingPlugin(null); + setView('plugins'); + setSelectedIndex(0); + }; + + // Build items based on current view + const items = useMemo<ListItem[]>(() => { + if (view === 'marketplaces') { + const list: ListItem[] = []; + + // Back option first + list.push({ type: 'back' }); + + // Uninstalled default marketplaces (setup prompts) + for (const def of uninstalledDefaults) { + list.push({ + type: 'default-marketplace', + name: def.name, + sourceValue: def.source.value, + sourceType: def.source.type, + }); + } + + // Installed marketplaces + for (const m of marketplaces) { + const pluginCount = plugins.filter((p) => p.marketplace === m.name).length; + list.push({ + type: 'marketplace', + marketplace: m, + pluginCount, + }); + } + + // Add marketplace option + list.push({ type: 'add-new' }); + + return list; + } else if (view === 'scope-select') { + // Scope selection view + const list: ListItem[] = [{ type: 'back' }]; + + list.push({ + type: 'scope', + scope: 'user', + label: 'Global (user)', + description: 'Available in all projects', + icon: '🌐', + }); + + list.push({ + type: 'scope', + scope: 'project', + label: 'Project only', + description: 'Only in current project, can be committed to git', + icon: '📁', + }); + + return list; + } else { + // Plugins view + const filteredPlugins = selectedMarketplace + ? plugins.filter((p) => p.marketplace === selectedMarketplace) + : plugins; + + const list: ListItem[] = [{ type: 'back' }]; + + for (const plugin of filteredPlugins) { + const isInstalled = installedPluginNames.has(plugin.name.toLowerCase()); + list.push({ + type: 'plugin', + plugin, + isInstalled, + }); + } + + return list; + } + }, [ + view, + marketplaces, + plugins, + selectedMarketplace, + uninstalledDefaults, + installedPluginNames, + ]); + + // Format item for display + const formatItem = (item: ListItem, isSelected: boolean) => { + // Back option + if (item.type === 'back') { + const label = + view === 'scope-select' + ? 'Back to plugins' + : view === 'plugins' + ? 'Back to marketplaces' + : 'Back to menu'; + return ( + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}>{isSelected ? '▸ ' : ' '}</Text> + <Text color="gray">← </Text> + <Text color={isSelected ? 'white' : 'gray'}>{label}</Text> + </Box> + ); + } + + // Scope selection option + if (item.type === 'scope') { + return ( + <Box flexDirection="column"> + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}> + {isSelected ? '▸ ' : ' '} + </Text> + <Text>{item.icon} </Text> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {item.label} + </Text> + </Box> + {isSelected && ( + <Box marginLeft={4}> + <Text color="gray" dimColor> + {item.description} + </Text> + </Box> + )} + </Box> + ); + } + + // Add marketplace option + if (item.type === 'add-new') { + return ( + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}>{isSelected ? '▸ ' : ' '}</Text> + <Text color={isSelected ? 'green' : 'gray'}>+ </Text> + <Text color={isSelected ? 'green' : 'gray'}>Add custom marketplace</Text> + </Box> + ); + } + + // Default (uninstalled) marketplace + if (item.type === 'default-marketplace') { + const icon = getSourceIcon(item.sourceType); + return ( + <Box flexDirection="column"> + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}> + {isSelected ? '▸ ' : ' '} + </Text> + <Text>{icon} </Text> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {item.name} + </Text> + <Text color="yellow"> (not installed)</Text> + </Box> + {isSelected && ( + <Box marginLeft={4}> + <Text color="gray" dimColor> + {item.sourceValue} + </Text> + </Box> + )} + </Box> + ); + } + + // Installed marketplace + if (item.type === 'marketplace') { + const m = item.marketplace; + const icon = getSourceIcon(m.source.type); + + return ( + <Box flexDirection="column"> + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}> + {isSelected ? '▸ ' : ' '} + </Text> + <Text>{icon} </Text> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {m.name} + </Text> + <Text color="gray" dimColor> + {' '} + ({item.pluginCount} plugins) + </Text> + </Box> + {isSelected && ( + <Box marginLeft={4}> + <Text color="gray" dimColor> + {m.source.value} + </Text> + </Box> + )} + </Box> + ); + } + + // Plugin item + const p = item.plugin; + const statusBadge = item.isInstalled ? <Text color="green"> ✓</Text> : null; + + return ( + <Box flexDirection="column"> + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}>{isSelected ? '▸ ' : ' '}</Text> + <Text color={isSelected ? 'white' : 'gray'}>📦 </Text> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {p.name} + </Text> + {p.version && ( + <Text color="gray" dimColor> + @{p.version} + </Text> + )} + {p.category && ( + <Text color="magenta" dimColor> + {' '} + [{p.category}] + </Text> + )} + {statusBadge} + </Box> + {isSelected && p.description && ( + <Box marginLeft={4}> + <Text color="gray">{p.description}</Text> + </Box> + )} + {isSelected && item.isInstalled && ( + <Box marginLeft={4}> + <Text color="gray" dimColor> + Already installed + </Text> + </Box> + )} + </Box> + ); + }; + + // Handle selection + const handleSelect = async (item: ListItem) => { + if (item.type === 'back') { + if (view === 'scope-select') { + goBackToPlugins(); + } else if (view === 'plugins') { + goBackToMarketplaces(); + } else { + onClose(); + } + return; + } + + if (item.type === 'add-new') { + onAction({ type: 'add-marketplace' }); + return; + } + + if (item.type === 'default-marketplace' && !isInstalling) { + setIsInstalling(true); + try { + await addMarketplace(item.sourceValue, { name: item.name }); + onAction({ type: 'marketplace-added', marketplaceName: item.name }); + loadMarketplaces(); + } catch (error) { + logger.error( + `Failed to add marketplace ${item.name}: ${error instanceof Error ? error.message : String(error)}` + ); + } finally { + setIsInstalling(false); + } + return; + } + + if (item.type === 'marketplace') { + showMarketplacePlugins(item.marketplace.name); + return; + } + + // Show scope selection for plugin + if (item.type === 'plugin' && !item.isInstalled) { + showScopeSelection(item.plugin); + return; + } + + // Install plugin with selected scope + if (item.type === 'scope' && pendingPlugin && !isInstalling) { + setIsInstalling(true); + try { + const result = await installPluginFromMarketplace( + `${pendingPlugin.name}@${pendingPlugin.marketplace}`, + { scope: item.scope } + ); + setInstalledPluginNames((prev) => { + const next = new Set(prev); + next.add(result.pluginName.toLowerCase()); + return next; + }); + onAction({ + type: 'plugin-installed', + pluginName: result.pluginName, + marketplace: result.marketplace, + }); + // Go back to plugins view after successful install + goBackToPlugins(); + } catch (error) { + logger.error( + `Failed to install ${pendingPlugin.name}: ${error instanceof Error ? error.message : String(error)}` + ); + } finally { + setIsInstalling(false); + } + } + }; + + // Get title based on view + const getTitle = () => { + if (view === 'scope-select' && pendingPlugin) { + return `Install ${pendingPlugin.name} › Choose Scope`; + } + if (view === 'plugins' && selectedMarketplace) { + return `${selectedMarketplace} › Plugins`; + } + return 'Marketplace'; + }; + + // Get empty message + const getEmptyMessage = () => { + if (loadError) return loadError; + if (view === 'plugins') return 'No plugins found in this marketplace'; + return 'No marketplaces. Add one to browse plugins.'; + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={items} + isVisible={isVisible} + isLoading={isLoading || isInstalling} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title={getTitle()} + borderColor="green" + emptyMessage={getEmptyMessage()} + loadingMessage={isInstalling ? 'Installing...' : 'Loading...'} + /> + ); + } +); + +export default MarketplaceBrowser; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpAddChoice.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpAddChoice.tsx new file mode 100644 index 00000000..d3d4690b --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpAddChoice.tsx @@ -0,0 +1,118 @@ +/** + * McpAddChoice Component + * Asks user whether to add from registry or custom + * Shown when "Add new server" is selected from McpServerList + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type McpAddChoiceType = 'registry' | 'custom' | 'back'; + +interface McpAddChoiceProps { + isVisible: boolean; + onSelect: (choice: McpAddChoiceType) => void; + onClose: () => void; +} + +export interface McpAddChoiceHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ChoiceItem { + id: string; + type: McpAddChoiceType; + label: string; + description: string; + icon: string; +} + +const CHOICES: ChoiceItem[] = [ + { + id: 'registry', + type: 'registry', + label: 'Explore registry', + description: 'Browse available MCP server presets', + icon: '📦', + }, + { + id: 'custom', + type: 'custom', + label: 'Add custom server', + description: 'Configure your own MCP server', + icon: '⚙️', + }, + { + id: 'back', + type: 'back', + label: 'Back', + description: 'Return to server list', + icon: '←', + }, +]; + +/** + * MCP Add Choice - registry vs custom + */ +const McpAddChoice = forwardRef<McpAddChoiceHandle, McpAddChoiceProps>(function McpAddChoice( + { isVisible, onSelect, onClose }, + ref +) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + // Reset selection when becoming visible + useEffect(() => { + if (isVisible) { + setSelectedIndex(0); + } + }, [isVisible]); + + // Format item for display + const formatItem = (item: ChoiceItem, isSelected: boolean) => ( + <Box> + <Text>{item.icon} </Text> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {item.label} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> - {item.description}</Text> + </Box> + ); + + // Handle selection + const handleSelect = (item: ChoiceItem) => { + onSelect(item.type); + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={CHOICES} + isVisible={isVisible} + isLoading={false} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Add MCP Server" + borderColor="green" + emptyMessage="No options available" + /> + ); +}); + +export default McpAddChoice; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpAddSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpAddSelector.tsx new file mode 100644 index 00000000..35ab109c --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpAddSelector.tsx @@ -0,0 +1,139 @@ +/** + * McpAddSelector Component + * Shows registry presets for adding MCP servers + * (Custom server options are handled separately via McpSelector "add custom") + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { serverRegistry, type ServerRegistryEntry } from '@dexto/registry'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type McpAddResult = { type: 'preset'; entry: ServerRegistryEntry }; + +interface McpAddSelectorProps { + isVisible: boolean; + onSelect: (result: McpAddResult) => void; + onClose: () => void; +} + +export interface McpAddSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface McpAddOption { + id: string; + label: string; + description: string; + icon: string; + entry: ServerRegistryEntry; +} + +/** + * MCP add selector - shows registry presets only + */ +const McpAddSelector = forwardRef<McpAddSelectorHandle, McpAddSelectorProps>( + function McpAddSelector({ isVisible, onSelect, onClose }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [options, setOptions] = useState<McpAddOption[]>([]); + const [isLoading, setIsLoading] = useState(false); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Fetch registry entries + useEffect(() => { + if (!isVisible) return; + + let cancelled = false; + setIsLoading(true); + + const fetchEntries = async () => { + try { + const entries = await serverRegistry.getEntries(); + + if (cancelled) return; + + // Only show presets that are not already installed + const availablePresets = entries.filter((e) => !e.isInstalled); + const optionList: McpAddOption[] = availablePresets.map((entry) => ({ + id: entry.id, + label: entry.name, + description: entry.description, + icon: entry.icon || '📦', + entry, + })); + + setOptions(optionList); + setSelectedIndex(0); + } catch { + // On error, options remain empty - BaseSelector will show emptyMessage + if (!cancelled) { + setOptions([]); + } + } finally { + if (!cancelled) { + setIsLoading(false); + } + } + }; + + void fetchEntries(); + + return () => { + cancelled = true; + }; + }, [isVisible]); + + // Format option for display + const formatItem = (option: McpAddOption, isSelected: boolean) => ( + <> + <Text>{option.icon} </Text> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {option.label} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> + {' '} + - {option.description.slice(0, 40)} + {option.description.length > 40 ? '...' : ''} + </Text> + </> + ); + + // Handle selection + const handleSelect = (option: McpAddOption) => { + onSelect({ type: 'preset', entry: option.entry }); + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={options} + isVisible={isVisible} + isLoading={isLoading} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Add MCP Server (Presets)" + borderColor="green" + loadingMessage="Loading server presets..." + emptyMessage="No server presets available" + /> + ); + } +); + +export default McpAddSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpCustomTypeSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpCustomTypeSelector.tsx new file mode 100644 index 00000000..c0f96446 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpCustomTypeSelector.tsx @@ -0,0 +1,108 @@ +/** + * McpCustomTypeSelector Component + * Shows server type options (stdio/http/sse) for custom MCP server + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { McpServerType } from '@dexto/core'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +interface McpCustomTypeSelectorProps { + isVisible: boolean; + onSelect: (serverType: McpServerType) => void; + onClose: () => void; +} + +export interface McpCustomTypeSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ServerTypeOption { + type: McpServerType; + label: string; + description: string; + icon: string; +} + +const SERVER_TYPE_OPTIONS: ServerTypeOption[] = [ + { + type: 'stdio', + label: 'STDIO', + description: 'Local process (npx, uvx, node, python)', + icon: '▶️', + }, + { + type: 'http', + label: 'HTTP', + description: 'Remote HTTP server', + icon: '🌐', + }, + { + type: 'sse', + label: 'SSE', + description: 'Server-Sent Events endpoint', + icon: '📡', + }, +]; + +/** + * MCP custom type selector - picks server transport type + */ +const McpCustomTypeSelector = forwardRef<McpCustomTypeSelectorHandle, McpCustomTypeSelectorProps>( + function McpCustomTypeSelector({ isVisible, onSelect, onClose }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [selectedIndex, setSelectedIndex] = useState(0); + + useEffect(() => { + if (isVisible) { + setSelectedIndex(0); + } + }, [isVisible]); + + const formatItem = (option: ServerTypeOption, isSelected: boolean) => ( + <> + <Text>{option.icon} </Text> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {option.label} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> - {option.description}</Text> + </> + ); + + const handleSelect = (option: ServerTypeOption) => { + onSelect(option.type); + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={SERVER_TYPE_OPTIONS} + isVisible={isVisible} + isLoading={false} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Select Server Type" + borderColor="yellowBright" + emptyMessage="No options available" + /> + ); + } +); + +export default McpCustomTypeSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpCustomWizard.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpCustomWizard.tsx new file mode 100644 index 00000000..ab56c5b0 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpCustomWizard.tsx @@ -0,0 +1,312 @@ +/** + * McpCustomWizard Component + * Multi-step wizard for collecting custom MCP server configuration + */ + +import React, { useState, useEffect, forwardRef, useImperativeHandle, useCallback } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { McpServerType } from '@dexto/core'; + +export interface McpCustomConfig { + serverType: McpServerType; + name: string; + // STDIO fields + command?: string; + args?: string[]; + // HTTP/SSE fields + url?: string; +} + +interface WizardStep { + field: string; + label: string; + placeholder: string; + required: boolean; + validate?: (value: string) => string | null; // Returns error message or null if valid +} + +const STDIO_STEPS: WizardStep[] = [ + { + field: 'name', + label: 'Server Name', + placeholder: 'e.g., my-server', + required: true, + validate: (v) => (v.trim() ? null : 'Name is required'), + }, + { + field: 'command', + label: 'Command', + placeholder: 'e.g., npx, uvx, node, python', + required: true, + validate: (v) => (v.trim() ? null : 'Command is required'), + }, + { + field: 'args', + label: 'Arguments (space-separated, optional)', + placeholder: 'e.g., -y @modelcontextprotocol/server-filesystem .', + required: false, + }, +]; + +const HTTP_STEPS: WizardStep[] = [ + { + field: 'name', + label: 'Server Name', + placeholder: 'e.g., my-http-server', + required: true, + validate: (v) => (v.trim() ? null : 'Name is required'), + }, + { + field: 'url', + label: 'Server URL', + placeholder: 'e.g., http://localhost:8080', + required: true, + validate: (v) => { + if (!v.trim()) return 'URL is required'; + try { + new URL(v); + return null; + } catch { + return 'Invalid URL format'; + } + }, + }, +]; + +const SSE_STEPS: WizardStep[] = [ + { + field: 'name', + label: 'Server Name', + placeholder: 'e.g., my-sse-server', + required: true, + validate: (v) => (v.trim() ? null : 'Name is required'), + }, + { + field: 'url', + label: 'SSE Endpoint URL', + placeholder: 'e.g., http://localhost:9000/events', + required: true, + validate: (v) => { + if (!v.trim()) return 'URL is required'; + try { + new URL(v); + return null; + } catch { + return 'Invalid URL format'; + } + }, + }, +]; + +function getStepsForType(serverType: McpServerType): WizardStep[] { + switch (serverType) { + case 'stdio': + return STDIO_STEPS; + case 'http': + return HTTP_STEPS; + case 'sse': + return SSE_STEPS; + } +} + +interface McpCustomWizardProps { + isVisible: boolean; + serverType: McpServerType; + onComplete: (config: McpCustomConfig) => void; + onClose: () => void; +} + +export interface McpCustomWizardHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * Multi-step wizard for custom MCP server configuration + */ +const McpCustomWizard = forwardRef<McpCustomWizardHandle, McpCustomWizardProps>( + function McpCustomWizard({ isVisible, serverType, onComplete, onClose }, ref) { + const steps = getStepsForType(serverType); + const [currentStep, setCurrentStep] = useState(0); + const [values, setValues] = useState<Record<string, string>>({}); + const [currentInput, setCurrentInput] = useState(''); + const [error, setError] = useState<string | null>(null); + + // Reset when becoming visible or server type changes + useEffect(() => { + if (isVisible) { + setCurrentStep(0); + setValues({}); + setCurrentInput(''); + setError(null); + } + }, [isVisible, serverType]); + + const currentStepConfig = steps[currentStep]; + + const handleNext = useCallback(() => { + if (!currentStepConfig) return; + + const value = currentInput.trim(); + + // Validate + if (currentStepConfig.validate) { + const validationError = currentStepConfig.validate(value); + if (validationError) { + setError(validationError); + return; + } + } else if (currentStepConfig.required && !value) { + setError(`${currentStepConfig.label} is required`); + return; + } + + // Save value + const newValues = { ...values, [currentStepConfig.field]: value }; + setValues(newValues); + setError(null); + setCurrentInput(''); + + // Check if we're done + if (currentStep >= steps.length - 1) { + // Build config and complete + const config: McpCustomConfig = { + serverType, + name: newValues.name || '', + }; + + if (serverType === 'stdio') { + if (newValues.command) { + config.command = newValues.command; + } + if (newValues.args?.trim()) { + config.args = newValues.args.split(/\s+/).filter(Boolean); + } + } else { + if (newValues.url) { + config.url = newValues.url; + } + } + + onComplete(config); + } else { + setCurrentStep(currentStep + 1); + } + }, [ + currentInput, + currentStep, + currentStepConfig, + onComplete, + serverType, + steps.length, + values, + ]); + + const handleBack = useCallback(() => { + if (currentStep > 0) { + setCurrentStep(currentStep - 1); + // Restore previous value + const prevStep = steps[currentStep - 1]; + if (prevStep) { + setCurrentInput(values[prevStep.field] || ''); + } + setError(null); + } else { + onClose(); + } + }, [currentStep, onClose, steps, values]); + + // Handle keyboard input + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape to go back/close + if (key.escape) { + handleBack(); + return true; + } + + // Enter to submit current step + if (key.return) { + handleNext(); + return true; + } + + // Backspace + if (key.backspace || key.delete) { + setCurrentInput((prev) => prev.slice(0, -1)); + setError(null); + return true; + } + + // Regular character input + if (input && !key.ctrl && !key.meta) { + setCurrentInput((prev) => prev + input); + setError(null); + return true; + } + + return false; + }, + }), + [isVisible, handleBack, handleNext] + ); + + if (!isVisible || !currentStepConfig) return null; + + const serverTypeLabel = serverType.toUpperCase(); + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="yellowBright" + paddingX={1} + marginTop={1} + > + {/* Header */} + <Box marginBottom={1}> + <Text bold color="yellowBright"> + Add Custom {serverTypeLabel} Server + </Text> + <Text color="gray"> + {' '} + (Step {currentStep + 1}/{steps.length}) + </Text> + </Box> + + {/* Current step prompt */} + <Box flexDirection="column"> + <Text bold>{currentStepConfig.label}:</Text> + <Text color="gray">{currentStepConfig.placeholder}</Text> + </Box> + + {/* Input field */} + <Box marginTop={1}> + <Text color="cyan">> </Text> + <Text>{currentInput}</Text> + <Text color="cyan">_</Text> + </Box> + + {/* Error message */} + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + {/* Help text */} + <Box marginTop={1}> + <Text color="gray"> + Enter to continue • Esc to {currentStep > 0 ? 'go back' : 'cancel'} + </Text> + </Box> + </Box> + ); + } +); + +export default McpCustomWizard; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpRemoveSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpRemoveSelector.tsx new file mode 100644 index 00000000..788e2678 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpRemoveSelector.tsx @@ -0,0 +1,143 @@ +/** + * McpRemoveSelector Component + * Shows installed MCP servers for removal + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent, McpServerStatus, McpConnectionStatus } from '@dexto/core'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +interface McpRemoveSelectorProps { + isVisible: boolean; + onSelect: (serverName: string) => void; + onClose: () => void; + agent: DextoAgent; +} + +export interface McpRemoveSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * MCP remove selector - shows installed servers for removal + */ +const McpRemoveSelector = forwardRef<McpRemoveSelectorHandle, McpRemoveSelectorProps>( + function McpRemoveSelector({ isVisible, onSelect, onClose, agent }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [servers, setServers] = useState<McpServerStatus[]>([]); + const [isLoading, setIsLoading] = useState(false); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Fetch installed servers + useEffect(() => { + if (!isVisible) return; + + setIsLoading(true); + + try { + // Get servers with computed status from agent + const serverList = agent.getMcpServersWithStatus(); + + // Sort alphabetically + serverList.sort((a, b) => a.name.localeCompare(b.name)); + + setServers(serverList); + setSelectedIndex(0); + } finally { + setIsLoading(false); + } + }, [isVisible, agent]); + + // Get display icon for status + const getStatusIcon = (status: McpConnectionStatus): string => { + switch (status) { + case 'connected': + return '🔌'; + case 'disconnected': + return '⏸️'; + case 'error': + return '❌'; + } + }; + + // Get display text for status + const getStatusText = (status: McpConnectionStatus): string => { + switch (status) { + case 'connected': + return 'Connected'; + case 'disconnected': + return 'Disabled'; + case 'error': + return 'Failed'; + } + }; + + // Format server item for display + const formatItem = (server: McpServerStatus, isSelected: boolean) => ( + <> + <Text>{getStatusIcon(server.status)} </Text> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {server.name} + </Text> + <Text + color={ + server.status === 'connected' + ? 'green' + : server.status === 'disconnected' + ? 'gray' + : 'red' + } + > + {' '} + - {getStatusText(server.status)} + </Text> + {server.error && ( + <Text color="gray"> + {' '} + ({server.error.slice(0, 30)} + {server.error.length > 30 ? '...' : ''}) + </Text> + )} + </> + ); + + // Handle selection + const handleSelect = (server: McpServerStatus) => { + onSelect(server.name); + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={servers} + isVisible={isVisible} + isLoading={isLoading} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Remove MCP Server" + borderColor="red" + loadingMessage="Loading servers..." + emptyMessage="No MCP servers installed" + /> + ); + } +); + +export default McpRemoveSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpSelector.tsx new file mode 100644 index 00000000..6f6c790e --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpSelector.tsx @@ -0,0 +1,111 @@ +/** + * McpSelector Component + * Main MCP selector - shows list, add, remove options + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type McpAction = 'list' | 'add-preset' | 'add-custom' | 'remove'; + +interface McpSelectorProps { + isVisible: boolean; + onSelect: (action: McpAction) => void; + onClose: () => void; +} + +export interface McpSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface McpOption { + action: McpAction; + label: string; + description: string; + icon: string; +} + +const MCP_OPTIONS: McpOption[] = [ + { action: 'list', label: 'list', description: 'List connected servers', icon: '📋' }, + { + action: 'add-preset', + label: 'add preset', + description: 'Add from registry presets', + icon: '📦', + }, + { + action: 'add-custom', + label: 'add custom', + description: 'Add custom server (stdio/http/sse)', + icon: '⚙️', + }, + { action: 'remove', label: 'remove', description: 'Remove a server', icon: '🗑️' }, +]; + +/** + * MCP selector - shows main MCP actions + */ +const McpSelector = forwardRef<McpSelectorHandle, McpSelectorProps>(function McpSelector( + { isVisible, onSelect, onClose }, + ref +) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [options] = useState<McpOption[]>(MCP_OPTIONS); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Reset selection when becoming visible + useEffect(() => { + if (isVisible) { + setSelectedIndex(0); + } + }, [isVisible]); + + // Format option for display + const formatItem = (option: McpOption, isSelected: boolean) => ( + <> + <Text>{option.icon} </Text> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {option.label} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> - {option.description}</Text> + </> + ); + + // Handle selection + const handleSelect = (option: McpOption) => { + onSelect(option.action); + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={options} + isVisible={isVisible} + isLoading={false} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="MCP Servers" + borderColor="magenta" + emptyMessage="No options available" + /> + ); +}); + +export default McpSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpServerActions.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpServerActions.tsx new file mode 100644 index 00000000..8888a304 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpServerActions.tsx @@ -0,0 +1,157 @@ +/** + * McpServerActions Component + * Shows actions for a selected MCP server: enable/disable, delete + * Second screen when selecting a server from McpServerList + */ + +import React, { + useState, + useEffect, + forwardRef, + useRef, + useImperativeHandle, + useMemo, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { McpServerStatus } from '@dexto/core'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type McpServerActionType = 'enable' | 'disable' | 'delete' | 'back'; + +export interface McpServerAction { + type: McpServerActionType; + server: McpServerStatus; +} + +interface McpServerActionsProps { + isVisible: boolean; + server: McpServerStatus | null; + onAction: (action: McpServerAction) => void; + onClose: () => void; +} + +export interface McpServerActionsHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ActionItem { + id: string; + type: McpServerActionType; + label: string; + icon: string; + color: string; +} + +/** + * MCP Server Actions - enable/disable, delete for a specific server + */ +const McpServerActions = forwardRef<McpServerActionsHandle, McpServerActionsProps>( + function McpServerActions({ isVisible, server, onAction, onClose }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + // Reset selection when becoming visible or server changes + useEffect(() => { + if (isVisible) { + setSelectedIndex(0); + } + }, [isVisible, server]); + + // Build action items based on server state + const items = useMemo<ActionItem[]>(() => { + if (!server) return []; + + const actions: ActionItem[] = []; + + // Enable/Disable based on current state + if (server.enabled) { + actions.push({ + id: 'disable', + type: 'disable', + label: 'Disable server', + icon: '⏸️', + color: 'orange', + }); + } else { + actions.push({ + id: 'enable', + type: 'enable', + label: 'Enable server', + icon: '▶️', + color: 'green', + }); + } + + // Delete option + actions.push({ + id: 'delete', + type: 'delete', + label: 'Delete server', + icon: '🗑️', + color: 'red', + }); + + // Back option + actions.push({ + id: 'back', + type: 'back', + label: 'Back to server list', + icon: '←', + color: 'gray', + }); + + return actions; + }, [server]); + + // Format item for display + const formatItem = (item: ActionItem, isSelected: boolean) => { + return ( + <Box> + <Text>{item.icon} </Text> + <Text color={isSelected ? item.color : 'gray'} bold={isSelected}> + {item.label} + </Text> + </Box> + ); + }; + + // Handle selection + const handleSelect = (item: ActionItem) => { + if (!server) return; + onAction({ type: item.type, server }); + }; + + if (!server) return null; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={items} + isVisible={isVisible} + isLoading={false} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title={`Server: ${server.name}`} + borderColor="magenta" + emptyMessage="No actions available" + /> + ); + } +); + +export default McpServerActions; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpServerList.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpServerList.tsx new file mode 100644 index 00000000..11f9c559 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/McpServerList.tsx @@ -0,0 +1,198 @@ +/** + * McpServerList Component + * Shows list of configured MCP servers with their status + * First screen of /mcp command - select a server or add new + */ + +import React, { + useState, + useEffect, + forwardRef, + useRef, + useImperativeHandle, + useMemo, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent, McpServerStatus, McpConnectionStatus } from '@dexto/core'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type McpServerListAction = + | { type: 'select-server'; server: McpServerStatus } + | { type: 'add-new' }; + +interface McpServerListProps { + isVisible: boolean; + onAction: (action: McpServerListAction) => void; + onClose: () => void; + agent: DextoAgent; +} + +export interface McpServerListHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ListItem { + id: string; + isAddNew: boolean; + server?: McpServerStatus; +} + +/** + * Get status icon based on server state + */ +function getStatusIcon(status: McpConnectionStatus): string { + switch (status) { + case 'connected': + return '🟢'; + case 'disconnected': + return '⚪'; + case 'error': + return '🔴'; + } +} + +/** + * Get status text based on server state + */ +function getStatusText(status: McpConnectionStatus): string { + switch (status) { + case 'connected': + return 'connected'; + case 'disconnected': + return 'disabled'; + case 'error': + return 'failed'; + } +} + +/** + * MCP Server List - shows all configured servers and add option + */ +const McpServerList = forwardRef<McpServerListHandle, McpServerListProps>(function McpServerList( + { isVisible, onAction, onClose, agent }, + ref +) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + const [selectedIndex, setSelectedIndex] = useState(0); + const [servers, setServers] = useState<McpServerStatus[]>([]); + const [isLoading, setIsLoading] = useState(true); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + // Load servers when becoming visible + useEffect(() => { + if (isVisible) { + setIsLoading(true); + setSelectedIndex(0); + + // Get all servers with computed status from agent + const serverList = agent.getMcpServersWithStatus(); + + // Sort: connected first, then disconnected, then error + serverList.sort((a, b) => { + const order: Record<McpConnectionStatus, number> = { + connected: 0, + disconnected: 1, + error: 2, + }; + return order[a.status] - order[b.status]; + }); + + setServers(serverList); + setIsLoading(false); + } + }, [isVisible, agent]); + + // Build list items: servers + "Add new server" at bottom + const items = useMemo<ListItem[]>(() => { + const list: ListItem[] = servers.map((server) => ({ + id: server.name, + isAddNew: false, + server, + })); + + // Add "Add new server" option at the end + list.push({ + id: '__add_new__', + isAddNew: true, + }); + + return list; + }, [servers]); + + // Format item for display + const formatItem = (item: ListItem, isSelected: boolean) => { + if (item.isAddNew) { + return ( + <Box> + <Text color={isSelected ? 'green' : 'gray'} bold={isSelected}> + + Add new server + </Text> + </Box> + ); + } + + const server = item.server!; + const statusIcon = getStatusIcon(server.status); + const statusText = getStatusText(server.status); + + return ( + <Box> + <Text>{statusIcon} </Text> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {server.name} + </Text> + <Text color="gray"> ({server.type}) </Text> + <Text + color={ + server.status === 'connected' + ? 'green' + : server.status === 'disconnected' + ? 'gray' + : 'red' + } + > + [{statusText}] + </Text> + </Box> + ); + }; + + // Handle selection + const handleSelect = (item: ListItem) => { + if (item.isAddNew) { + onAction({ type: 'add-new' }); + } else if (item.server) { + onAction({ type: 'select-server', server: item.server }); + } + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={items} + isVisible={isVisible} + isLoading={isLoading} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="MCP Servers" + borderColor="magenta" + emptyMessage="No servers configured" + /> + ); +}); + +export default McpServerList; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/ModelSelectorRefactored.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ModelSelectorRefactored.tsx new file mode 100644 index 00000000..3a8e436a --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ModelSelectorRefactored.tsx @@ -0,0 +1,821 @@ +/** + * ModelSelector Component (Refactored) + * Features: + * - Search filtering + * - Custom models support (add/edit/delete via arrow navigation) + */ + +import { + useState, + useEffect, + forwardRef, + useRef, + useImperativeHandle, + useMemo, + useCallback, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent, LLMProvider } from '@dexto/core'; +import { + listOllamaModels, + DEFAULT_OLLAMA_URL, + getLocalModelById, + isReasoningCapableModel, +} from '@dexto/core'; +import { + loadCustomModels, + deleteCustomModel, + getAllInstalledModels, + type CustomModel, +} from '@dexto/agent-management'; + +type ReasoningEffort = 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; + +interface ModelSelectorProps { + isVisible: boolean; + onSelectModel: ( + provider: LLMProvider, + model: string, + displayName?: string, + baseURL?: string, + reasoningEffort?: ReasoningEffort + ) => void; + onClose: () => void; + onAddCustomModel: () => void; + onEditCustomModel: (model: CustomModel) => void; + agent: DextoAgent; +} + +export interface ModelSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ModelOption { + provider: LLMProvider; + name: string; + displayName: string | undefined; + maxInputTokens: number; + isDefault: boolean; + isCurrent: boolean; + isCustom: boolean; + baseURL?: string; + /** For gateway providers like dexto, the original provider this model comes from */ + originalProvider?: LLMProvider; +} + +// Special option for adding custom model +interface AddCustomOption { + type: 'add-custom'; +} + +type SelectorItem = ModelOption | AddCustomOption; + +function isAddCustomOption(item: SelectorItem): item is AddCustomOption { + return 'type' in item && item.type === 'add-custom'; +} + +const MAX_VISIBLE_ITEMS = 10; + +// Reasoning effort options - defined at module scope to avoid recreation on each render +const REASONING_EFFORT_OPTIONS: { + value: ReasoningEffort | 'auto'; + label: string; + description: string; +}[] = [ + { + value: 'auto', + label: 'Auto', + description: 'Let the model decide (recommended for most tasks)', + }, + { value: 'none', label: 'None', description: 'No reasoning, fastest responses' }, + { value: 'minimal', label: 'Minimal', description: 'Barely any reasoning, very fast' }, + { value: 'low', label: 'Low', description: 'Light reasoning, fast responses' }, + { + value: 'medium', + label: 'Medium', + description: 'Balanced reasoning (OpenAI recommended)', + }, + { value: 'high', label: 'High', description: 'Thorough reasoning for complex tasks' }, + { value: 'xhigh', label: 'Extra High', description: 'Maximum quality, slower/costlier' }, +]; + +/** + * Model selector with search and custom model support + */ +const ModelSelector = forwardRef<ModelSelectorHandle, ModelSelectorProps>(function ModelSelector( + { isVisible, onSelectModel, onClose, onAddCustomModel, onEditCustomModel, agent }, + ref +) { + const [models, setModels] = useState<ModelOption[]>([]); + const [customModels, setCustomModels] = useState<CustomModel[]>([]); + const [isLoading, setIsLoading] = useState(false); + const [selectedIndex, setSelectedIndex] = useState(0); + const [searchQuery, setSearchQuery] = useState(''); + const [scrollOffset, setScrollOffset] = useState(0); + const [customModelAction, setCustomModelAction] = useState<'edit' | 'delete' | null>(null); + const [pendingDeleteConfirm, setPendingDeleteConfirm] = useState(false); + const selectedIndexRef = useRef(selectedIndex); + const deleteTimeoutRef = useRef<NodeJS.Timeout | null>(null); + + // Reasoning effort sub-step state + const [pendingReasoningModel, setPendingReasoningModel] = useState<ModelOption | null>(null); + const [reasoningEffortIndex, setReasoningEffortIndex] = useState(0); // Default to 'Auto' (index 0) + + // Keep ref in sync + selectedIndexRef.current = selectedIndex; + + // Clear delete confirmation timeout on unmount + useEffect(() => { + return () => { + if (deleteTimeoutRef.current) { + clearTimeout(deleteTimeoutRef.current); + } + }; + }, []); + + // Fetch models from agent and load custom models + useEffect(() => { + if (!isVisible) return; + + let cancelled = false; + setIsLoading(true); + setSearchQuery(''); + setSelectedIndex(0); + setScrollOffset(0); + setCustomModelAction(null); + setPendingDeleteConfirm(false); + setPendingReasoningModel(null); + setReasoningEffortIndex(0); // Default to 'Auto' + if (deleteTimeoutRef.current) { + clearTimeout(deleteTimeoutRef.current); + deleteTimeoutRef.current = null; + } + + const fetchModels = async () => { + try { + const [allModels, providers, currentConfig, loadedCustomModels] = await Promise.all( + [ + Promise.resolve(agent.getSupportedModels()), + Promise.resolve(agent.getSupportedProviders()), + Promise.resolve(agent.getCurrentLLMConfig()), + loadCustomModels(), + ] + ); + + const modelList: ModelOption[] = []; + + // Fetch dynamic models for local providers + let ollamaModels: Array<{ name: string; size?: number }> = []; + let localModels: Array<{ id: string; filePath: string; sizeBytes: number }> = []; + + try { + ollamaModels = await listOllamaModels(DEFAULT_OLLAMA_URL); + } catch (error) { + // Ollama not available, skip + agent.logger.debug('Ollama not available for model listing'); + } + + try { + localModels = await getAllInstalledModels(); + } catch (error) { + // Local models not available, skip + agent.logger.debug('Local models not available for listing'); + } + + // Add custom models first + for (const custom of loadedCustomModels) { + // Use provider from custom model, default to openai-compatible for legacy models + const customProvider = (custom.provider ?? 'openai-compatible') as LLMProvider; + const modelOption: ModelOption = { + provider: customProvider, + name: custom.name, + displayName: custom.displayName || custom.name, + maxInputTokens: custom.maxInputTokens || 128000, + isDefault: false, + isCurrent: + currentConfig.provider === customProvider && + currentConfig.model === custom.name, + isCustom: true, + }; + if (custom.baseURL) { + modelOption.baseURL = custom.baseURL; + } + modelList.push(modelOption); + } + + // Add registry models + for (const provider of providers) { + // Skip custom-only providers that don't have a static model list + // These are only accessible via the "Add custom model" wizard + // Note: 'dexto' is NOT skipped because it has supportsAllRegistryModels + if ( + provider === 'openai-compatible' || + provider === 'openrouter' || + provider === 'litellm' || + provider === 'glama' || + provider === 'bedrock' + ) + continue; + + // Skip ollama, local, and vertex - they'll be added dynamically below + if (provider === 'ollama' || provider === 'local' || provider === 'vertex') { + continue; + } + + const providerModels = allModels[provider]; + for (const model of providerModels) { + // For dexto provider, models have originalProvider field + // showing which provider the model originally came from + const originalProvider = + 'originalProvider' in model ? model.originalProvider : undefined; + + modelList.push({ + provider, + name: model.name, + displayName: model.displayName, + maxInputTokens: model.maxInputTokens, + isDefault: model.isDefault, + isCurrent: + provider === currentConfig.provider && + model.name === currentConfig.model, + isCustom: false, + // Store original provider for display purposes + ...(originalProvider && { originalProvider }), + }); + } + } + + // Add Ollama models dynamically + for (const ollamaModel of ollamaModels) { + modelList.push({ + provider: 'ollama', + name: ollamaModel.name, + displayName: ollamaModel.name, + maxInputTokens: 128000, // Default, actual varies by model + isDefault: false, + isCurrent: + currentConfig.provider === 'ollama' && + currentConfig.model === ollamaModel.name, + isCustom: false, + }); + } + + // Add local models dynamically + for (const localModel of localModels) { + // Get display name from registry if available + const modelInfo = getLocalModelById(localModel.id); + const displayName = modelInfo?.name || localModel.id; + const maxInputTokens = modelInfo?.contextLength || 128000; + + modelList.push({ + provider: 'local', + name: localModel.id, + displayName, + maxInputTokens, + isDefault: false, + isCurrent: + currentConfig.provider === 'local' && + currentConfig.model === localModel.id, + isCustom: false, + }); + } + + // Add Vertex AI models from registry + const vertexModels = allModels['vertex']; + if (vertexModels) { + for (const model of vertexModels) { + modelList.push({ + provider: 'vertex', + name: model.name, + displayName: model.displayName, + maxInputTokens: model.maxInputTokens, + isDefault: model.isDefault, + isCurrent: + currentConfig.provider === 'vertex' && + currentConfig.model === model.name, + isCustom: false, + }); + } + } + + if (!cancelled) { + setModels(modelList); + setCustomModels(loadedCustomModels); + setIsLoading(false); + + // Set initial selection to current model (offset by 1 for "Add custom" option) + const currentIndex = modelList.findIndex((m) => m.isCurrent); + if (currentIndex >= 0) { + setSelectedIndex(currentIndex + 1); // +1 for "Add custom" at top + } + } + } catch (error) { + if (!cancelled) { + agent.logger.error( + `Failed to fetch models: ${error instanceof Error ? error.message : 'Unknown error'}` + ); + setModels([]); + setIsLoading(false); + } + } + }; + + void fetchModels(); + + return () => { + cancelled = true; + }; + }, [isVisible, agent]); + + // Filter models based on search query + const filteredItems = useMemo((): SelectorItem[] => { + const addCustomOption: AddCustomOption = { type: 'add-custom' }; + + if (!searchQuery.trim()) { + return [addCustomOption, ...models]; + } + + const query = searchQuery.toLowerCase(); + const filtered = models.filter((model) => { + const name = model.name.toLowerCase(); + const displayName = (model.displayName || '').toLowerCase(); + const provider = model.provider.toLowerCase(); + return name.includes(query) || displayName.includes(query) || provider.includes(query); + }); + + // Always show "Add custom" when searching (user might want to add what they're searching for) + return [addCustomOption, ...filtered]; + }, [models, searchQuery]); + + // Adjust selected index when filter changes + useEffect(() => { + if (selectedIndex >= filteredItems.length) { + setSelectedIndex(Math.max(0, filteredItems.length - 1)); + } + }, [filteredItems.length, selectedIndex]); + + // Calculate scroll offset + useEffect(() => { + if (selectedIndex < scrollOffset) { + setScrollOffset(selectedIndex); + } else if (selectedIndex >= scrollOffset + MAX_VISIBLE_ITEMS) { + setScrollOffset(selectedIndex - MAX_VISIBLE_ITEMS + 1); + } + }, [selectedIndex, scrollOffset]); + + // Handle delete custom model + const handleDeleteCustomModel = useCallback( + async (model: ModelOption) => { + if (!model.isCustom) return; + + try { + await deleteCustomModel(model.name); + // Refresh the list + const updated = await loadCustomModels(); + setCustomModels(updated); + // Update models list + setModels((prev) => prev.filter((m) => !(m.isCustom && m.name === model.name))); + } catch (error) { + agent.logger.error( + `Failed to delete custom model: ${error instanceof Error ? error.message : 'Unknown error'}` + ); + } + }, + [agent] + ); + + // Helper to clear action state + const clearActionState = () => { + setCustomModelAction(null); + setPendingDeleteConfirm(false); + if (deleteTimeoutRef.current) { + clearTimeout(deleteTimeoutRef.current); + deleteTimeoutRef.current = null; + } + }; + + // Expose handleInput method via ref + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Handle reasoning effort sub-step + if (pendingReasoningModel) { + if (key.escape) { + // Go back to model selection + setPendingReasoningModel(null); + return true; + } + if (key.upArrow) { + setReasoningEffortIndex((prev) => + prev > 0 ? prev - 1 : REASONING_EFFORT_OPTIONS.length - 1 + ); + return true; + } + if (key.downArrow) { + setReasoningEffortIndex((prev) => + prev < REASONING_EFFORT_OPTIONS.length - 1 ? prev + 1 : 0 + ); + return true; + } + if (key.return) { + const selectedOption = REASONING_EFFORT_OPTIONS[reasoningEffortIndex]; + const reasoningEffort = + selectedOption?.value === 'auto' ? undefined : selectedOption?.value; + onSelectModel( + pendingReasoningModel.provider, + pendingReasoningModel.name, + pendingReasoningModel.displayName, + pendingReasoningModel.baseURL, + reasoningEffort + ); + setPendingReasoningModel(null); + return true; + } + return true; // Consume all input in reasoning effort mode + } + + // Escape always works + if (key.escape) { + // If in action mode, just clear it first + if (customModelAction) { + clearActionState(); + return true; + } + onClose(); + return true; + } + + const itemsLength = filteredItems.length; + const currentItem = filteredItems[selectedIndexRef.current]; + const isOnCustomModel = + currentItem && !isAddCustomOption(currentItem) && currentItem.isCustom; + + // Right arrow - enter/advance action mode for custom models + if (key.rightArrow) { + if (!isOnCustomModel) return false; + + if (customModelAction === null) { + // Enter edit mode + setCustomModelAction('edit'); + return true; + } else if (customModelAction === 'edit') { + // Advance to delete mode + setCustomModelAction('delete'); + setPendingDeleteConfirm(false); + return true; + } else if (customModelAction === 'delete') { + // In delete mode, right arrow confirms deletion + if (pendingDeleteConfirm) { + // Second press - actually delete + if (deleteTimeoutRef.current) { + clearTimeout(deleteTimeoutRef.current); + deleteTimeoutRef.current = null; + } + clearActionState(); + void handleDeleteCustomModel(currentItem as ModelOption); + } else { + // First press in delete mode - set pending confirmation + setPendingDeleteConfirm(true); + if (deleteTimeoutRef.current) { + clearTimeout(deleteTimeoutRef.current); + } + deleteTimeoutRef.current = setTimeout(() => { + setPendingDeleteConfirm(false); + deleteTimeoutRef.current = null; + }, 3000); // 3 second timeout + } + return true; + } + } + + // Left arrow - go back in action mode + if (key.leftArrow) { + if (customModelAction === 'delete') { + setCustomModelAction('edit'); + setPendingDeleteConfirm(false); + if (deleteTimeoutRef.current) { + clearTimeout(deleteTimeoutRef.current); + deleteTimeoutRef.current = null; + } + return true; + } else if (customModelAction === 'edit') { + setCustomModelAction(null); + return true; + } + return false; + } + + // Handle character input for search + if (input && !key.return && !key.upArrow && !key.downArrow && !key.tab) { + // Any character input clears action state and adds to search + if (customModelAction) { + clearActionState(); + } + + // Backspace + if (key.backspace || key.delete) { + setSearchQuery((prev) => prev.slice(0, -1)); + return true; + } + + // Regular character - add to search + if (input.length === 1 && input.charCodeAt(0) >= 32) { + setSearchQuery((prev) => prev + input); + setSelectedIndex(0); + setScrollOffset(0); + return true; + } + } + + // Backspace when no other input + if (key.backspace || key.delete) { + setSearchQuery((prev) => prev.slice(0, -1)); + return true; + } + + if (itemsLength === 0) return false; + + if (key.upArrow) { + // Clear action state on vertical navigation + if (customModelAction) { + clearActionState(); + } + const nextIndex = (selectedIndexRef.current - 1 + itemsLength) % itemsLength; + setSelectedIndex(nextIndex); + selectedIndexRef.current = nextIndex; + return true; + } + + if (key.downArrow) { + // Clear action state on vertical navigation + if (customModelAction) { + clearActionState(); + } + const nextIndex = (selectedIndexRef.current + 1) % itemsLength; + setSelectedIndex(nextIndex); + selectedIndexRef.current = nextIndex; + return true; + } + + if (key.return && itemsLength > 0) { + const item = filteredItems[selectedIndexRef.current]; + if (item) { + if (isAddCustomOption(item)) { + onAddCustomModel(); + return true; + } + + // Handle action mode confirmations + if (customModelAction === 'edit' && item.isCustom) { + // Find the full custom model data + const customModel = customModels.find( + (cm) => + cm.name === item.name && + (cm.provider ?? 'openai-compatible') === item.provider + ); + if (customModel) { + onEditCustomModel(customModel); + } + return true; + } + + if (customModelAction === 'delete' && item.isCustom) { + if (pendingDeleteConfirm) { + // Already confirmed, delete + clearActionState(); + void handleDeleteCustomModel(item); + } else { + // Set pending confirmation + setPendingDeleteConfirm(true); + if (deleteTimeoutRef.current) { + clearTimeout(deleteTimeoutRef.current); + } + deleteTimeoutRef.current = setTimeout(() => { + setPendingDeleteConfirm(false); + deleteTimeoutRef.current = null; + }, 3000); + } + return true; + } + + // Normal selection - check if reasoning-capable + if (isReasoningCapableModel(item.name)) { + // Show reasoning effort sub-step + setPendingReasoningModel(item); + setReasoningEffortIndex(0); // Default to 'Auto' + return true; + } + onSelectModel(item.provider, item.name, item.displayName, item.baseURL); + return true; + } + } + + return false; + }, + }), + [ + isVisible, + filteredItems, + onClose, + onSelectModel, + onAddCustomModel, + onEditCustomModel, + customModelAction, + pendingDeleteConfirm, + customModels, + handleDeleteCustomModel, + pendingReasoningModel, + reasoningEffortIndex, + ] + ); + + if (!isVisible) return null; + + if (isLoading) { + return ( + <Box paddingX={0} paddingY={0}> + <Text color="gray">Loading models...</Text> + </Box> + ); + } + + // Reasoning effort sub-step UI + if (pendingReasoningModel) { + return ( + <Box flexDirection="column"> + <Box paddingX={0} paddingY={0}> + <Text color="cyan" bold> + Configure Reasoning Effort + </Text> + </Box> + <Box paddingX={0} paddingY={0}> + <Text color="gray"> + for {pendingReasoningModel.displayName || pendingReasoningModel.name} + </Text> + </Box> + <Box paddingX={0} paddingY={0}> + <Text color="gray">↑↓ navigate, Enter select, Esc back</Text> + </Box> + <Box paddingX={0} paddingY={0}> + <Text color="gray">{'─'.repeat(50)}</Text> + </Box> + {REASONING_EFFORT_OPTIONS.map((option, index) => { + const isSelected = index === reasoningEffortIndex; + return ( + <Box key={option.value} paddingX={0} paddingY={0}> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {isSelected ? '› ' : ' '} + {option.label} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> + {' '} + - {option.description} + </Text> + </Box> + ); + })} + </Box> + ); + } + + const visibleItems = filteredItems.slice(scrollOffset, scrollOffset + MAX_VISIBLE_ITEMS); + const hasCustomModels = customModels.length > 0; + + return ( + <Box flexDirection="column"> + {/* Header */} + <Box paddingX={0} paddingY={0}> + <Text color="cyan" bold> + Select Model ({selectedIndex + 1}/{filteredItems.length}) + </Text> + </Box> + {/* Navigation hints */} + <Box paddingX={0} paddingY={0}> + <Text color="gray">↑↓ navigate, Enter select, Esc close</Text> + {hasCustomModels && <Text color="gray">, →← for custom actions</Text>} + </Box> + + {/* Search input */} + <Box paddingX={0} paddingY={0}> + <Text color="gray">🔍 </Text> + <Text color={searchQuery ? 'white' : 'gray'}> + {searchQuery || 'Type to search...'} + </Text> + <Text color="cyan">▌</Text> + </Box> + + {/* Separator */} + <Box paddingX={0} paddingY={0}> + <Text color="gray">{'─'.repeat(50)}</Text> + </Box> + + {/* Items */} + {visibleItems.map((item, visibleIndex) => { + const actualIndex = scrollOffset + visibleIndex; + const isSelected = actualIndex === selectedIndex; + + if (isAddCustomOption(item)) { + return ( + <Box key="add-custom" paddingX={0} paddingY={0}> + <Text color={isSelected ? 'green' : 'gray'} bold={isSelected}> + ➕ Add custom model... + </Text> + </Box> + ); + } + + // Show action buttons for selected custom models + const showActions = isSelected && item.isCustom; + + // Show original provider for gateway models (e.g., dexto showing openai models) + const providerDisplay = item.originalProvider + ? `${item.originalProvider} via ${item.provider}` + : item.provider; + + return ( + <Box key={`${item.provider}-${item.name}`} paddingX={0} paddingY={0}> + {item.isCustom && <Text color={isSelected ? 'orange' : 'gray'}>★ </Text>} + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {item.displayName || item.name} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> ({providerDisplay})</Text> + <Text color={isSelected ? 'white' : 'gray'}> + {' '} + • {item.maxInputTokens.toLocaleString()} tokens + </Text> + {item.isDefault && ( + <Text color={isSelected ? 'white' : 'gray'}> [DEFAULT]</Text> + )} + {item.isCurrent && !showActions && ( + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {' '} + ← Current + </Text> + )} + {/* Action buttons for custom models - always shown when selected */} + {showActions && ( + <> + <Text> </Text> + <Text + color={customModelAction === 'edit' ? 'green' : 'gray'} + bold={customModelAction === 'edit'} + inverse={customModelAction === 'edit'} + > + {' '} + Edit{' '} + </Text> + <Text> </Text> + <Text + color={customModelAction === 'delete' ? 'red' : 'gray'} + bold={customModelAction === 'delete'} + inverse={customModelAction === 'delete'} + > + {' '} + Delete{' '} + </Text> + </> + )} + </Box> + ); + })} + + {/* Scroll indicator */} + {filteredItems.length > MAX_VISIBLE_ITEMS && ( + <Box paddingX={0} paddingY={0}> + <Text color="gray"> + {scrollOffset > 0 ? '↑ more above' : ''} + {scrollOffset > 0 && scrollOffset + MAX_VISIBLE_ITEMS < filteredItems.length + ? ' | ' + : ''} + {scrollOffset + MAX_VISIBLE_ITEMS < filteredItems.length + ? '↓ more below' + : ''} + </Text> + </Box> + )} + + {/* Delete confirmation message */} + {customModelAction === 'delete' && pendingDeleteConfirm && ( + <Box paddingX={0} paddingY={0} marginTop={1}> + <Text color="yellowBright">⚠️ Press → or Enter again to confirm delete</Text> + </Box> + )} + {/* Action mode hints */} + {customModelAction && !pendingDeleteConfirm && ( + <Box paddingX={0} paddingY={0} marginTop={1}> + <Text color="gray"> + ← {customModelAction === 'edit' ? 'deselect' : 'edit'} | →{' '} + {customModelAction === 'edit' ? 'delete' : 'confirm'} | Enter{' '} + {customModelAction} + </Text> + </Box> + )} + </Box> + ); +}); + +export default ModelSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/PluginActions.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PluginActions.tsx new file mode 100644 index 00000000..728fad16 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PluginActions.tsx @@ -0,0 +1,145 @@ +/** + * PluginActions Component + * Clean action menu for a selected plugin + */ + +import React, { + useState, + useEffect, + forwardRef, + useRef, + useImperativeHandle, + useMemo, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { ListedPlugin } from '@dexto/agent-management'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type PluginActionType = 'uninstall' | 'back'; + +export interface PluginActionResult { + type: PluginActionType; + plugin: ListedPlugin; +} + +interface PluginActionsProps { + isVisible: boolean; + plugin: ListedPlugin | null; + onAction: (action: PluginActionResult) => void; + onClose: () => void; +} + +export interface PluginActionsHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ActionItem { + id: string; + type: PluginActionType; + label: string; + icon: string; + color: string; +} + +/** + * Plugin Actions - action menu for a specific plugin + */ +const PluginActions = forwardRef<PluginActionsHandle, PluginActionsProps>(function PluginActions( + { isVisible, plugin, onAction, onClose }, + ref +) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + // Reset selection when becoming visible or plugin changes + useEffect(() => { + if (isVisible) { + setSelectedIndex(0); + } + }, [isVisible, plugin]); + + // Build action items + const items = useMemo<ActionItem[]>(() => { + if (!plugin) return []; + + return [ + { + id: 'back', + type: 'back' as const, + label: 'Back to list', + icon: '←', + color: 'gray', + }, + { + id: 'uninstall', + type: 'uninstall' as const, + label: 'Uninstall', + icon: '🗑', + color: 'red', + }, + ]; + }, [plugin]); + + // Format item for display - clean single line + const formatItem = (item: ActionItem, isSelected: boolean) => { + const isBack = item.type === 'back'; + + return ( + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}>{isSelected ? '▸ ' : ' '}</Text> + <Text color={isBack ? 'gray' : isSelected ? item.color : 'gray'}>{item.icon} </Text> + <Text + color={ + isBack ? (isSelected ? 'white' : 'gray') : isSelected ? item.color : 'white' + } + bold={isSelected && !isBack} + > + {item.label} + </Text> + </Box> + ); + }; + + // Handle selection + const handleSelect = (item: ActionItem) => { + if (!plugin) return; + onAction({ type: item.type, plugin }); + }; + + if (!plugin) return null; + + const version = plugin.version || 'unknown'; + const scopeBadge = plugin.scope ? ` [${plugin.scope}]` : ''; + const title = `📦 ${plugin.name}@${version}${scopeBadge}`; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={items} + isVisible={isVisible} + isLoading={false} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title={title} + borderColor="magenta" + emptyMessage="No actions available" + /> + ); +}); + +export default PluginActions; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/PluginList.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PluginList.tsx new file mode 100644 index 00000000..4ec1a6aa --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PluginList.tsx @@ -0,0 +1,167 @@ +/** + * PluginList Component + * Clean table-like view of installed plugins + */ + +import React, { + useState, + useEffect, + forwardRef, + useRef, + useImperativeHandle, + useMemo, +} from 'react'; +import { Box, Text } from 'ink'; +import { listInstalledPlugins, type ListedPlugin } from '@dexto/agent-management'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +interface PluginListProps { + isVisible: boolean; + onPluginSelect: (plugin: ListedPlugin) => void; + onClose: () => void; +} + +export interface PluginListHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface BackItem { + type: 'back'; +} + +interface PluginItem { + type: 'plugin'; + plugin: ListedPlugin; +} + +type ListItem = BackItem | PluginItem; + +/** + * Plugin list overlay - shows installed plugins in clean table format + */ +const PluginList = forwardRef<PluginListHandle, PluginListProps>(function PluginList( + { isVisible, onPluginSelect, onClose }, + ref +) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [plugins, setPlugins] = useState<ListedPlugin[]>([]); + const [selectedIndex, setSelectedIndex] = useState(0); + const [isLoading, setIsLoading] = useState(true); + + // Load plugins when visible + useEffect(() => { + if (isVisible) { + setIsLoading(true); + try { + const installedPlugins = listInstalledPlugins(); + setPlugins(installedPlugins); + } catch { + setPlugins([]); + } finally { + setIsLoading(false); + setSelectedIndex(0); + } + } + }, [isVisible]); + + // Build list items with back option + const items = useMemo<ListItem[]>(() => { + const list: ListItem[] = [{ type: 'back' }]; + list.push(...plugins.map((plugin) => ({ type: 'plugin' as const, plugin }))); + return list; + }, [plugins]); + + // Format item for display - clean single line with optional details + const formatItem = (item: ListItem, isSelected: boolean) => { + if (item.type === 'back') { + return ( + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}>{isSelected ? '▸ ' : ' '}</Text> + <Text color="gray">← </Text> + <Text color={isSelected ? 'white' : 'gray'}>Back to menu</Text> + </Box> + ); + } + + const plugin = item.plugin; + const version = plugin.version || 'unknown'; + const scopeBadge = plugin.scope ? ` [${plugin.scope}]` : ''; + + return ( + <Box flexDirection="column"> + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}>{isSelected ? '▸ ' : ' '}</Text> + <Text color={isSelected ? 'white' : 'gray'}>📦 </Text> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {plugin.name} + </Text> + <Text color="gray" dimColor> + @{version} + </Text> + {scopeBadge && ( + <Text color="yellow" dimColor> + {scopeBadge} + </Text> + )} + </Box> + {/* Show description and path only when selected */} + {isSelected && plugin.description && ( + <Box marginLeft={4}> + <Text color="gray">{plugin.description}</Text> + </Box> + )} + {isSelected && ( + <Box marginLeft={4}> + <Text color="gray" dimColor> + {plugin.path} + </Text> + </Box> + )} + </Box> + ); + }; + + // Handle selection + const handleSelect = (item: ListItem) => { + if (item.type === 'back') { + onClose(); + } else { + onPluginSelect(item.plugin); + } + }; + + const pluginCount = plugins.length; + const title = pluginCount > 0 ? `Installed Plugins (${pluginCount})` : 'Installed Plugins'; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={items} + isVisible={isVisible} + isLoading={isLoading} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title={title} + borderColor="cyan" + emptyMessage="No plugins installed. Browse the marketplace to find plugins." + /> + ); +}); + +export default PluginList; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/PluginManager.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PluginManager.tsx new file mode 100644 index 00000000..5b3d7250 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PluginManager.tsx @@ -0,0 +1,128 @@ +/** + * PluginManager Component + * Main menu for plugin management - clean, minimal interface + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type PluginAction = 'list' | 'marketplace' | 'back'; + +interface PluginManagerProps { + isVisible: boolean; + onAction: (action: PluginAction) => void; + onClose: () => void; +} + +export interface PluginManagerHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface PluginOption { + action: PluginAction; + label: string; + hint: string; + icon: string; +} + +const PLUGIN_OPTIONS: PluginOption[] = [ + { + action: 'list', + label: 'Installed Plugins', + hint: 'View, manage, uninstall', + icon: '📦', + }, + { + action: 'marketplace', + label: 'Browse Marketplace', + hint: 'Find and install plugins', + icon: '🛒', + }, + { + action: 'back', + label: 'Back', + hint: '', + icon: '←', + }, +]; + +/** + * Plugin manager overlay - main menu for plugin management + */ +const PluginManager = forwardRef<PluginManagerHandle, PluginManagerProps>(function PluginManager( + { isVisible, onAction, onClose }, + ref +) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [selectedIndex, setSelectedIndex] = useState(0); + + // Reset selection when becoming visible + useEffect(() => { + if (isVisible) { + setSelectedIndex(0); + } + }, [isVisible]); + + // Format option for display - clean single line + const formatItem = (option: PluginOption, isSelected: boolean) => { + const isBack = option.action === 'back'; + + return ( + <Box> + <Text color={isSelected ? 'cyan' : 'gray'}>{isSelected ? '▸ ' : ' '}</Text> + <Text color={isBack ? 'gray' : isSelected ? 'white' : 'gray'}>{option.icon} </Text> + <Text color={isBack ? 'gray' : isSelected ? 'cyan' : 'white'} bold={isSelected}> + {option.label} + </Text> + {option.hint && ( + <Text color="gray" dimColor> + {' '} + — {option.hint} + </Text> + )} + </Box> + ); + }; + + // Handle selection + const handleSelect = (option: PluginOption) => { + if (option.action === 'back') { + onClose(); + } else { + onAction(option.action); + } + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={PLUGIN_OPTIONS} + isVisible={isVisible} + isLoading={false} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Plugins" + borderColor="magenta" + emptyMessage="No options available" + /> + ); +}); + +export default PluginManager; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptAddChoice.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptAddChoice.tsx new file mode 100644 index 00000000..8e554da9 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptAddChoice.tsx @@ -0,0 +1,112 @@ +/** + * PromptAddChoice Component + * Choose between adding a per-agent prompt or a shared prompt + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; +import type { PromptAddScope } from '../../state/types.js'; + +export type PromptAddChoiceResult = PromptAddScope | 'back'; + +interface PromptAddChoiceProps { + isVisible: boolean; + onSelect: (choice: PromptAddChoiceResult) => void; + onClose: () => void; +} + +export interface PromptAddChoiceHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ChoiceItem { + id: string; + scope: PromptAddChoiceResult; + label: string; + description: string; + recommended?: boolean; +} + +const CHOICES: ChoiceItem[] = [ + { + id: 'agent', + scope: 'agent', + label: 'For this agent only', + description: 'Stored in agent config directory', + recommended: true, + }, + { + id: 'shared', + scope: 'shared', + label: 'For all agents (shared)', + description: 'Stored in ~/.dexto/commands/', + }, +]; + +/** + * PromptAddChoice - select scope for new prompt + */ +const PromptAddChoice = forwardRef<PromptAddChoiceHandle, PromptAddChoiceProps>( + function PromptAddChoice({ isVisible, onSelect, onClose }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Reset selection when becoming visible + useEffect(() => { + if (isVisible) { + setSelectedIndex(0); + } + }, [isVisible]); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + // Format item for display + const formatItem = (item: ChoiceItem, isSelected: boolean) => { + return ( + <Box> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {item.label} + </Text> + {item.recommended && ( + <Text color={isSelected ? 'green' : 'gray'}> (Recommended)</Text> + )} + <Text color="gray"> · {item.description}</Text> + </Box> + ); + }; + + // Handle selection + const handleSelect = (item: ChoiceItem) => { + onSelect(item.scope); + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={CHOICES} + isVisible={isVisible} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Add Prompt" + borderColor="yellowBright" + emptyMessage="No options available" + /> + ); + } +); + +export default PromptAddChoice; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptAddWizard.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptAddWizard.tsx new file mode 100644 index 00000000..80690a30 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptAddWizard.tsx @@ -0,0 +1,271 @@ +/** + * PromptAddWizard Component + * Multi-step wizard for creating a new prompt + */ + +import React, { useState, useEffect, forwardRef, useImperativeHandle, useCallback } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { PromptAddScope } from '../../state/types.js'; + +export interface NewPromptData { + name: string; + title?: string; + description?: string; + argumentHint?: string; + content: string; +} + +interface WizardStep { + field: keyof NewPromptData; + label: string; + placeholder: string; + required: boolean; + multiline?: boolean; + validate?: (value: string) => string | null; +} + +const PROMPT_NAME_REGEX = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; + +const WIZARD_STEPS: WizardStep[] = [ + { + field: 'name', + label: 'Prompt Name', + placeholder: 'e.g., my-prompt (kebab-case)', + required: true, + validate: (v) => { + if (!v.trim()) return 'Name is required'; + if (!PROMPT_NAME_REGEX.test(v.trim())) { + return 'Name must be kebab-case (lowercase letters, numbers, hyphens)'; + } + return null; + }, + }, + { + field: 'title', + label: 'Title (optional)', + placeholder: 'e.g., My Custom Prompt', + required: false, + }, + { + field: 'description', + label: 'Description (optional)', + placeholder: 'e.g., Helps with specific task', + required: false, + }, + { + field: 'argumentHint', + label: 'Arguments (optional)', + placeholder: 'e.g., [style] [length?] - use ? for optional', + required: false, + }, + { + field: 'content', + label: 'Prompt Content', + placeholder: 'Use $1, $2 for args, $ARGUMENTS for remaining', + required: true, + multiline: true, + validate: (v) => (v.trim() ? null : 'Content is required'), + }, +]; + +interface PromptAddWizardProps { + isVisible: boolean; + scope: PromptAddScope; + onComplete: (data: NewPromptData) => void; + onClose: () => void; +} + +export interface PromptAddWizardHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * Multi-step wizard for creating a new prompt + */ +const PromptAddWizard = forwardRef<PromptAddWizardHandle, PromptAddWizardProps>( + function PromptAddWizard({ isVisible, scope, onComplete, onClose }, ref) { + const [currentStep, setCurrentStep] = useState(0); + const [values, setValues] = useState<Record<string, string>>({}); + const [currentInput, setCurrentInput] = useState(''); + const [error, setError] = useState<string | null>(null); + + // Reset when becoming visible + useEffect(() => { + if (isVisible) { + setCurrentStep(0); + setValues({}); + setCurrentInput(''); + setError(null); + } + }, [isVisible, scope]); + + const currentStepConfig = WIZARD_STEPS[currentStep]; + + const handleNext = useCallback(() => { + if (!currentStepConfig) return; + + const value = currentInput.trim(); + + // Validate + if (currentStepConfig.validate) { + const validationError = currentStepConfig.validate(value); + if (validationError) { + setError(validationError); + return; + } + } else if (currentStepConfig.required && !value) { + setError(`${currentStepConfig.label} is required`); + return; + } + + // Save value + const newValues = { ...values, [currentStepConfig.field]: value }; + setValues(newValues); + setError(null); + setCurrentInput(''); + + // Check if we're done + if (currentStep >= WIZARD_STEPS.length - 1) { + // Build data and complete + const data: NewPromptData = { + name: newValues.name || '', + content: newValues.content || '', + }; + + if (newValues.title?.trim()) { + data.title = newValues.title.trim(); + } + if (newValues.description?.trim()) { + data.description = newValues.description.trim(); + } + if (newValues.argumentHint?.trim()) { + data.argumentHint = newValues.argumentHint.trim(); + } + + onComplete(data); + } else { + setCurrentStep(currentStep + 1); + } + }, [currentInput, currentStep, currentStepConfig, onComplete, values]); + + const handleBack = useCallback(() => { + if (currentStep > 0) { + setCurrentStep(currentStep - 1); + // Restore previous value + const prevStep = WIZARD_STEPS[currentStep - 1]; + if (prevStep) { + setCurrentInput(values[prevStep.field] || ''); + } + setError(null); + } else { + onClose(); + } + }, [currentStep, onClose, values]); + + // Handle keyboard input + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape to go back/close + if (key.escape) { + handleBack(); + return true; + } + + // Enter to submit current step (or add newline in multiline mode) + if (key.return) { + if (currentStepConfig?.multiline && key.shift) { + // Shift+Enter adds newline in multiline mode + setCurrentInput((prev) => prev + '\n'); + return true; + } + handleNext(); + return true; + } + + // Backspace + if (key.backspace || key.delete) { + setCurrentInput((prev) => prev.slice(0, -1)); + setError(null); + return true; + } + + // Regular character input + if (input && !key.ctrl && !key.meta) { + setCurrentInput((prev) => prev + input); + setError(null); + return true; + } + + return false; + }, + }), + [isVisible, handleBack, handleNext, currentStepConfig] + ); + + if (!isVisible || !currentStepConfig) return null; + + const scopeLabel = scope === 'agent' ? 'Agent' : 'Shared'; + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="yellowBright" + paddingX={1} + marginTop={1} + > + {/* Header */} + <Box marginBottom={1}> + <Text bold color="yellowBright"> + Add {scopeLabel} Prompt + </Text> + <Text color="gray"> + {' '} + (Step {currentStep + 1}/{WIZARD_STEPS.length}) + </Text> + </Box> + + {/* Current step prompt */} + <Box flexDirection="column"> + <Text bold>{currentStepConfig.label}:</Text> + <Text color="gray">{currentStepConfig.placeholder}</Text> + </Box> + + {/* Input field */} + <Box marginTop={1} flexDirection="column"> + <Box> + <Text color="cyan">> </Text> + <Text>{currentInput}</Text> + <Text color="cyan">_</Text> + </Box> + {currentStepConfig.multiline && ( + <Text color="gray" italic> + (Shift+Enter for newline) + </Text> + )} + </Box> + + {/* Error message */} + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + {/* Help text */} + <Box marginTop={1}> + <Text color="gray"> + Enter to continue • Esc to {currentStep > 0 ? 'go back' : 'cancel'} + </Text> + </Box> + </Box> + ); + } +); + +export default PromptAddWizard; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptDeleteSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptDeleteSelector.tsx new file mode 100644 index 00000000..b9225370 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptDeleteSelector.tsx @@ -0,0 +1,203 @@ +/** + * PromptDeleteSelector Component + * Shows list of deletable prompts (config and shared only, not MCP) + */ + +import React, { + useState, + useEffect, + forwardRef, + useRef, + useImperativeHandle, + useMemo, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent, PromptInfo } from '@dexto/core'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export interface DeletablePrompt { + prompt: PromptInfo; + sourceType: 'config' | 'shared'; + filePath?: string | undefined; +} + +interface PromptDeleteSelectorProps { + isVisible: boolean; + onDelete: (prompt: DeletablePrompt) => void; + onClose: () => void; + agent: DextoAgent; +} + +export interface PromptDeleteSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * Check if a prompt is from commands directory (shared) + */ +function isSharedPrompt(prompt: PromptInfo): boolean { + const metadata = prompt.metadata as { filePath?: string } | undefined; + if (metadata?.filePath) { + return ( + metadata.filePath.includes('/commands/') || + metadata.filePath.includes('/.dexto/commands/') + ); + } + return false; +} + +/** + * Get file path from prompt metadata + */ +function getFilePath(prompt: PromptInfo): string | undefined { + const metadata = prompt.metadata as { filePath?: string } | undefined; + return metadata?.filePath; +} + +/** + * PromptDeleteSelector - shows deletable prompts + */ +const PromptDeleteSelector = forwardRef<PromptDeleteSelectorHandle, PromptDeleteSelectorProps>( + function PromptDeleteSelector({ isVisible, onDelete, onClose, agent }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + const [selectedIndex, setSelectedIndex] = useState(0); + const [deletablePrompts, setDeletablePrompts] = useState<DeletablePrompt[]>([]); + const [isLoading, setIsLoading] = useState(true); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + // Load deletable prompts when becoming visible + useEffect(() => { + if (isVisible) { + setIsLoading(true); + setSelectedIndex(0); + + agent + .listPrompts() + .then((promptSet) => { + // Filter to only deletable prompts (config and shared, not MCP) + const prompts = Object.values(promptSet); + const deletable: DeletablePrompt[] = []; + + for (const prompt of prompts) { + // Skip MCP prompts - they can't be deleted + if (prompt.source === 'mcp') continue; + + const filePath = getFilePath(prompt); + const isShared = isSharedPrompt(prompt); + + // Config prompts (inline or file-based in agent dir) + if (prompt.source === 'config' && !isShared) { + deletable.push({ + prompt, + sourceType: 'config', + filePath, + }); + } + // Shared prompts (from commands directory) + else if (prompt.source === 'config' && isShared) { + deletable.push({ + prompt, + sourceType: 'shared', + filePath, + }); + } + // Custom prompts (DB-stored) + else if (prompt.source === 'custom') { + deletable.push({ + prompt, + sourceType: 'config', // Treat as config for deletion purposes + }); + } + } + + // Sort: config first, then shared + deletable.sort((a, b) => { + if (a.sourceType !== b.sourceType) { + return a.sourceType === 'config' ? -1 : 1; + } + const aName = a.prompt.displayName || a.prompt.name; + const bName = b.prompt.displayName || b.prompt.name; + return aName.localeCompare(bName); + }); + + setDeletablePrompts(deletable); + setIsLoading(false); + }) + .catch(() => { + setDeletablePrompts([]); + setIsLoading(false); + }); + } + }, [isVisible, agent]); + + // Format item for display + const formatItem = (item: DeletablePrompt, isSelected: boolean) => { + const displayName = item.prompt.displayName || item.prompt.name; + const sourceLabel = item.sourceType === 'shared' ? 'shared' : 'config'; + const sourceColor = item.sourceType === 'shared' ? 'magenta' : 'blue'; + + return ( + <Box> + <Text color={isSelected ? 'red' : 'white'} bold={isSelected}> + {displayName} + </Text> + {item.prompt.title && <Text color="gray"> - {item.prompt.title}</Text>} + <Text color={sourceColor}> ({sourceLabel})</Text> + </Box> + ); + }; + + // Handle selection + const handleSelect = (item: DeletablePrompt) => { + onDelete(item); + }; + + // Build items with info message if empty + const items = useMemo(() => { + return deletablePrompts; + }, [deletablePrompts]); + + // Custom empty message + const emptyMessage = 'No deletable prompts found.\nMCP prompts cannot be deleted.'; + + return ( + <Box flexDirection="column"> + <BaseSelector + ref={baseSelectorRef} + items={items} + isVisible={isVisible} + isLoading={isLoading} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Delete Prompt" + borderColor="red" + emptyMessage={emptyMessage} + maxVisibleItems={10} + /> + {isVisible && !isLoading && items.length > 0 && ( + <Box marginTop={1}> + <Text color="gray" italic> + Note: MCP prompts cannot be deleted (they come from servers) + </Text> + </Box> + )} + </Box> + ); + } +); + +export default PromptDeleteSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptList.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptList.tsx new file mode 100644 index 00000000..78054b73 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/PromptList.tsx @@ -0,0 +1,246 @@ +/** + * PromptList Component + * Shows list of all prompts with Add/Delete actions + * Main screen of /prompts command + */ + +import React, { + useState, + useEffect, + forwardRef, + useRef, + useImperativeHandle, + useMemo, + useCallback, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent, PromptInfo } from '@dexto/core'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type PromptListAction = + | { type: 'select-prompt'; prompt: PromptInfo } + | { type: 'add-prompt' } + | { type: 'delete-prompt' }; + +interface PromptListProps { + isVisible: boolean; + onAction: (action: PromptListAction) => void; + onLoadIntoInput: (text: string) => void; + onClose: () => void; + agent: DextoAgent; +} + +export interface PromptListHandle { + handleInput: (input: string, key: Key) => boolean; + refresh: () => void; +} + +interface ListItem { + id: string; + type: 'prompt' | 'add' | 'delete'; + prompt?: PromptInfo; +} + +/** + * Get source indicator for display + */ +function getSourceIndicator(source: string): { icon: string; label: string; color: string } { + switch (source) { + case 'config': + return { icon: '📄', label: 'config', color: 'blue' }; + case 'custom': + return { icon: '✨', label: 'custom', color: 'magenta' }; + case 'mcp': + return { icon: '🔌', label: 'mcp', color: 'green' }; + default: + return { icon: '📝', label: source, color: 'gray' }; + } +} + +/** + * Check if a prompt is from commands directory (shared) + */ +function isSharedPrompt(prompt: PromptInfo): boolean { + const metadata = prompt.metadata as { filePath?: string } | undefined; + if (metadata?.filePath) { + // Normalize path separators for cross-platform compatibility (Windows uses \) + const normalizedPath = metadata.filePath.replaceAll('\\', '/'); + return ( + normalizedPath.includes('/commands/') || normalizedPath.includes('/.dexto/commands/') + ); + } + return false; +} + +/** + * PromptList - shows all prompts with Add/Delete actions + */ +const PromptList = forwardRef<PromptListHandle, PromptListProps>(function PromptList( + { isVisible, onAction, onLoadIntoInput, onClose, agent }, + ref +) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + const [selectedIndex, setSelectedIndex] = useState(0); + const [prompts, setPrompts] = useState<PromptInfo[]>([]); + const [isLoading, setIsLoading] = useState(true); + const [refreshKey, setRefreshKey] = useState(0); + + // Load prompts function + const loadPrompts = useCallback(() => { + setIsLoading(true); + setSelectedIndex(0); + + agent + .listPrompts() + .then((promptSet) => { + // Convert PromptSet to array and sort + const promptList = Object.values(promptSet); + + // Sort: config first, then custom, then mcp + promptList.sort((a, b) => { + const order = { config: 0, custom: 1, mcp: 2 }; + const aOrder = order[a.source] ?? 3; + const bOrder = order[b.source] ?? 3; + if (aOrder !== bOrder) return aOrder - bOrder; + // Within same source, sort alphabetically + return (a.displayName || a.name).localeCompare(b.displayName || b.name); + }); + + setPrompts(promptList); + setIsLoading(false); + }) + .catch((err) => { + console.error(`PromptList: Failed to load prompts: ${err}`); + setPrompts([]); + setIsLoading(false); + }); + }, [agent]); + + // Forward handleInput and refresh to ref + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + refresh: () => { + setRefreshKey((k) => k + 1); + }, + }), + [] + ); + + // Load prompts when becoming visible or refreshKey changes + useEffect(() => { + if (isVisible) { + loadPrompts(); + } + }, [isVisible, refreshKey, loadPrompts]); + + // Build list items: prompts + Add/Delete actions + const items = useMemo<ListItem[]>(() => { + const list: ListItem[] = prompts.map((prompt) => ({ + id: prompt.name, + type: 'prompt' as const, + prompt, + })); + + // Add action items at the end + list.push({ + id: '__add__', + type: 'add', + }); + + list.push({ + id: '__delete__', + type: 'delete', + }); + + return list; + }, [prompts]); + + // Format item for display + const formatItem = (item: ListItem, isSelected: boolean) => { + if (item.type === 'add') { + return ( + <Box> + <Text color={isSelected ? 'green' : 'gray'} bold={isSelected}> + + Add new prompt + </Text> + </Box> + ); + } + + if (item.type === 'delete') { + return ( + <Box> + <Text color={isSelected ? 'red' : 'gray'} bold={isSelected}> + - Delete a prompt + </Text> + </Box> + ); + } + + // Prompt item + const prompt = item.prompt!; + const displayName = prompt.displayName || prompt.name; + // For plugin skills, use namespace (plugin name) as source + const metadata = prompt.metadata as Record<string, unknown> | undefined; + const effectiveSource = metadata?.namespace ? String(metadata.namespace) : prompt.source; + const sourceInfo = getSourceIndicator(effectiveSource); + const isShared = isSharedPrompt(prompt); + const sourceLabel = isShared ? 'shared' : sourceInfo.label; + + return ( + <Box> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {displayName} + </Text> + {prompt.title && <Text color="gray"> - {prompt.title}</Text>} + <Text color={sourceInfo.color}> ({sourceLabel})</Text> + </Box> + ); + }; + + // Handle selection + const handleSelect = (item: ListItem) => { + if (item.type === 'add') { + onAction({ type: 'add-prompt' }); + } else if (item.type === 'delete') { + onAction({ type: 'delete-prompt' }); + } else if (item.prompt) { + onAction({ type: 'select-prompt', prompt: item.prompt }); + } + }; + + // Handle Tab to load into input + const handleTab = (item: ListItem) => { + if (item.type === 'prompt' && item.prompt) { + const displayName = item.prompt.displayName || item.prompt.name; + onLoadIntoInput(`/${displayName} `); + } + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={items} + isVisible={isVisible} + isLoading={isLoading} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + onTab={handleTab} + supportsTab={true} + title="Prompts" + borderColor="yellowBright" + emptyMessage="No prompts configured" + maxVisibleItems={12} + /> + ); +}); + +export default PromptList; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/SearchOverlay.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/SearchOverlay.tsx new file mode 100644 index 00000000..7cd435a9 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/SearchOverlay.tsx @@ -0,0 +1,353 @@ +/** + * SearchOverlay Component + * Interactive search overlay with real-time results and navigation + */ + +import React, { + useState, + useEffect, + useRef, + forwardRef, + useImperativeHandle, + useCallback, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent } from '@dexto/core'; +import type { SearchResult } from '@dexto/core'; + +export interface SearchOverlayProps { + isVisible: boolean; + onClose: () => void; + onSelectResult?: (result: SearchResult) => void; + agent: DextoAgent; +} + +export interface SearchOverlayHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface SearchState { + query: string; + results: SearchResult[]; + isLoading: boolean; + selectedIndex: number; + total: number; + hasMore: boolean; + error: string | null; +} + +const MAX_VISIBLE_RESULTS = 6; +const SEARCH_DEBOUNCE_MS = 300; + +/** + * Interactive search overlay - search messages across sessions + */ +const SearchOverlay = forwardRef<SearchOverlayHandle, SearchOverlayProps>(function SearchOverlay( + { isVisible, onClose, onSelectResult, agent }, + ref +) { + const [state, setState] = useState<SearchState>({ + query: '', + results: [], + isLoading: false, + selectedIndex: 0, + total: 0, + hasMore: false, + error: null, + }); + + const searchTimeoutRef = useRef<NodeJS.Timeout | null>(null); + const scrollOffset = useRef(0); + // Monotonic counter to prevent out-of-order async responses from overwriting newer results + const searchSeqRef = useRef(0); + + // Reset state when becoming visible + useEffect(() => { + if (isVisible) { + setState({ + query: '', + results: [], + isLoading: false, + selectedIndex: 0, + total: 0, + hasMore: false, + error: null, + }); + scrollOffset.current = 0; + } + }, [isVisible]); + + // Debounced search + const performSearch = useCallback( + async (query: string) => { + // Increment sequence to track this request + const seq = ++searchSeqRef.current; + + if (!query.trim()) { + setState((prev) => ({ + ...prev, + results: [], + total: 0, + hasMore: false, + isLoading: false, + error: null, + })); + return; + } + + setState((prev) => ({ ...prev, isLoading: true, error: null })); + + try { + const response = await agent.searchMessages(query, { limit: 20 }); + // Only apply results if this is still the latest request + if (seq !== searchSeqRef.current) return; + setState((prev) => ({ + ...prev, + results: response.results, + total: response.total, + hasMore: response.hasMore, + isLoading: false, + selectedIndex: 0, + })); + scrollOffset.current = 0; + } catch (error) { + // Only apply error if this is still the latest request + if (seq !== searchSeqRef.current) return; + setState((prev) => ({ + ...prev, + results: [], + total: 0, + hasMore: false, + isLoading: false, + error: error instanceof Error ? error.message : 'Search failed', + })); + } + }, + [agent] + ); + + // Handle input changes with debounce + const updateQuery = useCallback( + (newQuery: string) => { + setState((prev) => ({ ...prev, query: newQuery })); + + // Clear existing timeout + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + + // Debounce search + searchTimeoutRef.current = setTimeout(() => { + void performSearch(newQuery); + }, SEARCH_DEBOUNCE_MS); + }, + [performSearch] + ); + + // Cleanup timeout on unmount + useEffect(() => { + return () => { + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + }; + }, []); + + // Handle keyboard input + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape to close + if (key.escape) { + onClose(); + return true; + } + + // Enter to select result + if (key.return && state.results.length > 0) { + const selectedResult = state.results[state.selectedIndex]; + if (selectedResult && onSelectResult) { + onSelectResult(selectedResult); + } + onClose(); + return true; + } + + // Arrow up + if (key.upArrow) { + setState((prev) => { + const newIndex = Math.max(0, prev.selectedIndex - 1); + // Adjust scroll offset if needed + if (newIndex < scrollOffset.current) { + scrollOffset.current = newIndex; + } + return { ...prev, selectedIndex: newIndex }; + }); + return true; + } + + // Arrow down + if (key.downArrow) { + setState((prev) => { + const newIndex = Math.min(prev.results.length - 1, prev.selectedIndex + 1); + // Adjust scroll offset if needed + if (newIndex >= scrollOffset.current + MAX_VISIBLE_RESULTS) { + scrollOffset.current = newIndex - MAX_VISIBLE_RESULTS + 1; + } + return { ...prev, selectedIndex: newIndex }; + }); + return true; + } + + // Backspace + if (key.backspace || key.delete) { + updateQuery(state.query.slice(0, -1)); + return true; + } + + // Regular character input + if (input && !key.ctrl && !key.meta) { + updateQuery(state.query + input); + return true; + } + + return false; + }, + }), + [isVisible, onClose, onSelectResult, state, updateQuery] + ); + + if (!isVisible) return null; + + // Calculate visible results window + const visibleResults = state.results.slice( + scrollOffset.current, + scrollOffset.current + MAX_VISIBLE_RESULTS + ); + + return ( + <Box flexDirection="column" borderStyle="round" borderColor="cyan" paddingX={1}> + {/* Header */} + <Box marginBottom={1}> + <Text bold color="cyan"> + Search Messages + </Text> + {state.total > 0 && ( + <Text color="gray"> + {' '} + ({state.total} result{state.total !== 1 ? 's' : ''}) + </Text> + )} + </Box> + + {/* Search input */} + <Box> + <Text color="cyan">> </Text> + <Text>{state.query}</Text> + <Text color="cyan">_</Text> + </Box> + + {/* Loading indicator */} + {state.isLoading && ( + <Box marginTop={1}> + <Text color="yellowBright">Searching...</Text> + </Box> + )} + + {/* Error message */} + {state.error && ( + <Box marginTop={1}> + <Text color="red">{state.error}</Text> + </Box> + )} + + {/* Results */} + {!state.isLoading && state.results.length > 0 && ( + <Box flexDirection="column" marginTop={1}> + {visibleResults.map((result, idx) => { + const actualIndex = scrollOffset.current + idx; + const isSelected = actualIndex === state.selectedIndex; + const roleColor = + result.message.role === 'user' + ? 'blue' + : result.message.role === 'assistant' + ? 'green' + : 'yellowBright'; + + return ( + <Box + key={`${result.sessionId}-${result.messageIndex}`} + flexDirection="column" + paddingLeft={isSelected ? 0 : 2} + > + <Box> + {isSelected && ( + <Text color="cyan" bold> + {'> '} + </Text> + )} + <Text color="gray">{result.sessionId.slice(0, 8)}</Text> + <Text> </Text> + <Text color={roleColor} bold={isSelected}> + [{result.message.role}] + </Text> + </Box> + <Box paddingLeft={isSelected ? 2 : 0}> + <Text color={isSelected ? 'white' : 'gray'}> + {' '} + {truncateContext(result.context, 60)} + </Text> + </Box> + </Box> + ); + })} + + {/* Scroll indicator */} + {state.results.length > MAX_VISIBLE_RESULTS && ( + <Box marginTop={1}> + <Text color="gray"> + Showing {scrollOffset.current + 1}- + {Math.min( + scrollOffset.current + MAX_VISIBLE_RESULTS, + state.results.length + )}{' '} + of {state.results.length} + {state.hasMore ? '+' : ''} + </Text> + </Box> + )} + </Box> + )} + + {/* No results message */} + {!state.isLoading && state.query && state.results.length === 0 && !state.error && ( + <Box marginTop={1}> + <Text color="gray">No results found for "{state.query}"</Text> + </Box> + )} + + {/* Help text */} + <Box marginTop={1}> + <Text color="gray">↑↓ navigate • Enter select • Esc close</Text> + </Box> + </Box> + ); +}); + +/** + * Truncate context text for display + */ +function truncateContext(context: string, maxLength: number): string { + // Clean up whitespace + const cleaned = context.replace(/\s+/g, ' ').trim(); + if (cleaned.length <= maxLength) { + return cleaned; + } + return cleaned.slice(0, maxLength - 3) + '...'; +} + +export default SearchOverlay; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/SessionRenameOverlay.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/SessionRenameOverlay.tsx new file mode 100644 index 00000000..ba00b664 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/SessionRenameOverlay.tsx @@ -0,0 +1,140 @@ +/** + * SessionRenameOverlay Component + * Interactive overlay for renaming the current session + */ + +import React, { useState, useEffect, forwardRef, useImperativeHandle, useCallback } from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; + +export interface SessionRenameOverlayProps { + isVisible: boolean; + currentTitle: string | undefined; + onRename: (newTitle: string) => void; + onClose: () => void; +} + +export interface SessionRenameOverlayHandle { + handleInput: (input: string, key: Key) => boolean; +} + +/** + * Session rename overlay - allows user to edit the session title + */ +const SessionRenameOverlay = forwardRef<SessionRenameOverlayHandle, SessionRenameOverlayProps>( + function SessionRenameOverlay({ isVisible, currentTitle, onRename, onClose }, ref) { + const [title, setTitle] = useState(currentTitle || ''); + const [error, setError] = useState<string | null>(null); + + // Reset when becoming visible + useEffect(() => { + if (isVisible) { + setTitle(currentTitle || ''); + setError(null); + } + }, [isVisible, currentTitle]); + + const handleSubmit = useCallback(() => { + const trimmedTitle = title.trim(); + + if (!trimmedTitle) { + setError('Title cannot be empty'); + return; + } + + onRename(trimmedTitle); + }, [title, onRename]); + + // Handle keyboard input + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape to close + if (key.escape) { + onClose(); + return true; + } + + // Enter to submit + if (key.return) { + handleSubmit(); + return true; + } + + // Backspace + if (key.backspace || key.delete) { + setTitle((prev) => prev.slice(0, -1)); + setError(null); + return true; + } + + // Regular character input + if (input && !key.ctrl && !key.meta) { + setTitle((prev) => prev + input); + setError(null); + return true; + } + + return false; + }, + }), + [isVisible, onClose, handleSubmit] + ); + + if (!isVisible) return null; + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="cyan" + paddingX={1} + marginTop={1} + > + {/* Header */} + <Box marginBottom={1}> + <Text bold color="cyan"> + Rename Session + </Text> + </Box> + + {/* Current title hint */} + {currentTitle && ( + <Box marginBottom={1}> + <Text color="gray">Current: </Text> + <Text color="white">{currentTitle}</Text> + </Box> + )} + + {/* Input prompt */} + <Box flexDirection="column"> + <Text bold>Enter new title:</Text> + </Box> + + {/* Input field */} + <Box marginTop={1}> + <Text color="cyan">> </Text> + <Text>{title}</Text> + <Text color="cyan">_</Text> + </Box> + + {/* Error message */} + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + {/* Help text */} + <Box marginTop={1}> + <Text color="gray">Enter to save • Esc to cancel</Text> + </Box> + </Box> + ); + } +); + +export default SessionRenameOverlay; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/SessionSelectorRefactored.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/SessionSelectorRefactored.tsx new file mode 100644 index 00000000..feecf3dc --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/SessionSelectorRefactored.tsx @@ -0,0 +1,198 @@ +/** + * SessionSelector Component (Refactored) + * Now a thin wrapper around BaseSelector + * Eliminates ~200 lines of code by using base component + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent, SessionMetadata } from '@dexto/core'; +import { logger } from '@dexto/core'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +interface SessionSelectorProps { + isVisible: boolean; + onSelectSession: (sessionId: string) => void; + onClose: () => void; + agent: DextoAgent; + currentSessionId?: string | undefined; +} + +export interface SessionSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface SessionOption { + id: string; + metadata: SessionMetadata | undefined; + isCurrent: boolean; +} + +/** + * Session selector - now a thin wrapper around BaseSelector + * Provides data fetching and formatting only + * Uses explicit currentSessionId prop (WebUI pattern) instead of getCurrentSessionId + */ +const SessionSelector = forwardRef<SessionSelectorHandle, SessionSelectorProps>( + function SessionSelector( + { isVisible, onSelectSession, onClose, agent, currentSessionId }, + ref + ) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [sessions, setSessions] = useState<SessionOption[]>([]); + const [isLoading, setIsLoading] = useState(false); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Fetch sessions from agent + useEffect(() => { + if (!isVisible) return; + + let cancelled = false; + setIsLoading(true); + + const fetchSessions = async () => { + try { + const sessionIds = await agent.listSessions(); + + // Fetch metadata for all sessions + const sessionList: SessionOption[] = await Promise.all( + sessionIds.map(async (id) => { + try { + const metadata = await agent.getSessionMetadata(id); + return { + id, + metadata, + isCurrent: id === currentSessionId, + }; + } catch { + return { + id, + metadata: undefined, + isCurrent: id === currentSessionId, + }; + } + }) + ); + + // Sort: current session first, then by last activity + sessionList.sort((a, b) => { + if (a.isCurrent) return -1; + if (b.isCurrent) return 1; + const aTime = a.metadata?.lastActivity || 0; + const bTime = b.metadata?.lastActivity || 0; + return bTime - aTime; // Most recent first + }); + + if (!cancelled) { + setSessions(sessionList); + setIsLoading(false); + // Current session is first, so index 0 + setSelectedIndex(0); + } + } catch (error) { + if (!cancelled) { + logger.error( + `Failed to fetch sessions: ${error instanceof Error ? error.message : 'Unknown error'}`, + { error } + ); + setSessions([]); + setIsLoading(false); + } + } + }; + + void fetchSessions(); + + return () => { + cancelled = true; + }; + }, [isVisible, agent, currentSessionId]); + + // Format session for display + const formatSession = (session: SessionOption): string => { + const parts: string[] = []; + + // Add title if available, otherwise use "New Session" as fallback + if (session.metadata?.title) { + parts.push(session.metadata.title); + } else { + parts.push('New Session'); + } + + // Always show short ID + parts.push(session.id.slice(0, 8)); + + // Show last activity time if available + if (session.metadata?.lastActivity) { + const now = Date.now(); + const diff = now - session.metadata.lastActivity; + const minutes = Math.floor(diff / 60000); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + if (days > 0) { + parts.push(`${days}d ago`); + } else if (hours > 0) { + parts.push(`${hours}h ago`); + } else if (minutes > 0) { + parts.push(`${minutes}m ago`); + } else { + parts.push('just now'); + } + } + return parts.join(' • '); + }; + + // Format session item for display + const formatItem = (session: SessionOption, isSelected: boolean) => ( + <> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {formatSession(session)} + </Text> + {session.isCurrent && ( + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {' '} + ← Current + </Text> + )} + </> + ); + + // Handle selection + const handleSelect = (session: SessionOption) => { + onSelectSession(session.id); + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={sessions} + isVisible={isVisible} + isLoading={isLoading} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Select Session" + borderColor="cyan" + loadingMessage="Loading sessions..." + emptyMessage="No sessions found" + /> + ); + } +); + +export default SessionSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/SessionSubcommandSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/SessionSubcommandSelector.tsx new file mode 100644 index 00000000..2d2af3eb --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/SessionSubcommandSelector.tsx @@ -0,0 +1,101 @@ +/** + * SessionSubcommandSelector Component + * Shows session management options (list, history, delete, switch) + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; + +export type SessionAction = 'list' | 'history' | 'delete' | 'switch'; + +interface SessionSubcommandSelectorProps { + isVisible: boolean; + onSelect: (action: SessionAction) => void; + onClose: () => void; +} + +export interface SessionSubcommandSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface SessionOption { + action: SessionAction; + label: string; + description: string; + icon: string; +} + +const SESSION_OPTIONS: SessionOption[] = [ + { action: 'list', label: 'list', description: 'List all sessions', icon: '📋' }, + { action: 'switch', label: 'switch', description: 'Switch to another session', icon: '🔄' }, + { action: 'history', label: 'history', description: 'Show session history', icon: '📜' }, + { action: 'delete', label: 'delete', description: 'Delete a session', icon: '🗑️' }, +]; + +/** + * Session subcommand selector - shows session management options + */ +const SessionSubcommandSelector = forwardRef< + SessionSubcommandSelectorHandle, + SessionSubcommandSelectorProps +>(function SessionSubcommandSelector({ isVisible, onSelect, onClose }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [options] = useState<SessionOption[]>(SESSION_OPTIONS); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Reset selection when becoming visible + useEffect(() => { + if (isVisible) { + setSelectedIndex(0); + } + }, [isVisible]); + + // Format option for display + const formatItem = (option: SessionOption, isSelected: boolean) => ( + <> + <Text>{option.icon} </Text> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {option.label} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> - {option.description}</Text> + </> + ); + + // Handle selection + const handleSelect = (option: SessionOption) => { + onSelect(option.action); + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={options} + isVisible={isVisible} + isLoading={false} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Session Management" + borderColor="blue" + emptyMessage="No options available" + /> + ); +}); + +export default SessionSubcommandSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/StreamSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/StreamSelector.tsx new file mode 100644 index 00000000..2ed17a93 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/StreamSelector.tsx @@ -0,0 +1,122 @@ +/** + * StreamSelector Component + * Interactive selector for toggling streaming mode + */ + +import React, { useState, useEffect, forwardRef, useRef, useImperativeHandle } from 'react'; +import { Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import { BaseSelector, type BaseSelectorHandle } from '../base/BaseSelector.js'; +import { isStreamingEnabled, setStreamingEnabled } from '../../state/streaming-state.js'; + +interface StreamSelectorProps { + isVisible: boolean; + onSelect: (enabled: boolean) => void; + onClose: () => void; +} + +export interface StreamSelectorHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface StreamOption { + id: 'enabled' | 'disabled'; + label: string; + description: string; + icon: string; + isCurrent: boolean; +} + +/** + * Stream selector - toggle streaming on/off + */ +const StreamSelector = forwardRef<StreamSelectorHandle, StreamSelectorProps>( + function StreamSelector({ isVisible, onSelect, onClose }, ref) { + const baseSelectorRef = useRef<BaseSelectorHandle>(null); + + // Forward handleInput to BaseSelector + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + return baseSelectorRef.current?.handleInput(input, key) ?? false; + }, + }), + [] + ); + + const [options, setOptions] = useState<StreamOption[]>([]); + const [selectedIndex, setSelectedIndex] = useState(0); + + // Build options list with current indicator + useEffect(() => { + if (!isVisible) return; + + const currentEnabled = isStreamingEnabled(); + const optionList: StreamOption[] = [ + { + id: 'enabled', + label: 'Enabled (Experimental)', + description: 'Show responses as they are generated', + icon: '▶️', + isCurrent: currentEnabled, + }, + { + id: 'disabled', + label: 'Disabled', + description: 'Show complete response when finished (default)', + icon: '⏸️', + isCurrent: !currentEnabled, + }, + ]; + + setOptions(optionList); + + // Set initial selection to current state + setSelectedIndex(currentEnabled ? 0 : 1); + }, [isVisible]); + + // Format option item for display + const formatItem = (option: StreamOption, isSelected: boolean) => ( + <> + <Text>{option.icon} </Text> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {option.label} + </Text> + <Text color={isSelected ? 'white' : 'gray'}> - {option.description}</Text> + {option.isCurrent && ( + <Text color="green" bold> + {' '} + ✓ + </Text> + )} + </> + ); + + // Handle selection + const handleSelect = (option: StreamOption) => { + const enabled = option.id === 'enabled'; + setStreamingEnabled(enabled); + onSelect(enabled); + }; + + return ( + <BaseSelector + ref={baseSelectorRef} + items={options} + isVisible={isVisible} + isLoading={false} + selectedIndex={selectedIndex} + onSelectIndex={setSelectedIndex} + onSelect={handleSelect} + onClose={onClose} + formatItem={formatItem} + title="Streaming Mode" + borderColor="cyan" + emptyMessage="No options available" + /> + ); + } +); + +export default StreamSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/ToolBrowser.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ToolBrowser.tsx new file mode 100644 index 00000000..a419b8be --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/ToolBrowser.tsx @@ -0,0 +1,671 @@ +/** + * ToolBrowser Component + * Interactive browser for exploring available tools + * Features: + * - Search/filter tools by name + * - View tool details (description, schema) + * - Shows source (MCP server or Internal) + */ + +import React, { + useState, + useEffect, + forwardRef, + useRef, + useImperativeHandle, + useMemo, +} from 'react'; +import { Box, Text } from 'ink'; +import type { Key } from '../../hooks/useInputOrchestrator.js'; +import type { DextoAgent } from '@dexto/core'; +import { useTerminalSize } from '../../hooks/useTerminalSize.js'; +import { writeToClipboard } from '../../utils/clipboardUtils.js'; + +interface ToolBrowserProps { + isVisible: boolean; + onClose: () => void; + agent: DextoAgent; +} + +export interface ToolBrowserHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ToolInfo { + name: string; + description: string; + source: 'internal' | 'mcp'; + serverName: string | undefined; + inputSchema: Record<string, unknown> | undefined; +} + +type ViewMode = 'list' | 'detail'; + +const MAX_VISIBLE_ITEMS = 12; +const MAX_DETAIL_LINES = 15; + +/** + * Tool browser with search and detail views + */ +const ToolBrowser = forwardRef<ToolBrowserHandle, ToolBrowserProps>(function ToolBrowser( + { isVisible, onClose, agent }, + ref +) { + const { columns, rows } = useTerminalSize(); + const [tools, setTools] = useState<ToolInfo[]>([]); + const [isLoading, setIsLoading] = useState(false); + const [selectedIndex, setSelectedIndex] = useState(0); + const [searchQuery, setSearchQuery] = useState(''); + const [scrollOffset, setScrollOffset] = useState(0); + const [viewMode, setViewMode] = useState<ViewMode>('list'); + const [selectedTool, setSelectedTool] = useState<ToolInfo | null>(null); + const [detailScrollOffset, setDetailScrollOffset] = useState(0); + const [copyFeedback, setCopyFeedback] = useState<string | null>(null); + const selectedIndexRef = useRef(selectedIndex); + const viewModeRef = useRef(viewMode); + const detailScrollOffsetRef = useRef(detailScrollOffset); + const detailMaxScrollOffsetRef = useRef(0); + const selectedToolRef = useRef<ToolInfo | null>(null); + + // Keep refs in sync + selectedIndexRef.current = selectedIndex; + viewModeRef.current = viewMode; + detailScrollOffsetRef.current = detailScrollOffset; + selectedToolRef.current = selectedTool; + + // Fetch tools from agent + useEffect(() => { + if (!isVisible) return; + + let cancelled = false; + setIsLoading(true); + setSearchQuery(''); + setSelectedIndex(0); + setScrollOffset(0); + setViewMode('list'); + setSelectedTool(null); + + const fetchTools = async () => { + try { + const [allTools, mcpTools] = await Promise.all([ + agent.getAllTools(), + agent.getAllMcpTools(), + ]); + + const toolList: ToolInfo[] = []; + const mcpToolNames = new Set(Object.keys(mcpTools)); + + for (const [toolName, toolInfo] of Object.entries(allTools)) { + const isMcpTool = mcpToolNames.has(toolName) || toolName.startsWith('mcp--'); + + // Extract server name from MCP tool name (format: mcp--serverName--toolName) + let serverName: string | undefined; + if (toolName.startsWith('mcp--')) { + const parts = toolName.split('--'); + if (parts.length >= 2) { + serverName = parts[1]; + } + } + + toolList.push({ + name: toolName, + description: toolInfo.description || 'No description available', + source: isMcpTool ? 'mcp' : 'internal', + serverName, + inputSchema: toolInfo.parameters as Record<string, unknown> | undefined, + }); + } + + // Sort: internal tools first, then MCP tools + toolList.sort((a, b) => { + if (a.source !== b.source) { + return a.source === 'internal' ? -1 : 1; + } + return a.name.localeCompare(b.name); + }); + + if (!cancelled) { + setTools(toolList); + setIsLoading(false); + } + } catch (error) { + if (!cancelled) { + setTools([]); + setIsLoading(false); + } + } + }; + + void fetchTools(); + + return () => { + cancelled = true; + }; + }, [isVisible, agent]); + + // Filter tools based on search query + const filteredTools = useMemo((): ToolInfo[] => { + if (!searchQuery.trim()) { + return tools; + } + + const query = searchQuery.toLowerCase(); + return tools.filter((tool) => { + const name = tool.name.toLowerCase(); + const desc = tool.description.toLowerCase(); + const server = (tool.serverName || '').toLowerCase(); + return name.includes(query) || desc.includes(query) || server.includes(query); + }); + }, [tools, searchQuery]); + + // Adjust selected index when filter changes + useEffect(() => { + if (selectedIndex >= filteredTools.length) { + setSelectedIndex(Math.max(0, filteredTools.length - 1)); + } + }, [filteredTools.length, selectedIndex]); + + // Calculate scroll offset + useEffect(() => { + if (selectedIndex < scrollOffset) { + setScrollOffset(selectedIndex); + } else if (selectedIndex >= scrollOffset + MAX_VISIBLE_ITEMS) { + setScrollOffset(selectedIndex - MAX_VISIBLE_ITEMS + 1); + } + }, [selectedIndex, scrollOffset]); + + // Handle showing tool details + const showToolDetails = (tool: ToolInfo) => { + setSelectedTool(tool); + setViewMode('detail'); + setDetailScrollOffset(0); + }; + + // Handle going back to list + const goBackToList = () => { + setViewMode('list'); + setSelectedTool(null); + }; + + // Expose handleInput method via ref + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible) return false; + + // In detail view (use ref to get latest value) + if (viewModeRef.current === 'detail') { + if (key.escape || key.backspace || key.delete) { + goBackToList(); + return true; + } + // Handle scrolling in detail view - check refs before setState to avoid flicker + if (key.upArrow) { + if (detailScrollOffsetRef.current > 0) { + setDetailScrollOffset((prev) => prev - 1); + } + return true; + } + if (key.downArrow) { + if (detailScrollOffsetRef.current < detailMaxScrollOffsetRef.current) { + setDetailScrollOffset((prev) => prev + 1); + } + return true; + } + // Copy schema to clipboard + if (input === 'c' || input === 'C') { + const tool = selectedToolRef.current; + if (tool) { + const schema = { + type: 'function', + name: tool.name, + description: tool.description, + parameters: tool.inputSchema || {}, + }; + void writeToClipboard(JSON.stringify(schema, null, 2)).then( + (success) => { + setCopyFeedback(success ? 'Copied!' : 'Copy failed'); + setTimeout(() => setCopyFeedback(null), 1500); + } + ); + } + return true; + } + return true; // Consume all input in detail view + } + + // In list view + // Escape closes + if (key.escape) { + onClose(); + return true; + } + + // Handle character input for search + if (input && !key.return && !key.upArrow && !key.downArrow && !key.tab) { + // Backspace + if (key.backspace || key.delete) { + setSearchQuery((prev) => prev.slice(0, -1)); + return true; + } + + // Regular character - add to search + if (input.length === 1 && input.charCodeAt(0) >= 32) { + setSearchQuery((prev) => prev + input); + setSelectedIndex(0); + setScrollOffset(0); + return true; + } + } + + // Backspace when no other input + if (key.backspace || key.delete) { + setSearchQuery((prev) => prev.slice(0, -1)); + return true; + } + + const itemsLength = filteredTools.length; + if (itemsLength === 0) return false; + + if (key.upArrow) { + const nextIndex = (selectedIndexRef.current - 1 + itemsLength) % itemsLength; + setSelectedIndex(nextIndex); + selectedIndexRef.current = nextIndex; + return true; + } + + if (key.downArrow) { + const nextIndex = (selectedIndexRef.current + 1) % itemsLength; + setSelectedIndex(nextIndex); + selectedIndexRef.current = nextIndex; + return true; + } + + if (key.return && itemsLength > 0) { + const tool = filteredTools[selectedIndexRef.current]; + if (tool) { + showToolDetails(tool); + return true; + } + } + + return false; + }, + }), + [isVisible, filteredTools, onClose, viewMode] + ); + + if (!isVisible) return null; + + if (isLoading) { + return ( + <Box paddingX={0} paddingY={0}> + <Text color="gray">Loading tools...</Text> + </Box> + ); + } + + // Detail view + if (viewMode === 'detail' && selectedTool) { + // Calculate max visible lines based on terminal height + const maxVisibleLines = Math.min(18, Math.max(5, rows - 6)); // Cap at 18 to reduce flicker + return ( + <ToolDetailView + tool={selectedTool} + columns={columns} + scrollOffset={detailScrollOffset} + maxVisibleLines={maxVisibleLines} + maxScrollOffsetRef={detailMaxScrollOffsetRef} + copyFeedback={copyFeedback} + /> + ); + } + + // List view + const visibleTools = filteredTools.slice(scrollOffset, scrollOffset + MAX_VISIBLE_ITEMS); + // Show counts based on filtered results, not total + const filteredInternalCount = filteredTools.filter((t) => t.source === 'internal').length; + const filteredMcpCount = filteredTools.filter((t) => t.source === 'mcp').length; + + return ( + <Box flexDirection="column" width={columns}> + {/* Header */} + <Box paddingX={0} paddingY={0}> + <Text color="cyan" bold> + Tool Browser + </Text> + <Text color="gray"> + {' '} + ({filteredTools.length} tools: {filteredInternalCount} internal,{' '} + {filteredMcpCount} MCP) + </Text> + </Box> + <Box paddingX={0} paddingY={0}> + <Text color="gray">↑↓ navigate, Enter view details, Esc close</Text> + </Box> + + {/* Search input */} + <Box paddingX={0} paddingY={0} marginTop={1}> + <Text color="gray">Search: </Text> + <Text color={searchQuery ? 'white' : 'gray'}> + {searchQuery || 'Type to filter...'} + </Text> + <Text color="cyan">▌</Text> + </Box> + + {/* Separator */} + <Box paddingX={0} paddingY={0}> + <Text color="gray">{'─'.repeat(Math.min(60, columns - 2))}</Text> + </Box> + + {/* Tools list */} + {filteredTools.length === 0 ? ( + <Box paddingX={0} paddingY={0}> + <Text color="gray">No tools match your search</Text> + </Box> + ) : ( + visibleTools.map((tool, visibleIndex) => { + const actualIndex = scrollOffset + visibleIndex; + const isSelected = actualIndex === selectedIndex; + + return ( + <Box key={tool.name} paddingX={0} paddingY={0}> + <Text color={isSelected ? 'cyan' : 'white'}> + {isSelected ? '▶ ' : ' '} + </Text> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {truncateText(tool.name, 35)} + </Text> + <Text color={tool.source === 'internal' ? 'magenta' : 'blue'}> + {' '} + [{tool.source === 'internal' ? 'Internal' : 'MCP'}] + </Text> + {tool.serverName && <Text color="gray"> ({tool.serverName})</Text>} + </Box> + ); + }) + )} + + {/* Scroll indicator */} + {filteredTools.length > MAX_VISIBLE_ITEMS && ( + <Box paddingX={0} paddingY={0} marginTop={1}> + <Text color="gray"> + {scrollOffset > 0 ? '↑ more above' : ''} + {scrollOffset > 0 && scrollOffset + MAX_VISIBLE_ITEMS < filteredTools.length + ? ' | ' + : ''} + {scrollOffset + MAX_VISIBLE_ITEMS < filteredTools.length + ? '↓ more below' + : ''} + </Text> + </Box> + )} + </Box> + ); +}); + +/** + * Content line types for detail view + */ +type DetailLineType = + | { type: 'title'; text: string } + | { type: 'source'; source: 'internal' | 'mcp'; serverName: string | undefined } + | { type: 'empty' } + | { type: 'header'; text: string } + | { type: 'description'; text: string } + | { type: 'param-name'; name: string; paramType: string; required: boolean } + | { type: 'param-desc'; text: string } + | { type: 'param-enum'; values: string[] }; + +/** + * Build detail line data for a tool (plain data, not React elements) + */ +function buildDetailLineData(tool: ToolInfo, maxWidth: number): DetailLineType[] { + const lines: DetailLineType[] = []; + + // Tool name + lines.push({ type: 'title', text: tool.name }); + + // Source + lines.push({ type: 'source', source: tool.source, serverName: tool.serverName }); + + // Empty line before description + lines.push({ type: 'empty' }); + + // Description header + lines.push({ type: 'header', text: 'Description:' }); + + // Description content (wrapped) + const descriptionLines = wrapText(tool.description, maxWidth - 2).split('\n'); + for (const descLine of descriptionLines) { + lines.push({ type: 'description', text: descLine }); + } + + // Parameters section + if (tool.inputSchema) { + const properties = tool.inputSchema.properties as + | Record<string, Record<string, unknown>> + | undefined; + const required = (tool.inputSchema.required as string[]) || []; + + if (properties && Object.keys(properties).length > 0) { + // Empty line before parameters + lines.push({ type: 'empty' }); + + // Parameters header + lines.push({ type: 'header', text: 'Parameters:' }); + + // Each parameter + for (const [propName, propSchema] of Object.entries(properties)) { + const paramType = (propSchema.type as string) || 'any'; + const description = propSchema.description as string | undefined; + const isRequired = required.includes(propName); + const enumValues = propSchema.enum as string[] | undefined; + + // Parameter name line + lines.push({ + type: 'param-name', + name: propName, + paramType, + required: isRequired, + }); + + // Parameter description (wrapped) + if (description) { + const paramDescLines = wrapText(description, maxWidth - 6).split('\n'); + for (const paramDescLine of paramDescLines) { + lines.push({ type: 'param-desc', text: paramDescLine }); + } + } + + // Enum values + if (enumValues) { + lines.push({ type: 'param-enum', values: enumValues }); + } + + // Empty line between parameters + lines.push({ type: 'empty' }); + } + } else { + // Empty line before "no parameters" + lines.push({ type: 'empty' }); + lines.push({ type: 'param-desc', text: 'No parameters' }); + } + } + + return lines; +} + +/** + * Render a single detail line from data + */ +function renderDetailLine(line: DetailLineType, index: number): React.ReactNode { + switch (line.type) { + case 'title': + return ( + <Text color="cyan" bold> + {line.text} + </Text> + ); + case 'source': + return ( + <> + <Text color="gray">Source: </Text> + <Text color={line.source === 'internal' ? 'magenta' : 'blue'}> + {line.source === 'internal' ? 'Internal' : 'MCP'} + </Text> + {line.serverName && <Text color="gray"> (server: {line.serverName})</Text>} + </> + ); + case 'empty': + return <Text> </Text>; + case 'header': + return <Text color="gray">{line.text}</Text>; + case 'description': + return ( + <> + <Text> </Text> + <Text>{line.text}</Text> + </> + ); + case 'param-name': + return ( + <> + <Text> </Text> + <Text color="cyan">{line.name}</Text> + <Text color="gray"> ({line.paramType})</Text> + {line.required && <Text color="red"> *required</Text>} + </> + ); + case 'param-desc': + return ( + <> + <Text> </Text> + <Text color="gray">{line.text}</Text> + </> + ); + case 'param-enum': + return ( + <> + <Text> </Text> + <Text color="gray">Allowed: {line.values.join(' | ')}</Text> + </> + ); + } +} + +/** + * Tool detail view component with scrolling support + */ +function ToolDetailView({ + tool, + columns, + scrollOffset, + maxVisibleLines, + maxScrollOffsetRef, + copyFeedback, +}: { + tool: ToolInfo; + columns: number; + scrollOffset: number; + maxVisibleLines: number; + maxScrollOffsetRef: React.MutableRefObject<number>; + copyFeedback: string | null; +}) { + const maxWidth = Math.min(80, columns - 4); + + // Build plain data for lines (memoized) + const lineData = useMemo(() => buildDetailLineData(tool, maxWidth), [tool, maxWidth]); + + // Calculate visible range and update max scroll offset ref + const totalLines = lineData.length; + const maxScrollOffset = Math.max(0, totalLines - maxVisibleLines); + maxScrollOffsetRef.current = maxScrollOffset; + const clampedOffset = Math.min(scrollOffset, maxScrollOffset); + + // Calculate scroll indicators + const hasMoreAbove = clampedOffset > 0; + const hasMoreBelow = clampedOffset + maxVisibleLines < totalLines; + + return ( + <Box flexDirection="column" width={columns}> + {/* Header */} + <Box paddingX={0} paddingY={0}> + <Text color="cyan" bold> + Tool Details + </Text> + <Text color="gray"> - ↑↓ scroll, c copy schema, Esc back</Text> + {copyFeedback && ( + <Text color="green" bold> + {' '} + {copyFeedback} + </Text> + )} + </Box> + + {/* Separator */} + <Box paddingX={0} paddingY={0}> + <Text color="gray">{'─'.repeat(Math.min(60, columns - 2))}</Text> + </Box> + + {/* Scroll indicator - above (always present to avoid layout shift) */} + <Box paddingX={0} paddingY={0}> + <Text color="gray">{hasMoreAbove ? `↑ ${clampedOffset} more above` : ' '}</Text> + </Box> + + {/* Visible content - render directly from data like TextBufferInput */} + {Array.from({ length: maxVisibleLines }, (_, i) => { + const absoluteIndex = clampedOffset + i; + const line = lineData[absoluteIndex]; + return ( + <Box key={i} paddingX={0} paddingY={0}> + {line ? renderDetailLine(line, absoluteIndex) : <Text> </Text>} + </Box> + ); + })} + + {/* Scroll indicator - below (always present to avoid layout shift) */} + <Box paddingX={0} paddingY={0}> + <Text color="gray"> + {hasMoreBelow + ? `↓ ${totalLines - clampedOffset - maxVisibleLines} more below` + : ' '} + </Text> + </Box> + </Box> + ); +} + +/** + * Truncate text to max length with ellipsis + */ +function truncateText(text: string, maxLength: number): string { + if (text.length <= maxLength) return text; + return text.slice(0, maxLength - 1) + '…'; +} + +/** + * Wrap text to fit within max width + */ +function wrapText(text: string, maxWidth: number): string { + if (text.length <= maxWidth) return text; + + const words = text.split(' '); + const lines: string[] = []; + let currentLine = ''; + + for (const word of words) { + if (currentLine.length + word.length + 1 <= maxWidth) { + currentLine += (currentLine ? ' ' : '') + word; + } else { + if (currentLine) lines.push(currentLine); + currentLine = word; + } + } + if (currentLine) lines.push(currentLine); + + return lines.join('\n'); +} + +export default ToolBrowser; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/LocalModelWizard.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/LocalModelWizard.tsx new file mode 100644 index 00000000..e65e3927 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/LocalModelWizard.tsx @@ -0,0 +1,1093 @@ +/** + * LocalModelWizard Component + * Specialized wizard for adding local GGUF models with: + * - Registry model selection with download support + * - Custom GGUF file path option + * - Download progress display + */ + +import React, { useState, useEffect, forwardRef, useImperativeHandle, useCallback } from 'react'; +import { Box, Text } from 'ink'; +import * as fs from 'fs'; +import * as path from 'path'; +import { spawn } from 'child_process'; +import type { Key } from '../../../hooks/useInputOrchestrator.js'; +import { + saveCustomModel, + getAllInstalledModels, + addInstalledModel, + removeInstalledModel, + getModelsDirectory, + formatSize, + getDextoGlobalPath, + type InstalledModel, + type CustomModel, +} from '@dexto/agent-management'; +import { promises as fsPromises } from 'fs'; +import { + getAllLocalModels, + getLocalModelById, + getRecommendedLocalModels, + downloadModel, + isNodeLlamaCppInstalled, + type LocalModelInfo, + type ModelDownloadProgress, +} from '@dexto/core'; +import { SetupInfoBanner } from './shared/index.js'; + +type WizardStep = + | 'install-node-llama' + | 'select-model' + | 'custom-path' + | 'display-name' + | 'downloading' + | 'installed-options'; + +interface LocalModelWizardProps { + isVisible: boolean; + onComplete: (model: CustomModel) => void; + onClose: () => void; +} + +export interface LocalModelWizardHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface ModelOption { + id: string; + name: string; + description: string; + sizeBytes: number; + isInstalled: boolean; + minVRAM: number | undefined; +} + +const MAX_VISIBLE_ITEMS = 8; + +/** + * Specialized wizard for local GGUF model setup. + * Similar UX to CLI setup flow but in Ink. + */ +const LocalModelWizard = forwardRef<LocalModelWizardHandle, LocalModelWizardProps>( + function LocalModelWizard({ isVisible, onComplete, onClose }, ref) { + const [step, setStep] = useState<WizardStep>('select-model'); + const [models, setModels] = useState<ModelOption[]>([]); + const [installedIds, setInstalledIds] = useState<Set<string>>(new Set()); + const [selectedIndex, setSelectedIndex] = useState(0); + const [scrollOffset, setScrollOffset] = useState(0); + const [showAllModels, setShowAllModels] = useState(false); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState<string | null>(null); + const [nodeLlamaInstalled, setNodeLlamaInstalled] = useState(true); + const [nodeLlamaChecked, setNodeLlamaChecked] = useState(false); // Track if we've checked installation + const [isInstallingNodeLlama, setIsInstallingNodeLlama] = useState(false); + const [installConfirmIndex, setInstallConfirmIndex] = useState(0); // 0 = Yes, 1 = No + const [installSpinnerFrame, setInstallSpinnerFrame] = useState(0); + const [refreshTrigger, setRefreshTrigger] = useState(0); // Increment to trigger data reload + + // Custom path input state + const [customPath, setCustomPath] = useState(''); + + // Display name input state + const [displayName, setDisplayName] = useState(''); + const [selectedModelId, setSelectedModelId] = useState<string | null>(null); + const [selectedModelPath, setSelectedModelPath] = useState<string | null>(null); + + // Download state + const [downloadProgress, setDownloadProgress] = useState<ModelDownloadProgress | null>( + null + ); + const [downloadError, setDownloadError] = useState<string | null>(null); + + // Installed model options state + const [installedOptionIndex, setInstalledOptionIndex] = useState(0); + const [selectedInstalledModel, setSelectedInstalledModel] = useState<{ + id: string; + filePath: string; + displayName: string; + } | null>(null); + const [isDeleting, setIsDeleting] = useState(false); + + // Reset state when becoming visible + useEffect(() => { + if (!isVisible) return; + + setStep('select-model'); + setSelectedIndex(0); + setScrollOffset(0); + setShowAllModels(false); + setCustomPath(''); + setDisplayName(''); + setSelectedModelId(null); + setSelectedModelPath(null); + setDownloadProgress(null); + setDownloadError(null); + setInstalledOptionIndex(0); + setSelectedInstalledModel(null); + setIsDeleting(false); + setError(null); + setIsInstallingNodeLlama(false); + setInstallConfirmIndex(0); + setInstallSpinnerFrame(0); + setNodeLlamaChecked(false); + }, [isVisible]); + + // Spinner animation for installation + useEffect(() => { + if (!isInstallingNodeLlama) return; + + const spinnerFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; + const interval = setInterval(() => { + setInstallSpinnerFrame((prev) => (prev + 1) % spinnerFrames.length); + }, 80); + + return () => clearInterval(interval); + }, [isInstallingNodeLlama]); + + // Load models when visible or when showAllModels changes + useEffect(() => { + if (!isVisible) return; + + let cancelled = false; + + const loadData = async () => { + setIsLoading(true); + + try { + // Check if node-llama-cpp is installed + // Skip if we've already checked AND it's installed (prevents re-check after install) + if (!nodeLlamaChecked || !nodeLlamaInstalled) { + const installed = await isNodeLlamaCppInstalled(); + if (!cancelled) { + setNodeLlamaInstalled(installed); + setNodeLlamaChecked(true); + if (!installed) { + setStep('install-node-llama'); + setIsLoading(false); + return; + } + } + } + + // Get installed models + const installedModels = await getAllInstalledModels(); + const installedSet = new Set(installedModels.map((m) => m.id)); + if (!cancelled) { + setInstalledIds(installedSet); + } + + // Get registry models based on showAllModels flag + const registryModels = showAllModels + ? getAllLocalModels() + : getRecommendedLocalModels(); + + const options: ModelOption[] = registryModels.map((m) => ({ + id: m.id, + name: m.name, + description: m.description, + sizeBytes: m.sizeBytes, + isInstalled: installedSet.has(m.id), + minVRAM: m.minVRAM, + })); + + if (!cancelled) { + setModels(options); + setIsLoading(false); + } + } catch (err) { + if (!cancelled) { + setError(err instanceof Error ? err.message : 'Failed to load models'); + setIsLoading(false); + } + } + }; + + void loadData(); + + return () => { + cancelled = true; + }; + }, [isVisible, showAllModels, refreshTrigger, nodeLlamaInstalled, nodeLlamaChecked]); + + // Calculate scroll offset + useEffect(() => { + const itemCount = models.length + 3; // +3 for special options + if (selectedIndex < scrollOffset) { + setScrollOffset(selectedIndex); + } else if (selectedIndex >= scrollOffset + MAX_VISIBLE_ITEMS) { + setScrollOffset( + Math.min(selectedIndex - MAX_VISIBLE_ITEMS + 1, itemCount - MAX_VISIBLE_ITEMS) + ); + } + }, [selectedIndex, models.length, scrollOffset]); + + // Handle model selection + const handleSelectModel = useCallback( + async (modelId: string) => { + // Check if already installed - show options (Use / Delete) + if (installedIds.has(modelId)) { + const modelInfo = getLocalModelById(modelId); + const installedModels = await getAllInstalledModels(); + const installedModel = installedModels.find((m) => m.id === modelId); + + setSelectedInstalledModel({ + id: modelId, + filePath: installedModel?.filePath || '', + displayName: modelInfo?.name || modelId, + }); + setInstalledOptionIndex(0); + setStep('installed-options'); + return; + } + + // Need to download - start download + setSelectedModelId(modelId); + setStep('downloading'); + setDownloadProgress(null); + setDownloadError(null); + + try { + const result = await downloadModel(modelId, { + targetDir: getModelsDirectory(), + events: { + onProgress: (progress: ModelDownloadProgress) => { + setDownloadProgress(progress); + }, + onComplete: () => { + // Download complete + }, + onError: (_id: string, err: Error) => { + setDownloadError(err.message); + }, + }, + }); + + // Register the installed model + const modelInfo = getLocalModelById(modelId); + const installedModel: InstalledModel = { + id: modelId, + filePath: result.filePath, + sizeBytes: result.sizeBytes, + downloadedAt: new Date().toISOString(), + source: 'huggingface', + filename: modelInfo?.filename || path.basename(result.filePath), + }; + + if (result.sha256) { + installedModel.sha256 = result.sha256; + } + + await addInstalledModel(installedModel); + + // Registry models are tracked in state.json, not custom-models.json + // Just complete - the model will appear in the selector via getAllInstalledModels() + onComplete({ + name: modelId, + provider: 'local', + displayName: modelInfo?.name || modelId, + }); + } catch (err) { + setDownloadError(err instanceof Error ? err.message : 'Download failed'); + } + }, + [installedIds, onComplete] + ); + + // Handle custom path submission + const handleCustomPathSubmit = useCallback(async () => { + const trimmedPath = customPath.trim(); + + // Validate path + if (!trimmedPath) { + setError('File path is required'); + return; + } + + if (!trimmedPath.toLowerCase().endsWith('.gguf')) { + setError('File must have .gguf extension'); + return; + } + + // Expand ~ to home directory + const expandedPath = trimmedPath.startsWith('~') + ? trimmedPath.replace('~', process.env.HOME || '') + : trimmedPath; + + if (!path.isAbsolute(expandedPath)) { + setError('Please enter an absolute path (starting with / or ~)'); + return; + } + + if (!fs.existsSync(expandedPath)) { + setError(`File not found: ${trimmedPath}`); + return; + } + + // Path is valid - move to display name step + setSelectedModelPath(expandedPath); + const filename = path.basename(expandedPath, '.gguf'); + setDisplayName(filename); + setStep('display-name'); + setError(null); + }, [customPath]); + + // Handle display name submission + const handleDisplayNameSubmit = useCallback(async () => { + if (!selectedModelPath) return; + + const trimmedName = displayName.trim(); + const filename = path.basename(selectedModelPath, '.gguf'); + + // Generate model ID from filename + const modelId = filename + .toLowerCase() + .replace(/\s+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .substring(0, 50); + + const model: CustomModel = { + name: modelId, + provider: 'local', + filePath: selectedModelPath, + displayName: trimmedName || filename, + }; + + try { + await saveCustomModel(model); + onComplete(model); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to save model'); + } + }, [selectedModelPath, displayName, onComplete]); + + // Handle installed model option selection + const handleInstalledOption = useCallback(async () => { + if (!selectedInstalledModel) return; + + if (installedOptionIndex === 0) { + // "Use this model" option + onComplete({ + name: selectedInstalledModel.id, + provider: 'local', + displayName: selectedInstalledModel.displayName, + }); + } else { + // "Delete model" option + setIsDeleting(true); + setError(null); + + try { + // Delete the GGUF file from disk + if (selectedInstalledModel.filePath) { + try { + await fsPromises.unlink(selectedInstalledModel.filePath); + } catch (err) { + // File might already be deleted - continue + const nodeErr = err as NodeJS.ErrnoException; + if (nodeErr.code !== 'ENOENT') { + throw err; + } + } + } + + // Remove from state.json + await removeInstalledModel(selectedInstalledModel.id); + + // Refresh the model list + setInstalledIds((prev) => { + const next = new Set(prev); + next.delete(selectedInstalledModel.id); + return next; + }); + + // Go back to model selection + setStep('select-model'); + setSelectedInstalledModel(null); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to delete model'); + } finally { + setIsDeleting(false); + } + } + }, [selectedInstalledModel, installedOptionIndex, onComplete]); + + // Install node-llama-cpp to global deps directory + const installNodeLlamaCpp = useCallback(async (): Promise<boolean> => { + const depsDir = getDextoGlobalPath('deps'); + + // Ensure deps directory exists + if (!fs.existsSync(depsDir)) { + fs.mkdirSync(depsDir, { recursive: true }); + } + + // Initialize package.json if it doesn't exist + const packageJsonPath = path.join(depsDir, 'package.json'); + if (!fs.existsSync(packageJsonPath)) { + fs.writeFileSync( + packageJsonPath, + JSON.stringify( + { + name: 'dexto-deps', + version: '1.0.0', + private: true, + description: 'Native dependencies for Dexto', + }, + null, + 2 + ) + ); + } + + return new Promise((resolve) => { + const child = spawn('npm', ['install', 'node-llama-cpp'], { + stdio: ['ignore', 'ignore', 'pipe'], // stdin ignored, stdout ignored (not needed), stderr piped for errors + cwd: depsDir, + }); + + let stderr = ''; + child.stderr?.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('close', (code) => { + if (code === 0) { + resolve(true); + } else { + setError(`Installation failed: ${stderr.slice(0, 200)}`); + resolve(false); + } + }); + + child.on('error', (err) => { + setError(`Installation failed: ${err.message}`); + resolve(false); + }); + }); + }, []); + + // Handle install confirmation + const handleInstallConfirm = useCallback(async () => { + if (installConfirmIndex === 1) { + // User chose "No" + onClose(); + return; + } + + // User chose "Yes" - start installation + setIsInstallingNodeLlama(true); + setError(null); + + const success = await installNodeLlamaCpp(); + + setIsInstallingNodeLlama(false); + + if (success) { + // Trust npm's exit code - set states and go directly to model selection + setNodeLlamaInstalled(true); + setNodeLlamaChecked(true); + setStep('select-model'); + setIsLoading(true); + // Trigger reload of models + setRefreshTrigger((prev) => prev + 1); + } else { + // Error should already be set by installNodeLlamaCpp, but ensure we show something + setError( + (prev) => + prev || 'Installation failed. Check your internet connection and try again.' + ); + } + }, [installConfirmIndex, installNodeLlamaCpp, onClose]); + + // Handle input + useImperativeHandle( + ref, + () => ({ + handleInput: (input: string, key: Key): boolean => { + if (!isVisible) return false; + + // Escape to go back/close + if (key.escape) { + if ( + step === 'custom-path' || + step === 'display-name' || + step === 'installed-options' + ) { + setStep('select-model'); + setError(null); + setSelectedInstalledModel(null); + return true; + } + if (step === 'downloading' && downloadError) { + setStep('select-model'); + setDownloadError(null); + return true; + } + onClose(); + return true; + } + + // Handle based on current step + if (step === 'install-node-llama') { + if (isInstallingNodeLlama) return true; // Don't allow input while installing + + if (key.upArrow || key.downArrow) { + setInstallConfirmIndex((prev) => (prev === 0 ? 1 : 0)); + return true; + } + + if (key.return) { + void handleInstallConfirm(); + return true; + } + + return true; + } + + if (step === 'select-model') { + const itemCount = models.length + 3; // +3 for special options + + if (key.upArrow) { + setSelectedIndex((prev) => (prev > 0 ? prev - 1 : itemCount - 1)); + return true; + } + + if (key.downArrow) { + setSelectedIndex((prev) => (prev < itemCount - 1 ? prev + 1 : 0)); + return true; + } + + if (key.return) { + // Special options are at the end + const showAllIndex = models.length; + const customPathIndex = models.length + 1; + const backIndex = models.length + 2; + + if (selectedIndex === backIndex) { + onClose(); + return true; + } + + if (selectedIndex === showAllIndex) { + setShowAllModels(!showAllModels); + setSelectedIndex(0); + setScrollOffset(0); + return true; + } + + if (selectedIndex === customPathIndex) { + setStep('custom-path'); + setCustomPath(''); + setError(null); + return true; + } + + // Model selected + const model = models[selectedIndex]; + if (model) { + void handleSelectModel(model.id); + } + return true; + } + + return true; // Consume all input in select mode + } + + if (step === 'custom-path') { + if (key.return) { + void handleCustomPathSubmit(); + return true; + } + + if (key.backspace || key.delete) { + setCustomPath((prev) => prev.slice(0, -1)); + setError(null); + return true; + } + + if (input && !key.ctrl && !key.meta) { + setCustomPath((prev) => prev + input); + setError(null); + return true; + } + + return true; + } + + if (step === 'display-name') { + if (key.return) { + void handleDisplayNameSubmit(); + return true; + } + + if (key.backspace || key.delete) { + setDisplayName((prev) => prev.slice(0, -1)); + return true; + } + + if (input && !key.ctrl && !key.meta) { + setDisplayName((prev) => prev + input); + return true; + } + + return true; + } + + if (step === 'installed-options') { + if (isDeleting) return true; // Don't allow input while deleting + + if (key.upArrow || key.downArrow) { + setInstalledOptionIndex((prev) => (prev === 0 ? 1 : 0)); + return true; + } + + if (key.return) { + void handleInstalledOption(); + return true; + } + + return true; + } + + if (step === 'downloading') { + // Only allow escape if there's an error + return true; + } + + return false; + }, + }), + [ + isVisible, + step, + models, + selectedIndex, + showAllModels, + customPath, + displayName, + downloadError, + isDeleting, + installedOptionIndex, + handleSelectModel, + handleCustomPathSubmit, + handleDisplayNameSubmit, + handleInstalledOption, + handleInstallConfirm, + isInstallingNodeLlama, + installConfirmIndex, + onClose, + ] + ); + + if (!isVisible) return null; + + // Node-llama-cpp install prompt + if (step === 'install-node-llama') { + const options = [ + { label: 'Yes', description: 'Install now (may take 1-2 minutes)' }, + { label: 'No', description: 'Go back' }, + ]; + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="yellow" + paddingX={1} + marginTop={1} + > + <Box marginBottom={1}> + <Text bold color="yellow"> + Dependency Required + </Text> + </Box> + + <Text>Local model execution requires node-llama-cpp.</Text> + <Text color="gray">This will compile native bindings for your system.</Text> + + {isInstallingNodeLlama ? ( + <Box marginTop={1} flexDirection="column"> + <Box> + <Text color="cyan"> + { + ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'][ + installSpinnerFrame + ] + }{' '} + Installing node-llama-cpp (compiling native bindings)... + </Text> + </Box> + <Text color="gray">This may take 1-2 minutes.</Text> + </Box> + ) : ( + <> + <Box marginTop={1} marginBottom={1}> + <Text>Install node-llama-cpp now?</Text> + </Box> + + {options.map((option, idx) => ( + <Box key={option.label}> + <Text color={idx === installConfirmIndex ? 'cyan' : 'white'}> + {idx === installConfirmIndex ? '❯ ' : ' '} + {option.label} + </Text> + {idx === installConfirmIndex && ( + <Text color="gray"> - {option.description}</Text> + )} + </Box> + ))} + </> + )} + + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + <Box marginTop={1}> + <Text color="gray"> + {isInstallingNodeLlama + ? 'Please wait...' + : '↑↓ navigate • Enter select • Esc cancel'} + </Text> + </Box> + </Box> + ); + } + + // Loading state + if (isLoading) { + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="green" + paddingX={1} + marginTop={1} + > + <Text color="gray">Loading models...</Text> + </Box> + ); + } + + // Download progress + if (step === 'downloading') { + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="green" + paddingX={1} + marginTop={1} + > + <Box marginBottom={1}> + <Text bold color="green"> + Downloading Model + </Text> + </Box> + + {downloadError ? ( + <> + <Text color="red">Download failed: {downloadError}</Text> + <Box marginTop={1}> + <Text color="gray">Press Esc to go back</Text> + </Box> + </> + ) : downloadProgress ? ( + <> + <Text> + {selectedModelId}: {downloadProgress.percentage.toFixed(1)}% + </Text> + <Text color="gray"> + {formatSize(downloadProgress.bytesDownloaded)} /{' '} + {formatSize(downloadProgress.totalBytes)} + {downloadProgress.speed + ? ` • ${formatSize(downloadProgress.speed)}/s` + : ''} + {downloadProgress.eta + ? ` • ETA: ${Math.round(downloadProgress.eta)}s` + : ''} + </Text> + {/* Simple progress bar */} + <Box marginTop={1}> + <Text color="green"> + {'█'.repeat( + Math.min(20, Math.floor(downloadProgress.percentage / 5)) + )} + </Text> + <Text color="gray"> + {'░'.repeat( + Math.max( + 0, + 20 - Math.floor(downloadProgress.percentage / 5) + ) + )} + </Text> + </Box> + </> + ) : ( + <Text color="gray">Starting download...</Text> + )} + </Box> + ); + } + + // Installed model options (Use / Delete) + if (step === 'installed-options' && selectedInstalledModel) { + const options = [ + { label: 'Use this model', description: 'Select this model for chat' }, + { label: 'Delete model', description: 'Remove from disk and uninstall' }, + ]; + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="green" + paddingX={1} + marginTop={1} + > + <Box marginBottom={1}> + <Text bold color="green"> + {selectedInstalledModel.displayName} + </Text> + <Text color="gray"> (installed)</Text> + </Box> + + {isDeleting ? ( + <Text color="yellow">Deleting model...</Text> + ) : ( + <> + {options.map((option, idx) => ( + <Box key={option.label}> + <Text color={idx === installedOptionIndex ? 'cyan' : 'white'}> + {idx === installedOptionIndex ? '❯ ' : ' '} + {option.label} + </Text> + {idx === installedOptionIndex && ( + <Text color="gray"> - {option.description}</Text> + )} + </Box> + ))} + </> + )} + + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + <Box marginTop={1}> + <Text color="gray"> + {isDeleting ? 'Please wait...' : 'Enter to select • Esc to go back'} + </Text> + </Box> + </Box> + ); + } + + // Custom path input + if (step === 'custom-path') { + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="green" + paddingX={1} + marginTop={1} + > + <Box marginBottom={1}> + <Text bold color="green"> + Custom GGUF File + </Text> + </Box> + + <Text>Enter path to GGUF file:</Text> + <Box marginTop={1}> + <Text color="gray">> </Text> + <Text>{customPath}</Text> + <Text color="cyan">▌</Text> + </Box> + <Text color="gray">e.g., /path/to/model.gguf or ~/models/llama.gguf</Text> + + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + <Box marginTop={1}> + <Text color="gray">Enter to continue • Esc to go back</Text> + </Box> + </Box> + ); + } + + // Display name input + if (step === 'display-name') { + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="green" + paddingX={1} + marginTop={1} + > + <Box marginBottom={1}> + <Text bold color="green"> + Display Name + </Text> + </Box> + + <Text>Display name (optional):</Text> + <Box marginTop={1}> + <Text color="gray">> </Text> + <Text>{displayName}</Text> + <Text color="cyan">▌</Text> + </Box> + + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + <Box marginTop={1}> + <Text color="gray">Enter to save • Esc to go back</Text> + </Box> + </Box> + ); + } + + // Model selection + const allItems = [ + ...models, + { type: 'show-all' as const }, + { type: 'custom-path' as const }, + { type: 'back' as const }, + ]; + + const visibleStart = scrollOffset; + const visibleEnd = Math.min(scrollOffset + MAX_VISIBLE_ITEMS, allItems.length); + const visibleItems = allItems.slice(visibleStart, visibleEnd); + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="green" + paddingX={1} + marginTop={1} + > + {/* Header */} + <Box marginBottom={1}> + <Text bold color="green"> + Local Model + </Text> + <Text color="gray"> + {' '} + ({selectedIndex + 1}/{allItems.length}) + </Text> + </Box> + + {/* Setup info */} + <SetupInfoBanner + title="Local Models" + description="Select a model to download, or use a custom GGUF file. Models run completely on your machine - free, private, and offline." + docsUrl="https://docs.dexto.ai/docs/guides/supported-llm-providers#local-models" + /> + + {/* Model list */} + {visibleItems.map((item, visibleIndex) => { + const actualIndex = scrollOffset + visibleIndex; + const isSelected = actualIndex === selectedIndex; + + if ('type' in item) { + if (item.type === 'show-all') { + return ( + <Box key="show-all" paddingY={0}> + <Text color={isSelected ? 'cyan' : 'blue'} bold={isSelected}> + {isSelected ? '› ' : ' '} + {showAllModels + ? '↩ Show recommended' + : '... Show all models'} + </Text> + <Text color="gray"> + {' '} + ({getAllLocalModels().length} available) + </Text> + </Box> + ); + } + if (item.type === 'custom-path') { + return ( + <Box key="custom-path" paddingY={0}> + <Text color={isSelected ? 'cyan' : 'blue'} bold={isSelected}> + {isSelected ? '› ' : ' '} + ... Use custom GGUF file + </Text> + </Box> + ); + } + if (item.type === 'back') { + return ( + <Box key="back" paddingY={0}> + <Text color={isSelected ? 'cyan' : 'gray'} bold={isSelected}> + {isSelected ? '› ' : ' '}← Back + </Text> + </Box> + ); + } + } + + // Model option + const model = item as ModelOption; + const statusIcon = model.isInstalled ? '✓' : '○'; + const statusColor = model.isInstalled ? 'green' : 'gray'; + const vramHint = model.minVRAM ? `${model.minVRAM}GB+ VRAM` : 'CPU OK'; + + return ( + <Box key={model.id} paddingY={0}> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {isSelected ? '› ' : ' '} + </Text> + <Text color={statusColor}>{statusIcon} </Text> + <Text color={isSelected ? 'cyan' : 'white'} bold={isSelected}> + {model.name} + </Text> + <Text color="gray"> + {' '} + {formatSize(model.sizeBytes)} | {vramHint} + {model.isInstalled ? ' (installed)' : ''} + </Text> + </Box> + ); + })} + + {/* Scroll indicator */} + {allItems.length > MAX_VISIBLE_ITEMS && ( + <Box marginTop={1}> + <Text color="gray"> + {scrollOffset > 0 ? '↑ more above ' : ''} + {visibleEnd < allItems.length ? '↓ more below' : ''} + </Text> + </Box> + )} + + {/* Help text */} + <Box marginTop={1}> + <Text color="gray">↑↓ navigate • Enter select • Esc back</Text> + </Box> + + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + </Box> + ); + } +); + +export default LocalModelWizard; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/index.ts b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/index.ts new file mode 100644 index 00000000..085f33ac --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/index.ts @@ -0,0 +1,15 @@ +/** + * CustomModelWizard module exports. + * + * Architecture: + * - types.ts: Shared interfaces (WizardStep, ProviderConfig, validators) + * - provider-config.ts: Provider registry with steps, display names, validation + * - shared/: Reusable UI components (ProviderSelector, WizardStepInput, etc.) + * + * The main CustomModelWizard.tsx uses these modules instead of having + * all provider-specific logic scattered throughout. + */ + +export * from './types.js'; +export * from './provider-config.js'; +export * from './shared/index.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/provider-config.ts b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/provider-config.ts new file mode 100644 index 00000000..b38d5252 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/provider-config.ts @@ -0,0 +1,566 @@ +/** + * Provider configuration registry for the CustomModelWizard. + * Each provider has its own config with display name, description, steps, and model builder. + */ + +import type { CustomModel, CustomModelProvider } from '@dexto/agent-management'; +import { CUSTOM_MODEL_PROVIDERS, isDextoAuthEnabled } from '@dexto/agent-management'; +import { + lookupOpenRouterModel, + refreshOpenRouterModelCache, + getLocalModelById, + isReasoningCapableModel, +} from '@dexto/core'; +import type { ProviderConfig, WizardStep } from './types.js'; +import { validators } from './types.js'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +/** + * Common API key step - reused across providers that support API keys. + */ +const API_KEY_STEP: WizardStep = { + field: 'apiKey', + label: 'API Key (optional)', + placeholder: 'Enter API key for authentication', + required: false, +}; + +/** + * Common max input tokens step - reused across providers that support it. + */ +const MAX_INPUT_TOKENS_STEP: WizardStep = { + field: 'maxInputTokens', + label: 'Max Input Tokens (optional)', + placeholder: 'e.g., 128000 (leave blank for default)', + required: false, + validate: validators.positiveNumber, +}; + +/** + * Common display name step. + */ +const DISPLAY_NAME_STEP: WizardStep = { + field: 'displayName', + label: 'Display Name (optional)', + placeholder: 'e.g., My Custom Model', + required: false, +}; + +/** + * Common reasoning effort step - for OpenAI reasoning models (o1, o3, codex, gpt-5.x). + * Only shown when the model name indicates reasoning capability. + */ +const REASONING_EFFORT_STEP: WizardStep = { + field: 'reasoningEffort', + label: 'Reasoning Effort (optional)', + placeholder: 'none | minimal | low | medium | high | xhigh (blank for auto)', + required: false, + validate: (value: string) => { + if (!value?.trim()) return null; + const validValues = ['none', 'minimal', 'low', 'medium', 'high', 'xhigh']; + if (!validValues.includes(value.toLowerCase())) { + return `Invalid reasoning effort. Use: ${validValues.join(', ')}`; + } + return null; + }, + condition: (values) => { + const modelName = values.name || ''; + return isReasoningCapableModel(modelName); + }, +}; + +/** + * Provider configuration registry. + * Keys are CustomModelProvider values. + */ +export const PROVIDER_CONFIGS: Record<CustomModelProvider, ProviderConfig> = { + 'openai-compatible': { + displayName: 'OpenAI-Compatible', + description: 'Custom or self-hosted endpoints (vLLM, LM Studio)', + steps: [ + { + field: 'name', + label: 'Model Name', + placeholder: 'e.g., llama-3-70b, mixtral-8x7b', + required: true, + validate: validators.required('Model name'), + }, + { + field: 'baseURL', + label: 'API Base URL', + placeholder: 'e.g., http://localhost:11434/v1', + required: true, + validate: validators.url, + }, + { ...DISPLAY_NAME_STEP, placeholder: 'e.g., My Local Llama 3' }, + MAX_INPUT_TOKENS_STEP, + REASONING_EFFORT_STEP, + API_KEY_STEP, + ], + buildModel: (values, provider) => { + const model: CustomModel = { + name: values.name || '', + provider, + }; + if (values.baseURL) { + model.baseURL = values.baseURL; + } + if (values.displayName?.trim()) { + model.displayName = values.displayName.trim(); + } + if (values.maxInputTokens?.trim()) { + model.maxInputTokens = parseInt(values.maxInputTokens, 10); + } + if (values.reasoningEffort?.trim()) { + model.reasoningEffort = + values.reasoningEffort.toLowerCase() as CustomModel['reasoningEffort']; + } + return model; + }, + }, + + openrouter: { + displayName: 'OpenRouter', + description: '100+ cloud models via unified API', + steps: [ + { + field: 'name', + label: 'OpenRouter Model ID', + placeholder: 'e.g., anthropic/claude-3.5-sonnet, openai/gpt-4o', + required: true, + validate: validators.slashFormat, + }, + { ...DISPLAY_NAME_STEP, placeholder: 'e.g., Claude 3.5 Sonnet' }, + REASONING_EFFORT_STEP, + { + ...API_KEY_STEP, + placeholder: 'Saved as OPENROUTER_API_KEY if not set, otherwise per-model', + }, + ], + buildModel: (values, provider) => { + const model: CustomModel = { + name: values.name || '', + provider, + }; + if (values.displayName?.trim()) { + model.displayName = values.displayName.trim(); + } + if (values.reasoningEffort?.trim()) { + model.reasoningEffort = + values.reasoningEffort.toLowerCase() as CustomModel['reasoningEffort']; + } + return model; + }, + asyncValidation: { + field: 'name', + validate: async (modelId: string) => { + let status = lookupOpenRouterModel(modelId); + + // If cache is stale/empty, try to refresh + if (status === 'unknown') { + try { + await refreshOpenRouterModelCache(); + status = lookupOpenRouterModel(modelId); + } catch { + // Network failed - allow the model (graceful degradation) + return null; + } + } + + if (status === 'invalid') { + return `Model '${modelId}' not found in OpenRouter. Check the model ID at https://openrouter.ai/models`; + } + + return null; + }, + }, + }, + + glama: { + displayName: 'Glama', + description: 'OpenAI-compatible gateway', + steps: [ + { + field: 'name', + label: 'Glama Model ID', + placeholder: 'e.g., openai/gpt-4o, anthropic/claude-3-sonnet', + required: true, + validate: validators.slashFormat, + }, + { ...DISPLAY_NAME_STEP, placeholder: 'e.g., GPT-4o via Glama' }, + REASONING_EFFORT_STEP, + { + ...API_KEY_STEP, + placeholder: 'Saved as GLAMA_API_KEY if not set, otherwise per-model', + }, + ], + buildModel: (values, provider) => { + const model: CustomModel = { + name: values.name || '', + provider, + }; + if (values.displayName?.trim()) { + model.displayName = values.displayName.trim(); + } + if (values.reasoningEffort?.trim()) { + model.reasoningEffort = + values.reasoningEffort.toLowerCase() as CustomModel['reasoningEffort']; + } + return model; + }, + }, + + litellm: { + displayName: 'LiteLLM', + description: 'Unified proxy for 100+ providers', + steps: [ + { + field: 'name', + label: 'Model Name', + placeholder: 'e.g., gpt-4, claude-3-sonnet, bedrock/anthropic.claude-v2', + required: true, + validate: validators.required('Model name'), + }, + { + field: 'baseURL', + label: 'LiteLLM Proxy URL', + placeholder: 'e.g., http://localhost:4000', + required: true, + validate: validators.url, + }, + { ...DISPLAY_NAME_STEP, placeholder: 'e.g., My LiteLLM GPT-4' }, + MAX_INPUT_TOKENS_STEP, + REASONING_EFFORT_STEP, + { + ...API_KEY_STEP, + placeholder: 'Saved as LITELLM_API_KEY if not set, otherwise per-model', + }, + ], + buildModel: (values, provider) => { + const model: CustomModel = { + name: values.name || '', + provider, + }; + if (values.baseURL) { + model.baseURL = values.baseURL; + } + if (values.displayName?.trim()) { + model.displayName = values.displayName.trim(); + } + if (values.maxInputTokens?.trim()) { + model.maxInputTokens = parseInt(values.maxInputTokens, 10); + } + if (values.reasoningEffort?.trim()) { + model.reasoningEffort = + values.reasoningEffort.toLowerCase() as CustomModel['reasoningEffort']; + } + return model; + }, + }, + + bedrock: { + displayName: 'AWS Bedrock', + description: 'Custom model IDs via AWS credentials', + steps: [ + { + field: 'name', + label: 'Bedrock Model ID', + placeholder: 'e.g., anthropic.claude-3-haiku-20240307-v1:0', + required: true, + validate: validators.required('Model ID'), + }, + { ...DISPLAY_NAME_STEP, placeholder: 'e.g., Claude 3 Haiku' }, + { + ...MAX_INPUT_TOKENS_STEP, + placeholder: 'e.g., 200000 (leave blank for default)', + }, + // NO apiKey step - Bedrock uses AWS credentials from environment + ], + buildModel: (values, provider) => { + const model: CustomModel = { + name: values.name || '', + provider, + }; + if (values.displayName?.trim()) { + model.displayName = values.displayName.trim(); + } + if (values.maxInputTokens?.trim()) { + model.maxInputTokens = parseInt(values.maxInputTokens, 10); + } + return model; + }, + setupInfo: { + title: 'AWS Bedrock Setup', + description: + 'Bedrock uses AWS credentials from your environment. Ensure AWS_REGION and either AWS_BEARER_TOKEN_BEDROCK or IAM credentials are set.', + docsUrl: 'https://docs.dexto.ai/docs/guides/supported-llm-providers#amazon-bedrock', + }, + }, + + ollama: { + displayName: 'Ollama', + description: 'Local Ollama server models', + steps: [ + { + field: 'name', + label: 'Model Name', + placeholder: 'e.g., llama3.3:70b, qwen3n:e2b', + required: true, + validate: validators.required('Model name'), + }, + { + field: 'baseURL', + label: 'Ollama Server URL (optional)', + placeholder: 'Default: http://localhost:11434', + required: false, + validate: (value) => { + if (!value?.trim()) return null; + return validators.url(value); + }, + }, + { ...DISPLAY_NAME_STEP, placeholder: 'e.g., Llama 3.3 70B' }, + MAX_INPUT_TOKENS_STEP, + ], + buildModel: (values, provider) => { + const model: CustomModel = { + name: values.name || '', + provider, + }; + if (values.baseURL?.trim()) { + model.baseURL = values.baseURL.trim(); + } + if (values.displayName?.trim()) { + model.displayName = values.displayName.trim(); + } + if (values.maxInputTokens?.trim()) { + model.maxInputTokens = parseInt(values.maxInputTokens, 10); + } + return model; + }, + setupInfo: { + title: 'Ollama Setup', + description: + 'Add custom Ollama models by name. Ensure Ollama is running (default: http://localhost:11434). Pull models with: ollama pull <model>', + docsUrl: 'https://docs.dexto.ai/docs/guides/supported-llm-providers#ollama', + }, + }, + + local: { + displayName: 'Local (node-llama)', + description: 'Custom GGUF models via node-llama-cpp', + steps: [ + { + field: 'name', + label: 'Model ID or Path', + placeholder: 'e.g., llama-3.3-8b-q4 or /path/to/model.gguf', + required: true, + validate: validators.required('Model ID or path'), + }, + { ...DISPLAY_NAME_STEP, placeholder: 'e.g., My Custom Llama' }, + // Note: No MAX_INPUT_TOKENS_STEP - node-llama-cpp auto-detects context size + ], + buildModel: (values, provider) => { + const model: CustomModel = { + name: values.name || '', + provider, + }; + if (values.displayName?.trim()) { + model.displayName = values.displayName.trim(); + } + return model; + }, + asyncValidation: { + field: 'name', + validate: async (value: string) => { + const trimmed = value.trim(); + + // Check if it looks like a file path (contains path separator or ends with .gguf) + const isFilePath = + trimmed.includes(path.sep) || + trimmed.startsWith('/') || + trimmed.startsWith('~') || + trimmed.toLowerCase().endsWith('.gguf'); + + if (isFilePath) { + // Validate as file path + if (!trimmed.toLowerCase().endsWith('.gguf')) { + return 'File path must end with .gguf'; + } + + // Expand ~ to home directory (use os.homedir() for cross-platform support) + const expandedPath = trimmed.startsWith('~') + ? trimmed.replace('~', os.homedir()) + : trimmed; + + if (!fs.existsSync(expandedPath)) { + return `File not found: ${trimmed}`; + } + + return null; // Valid file path + } + + // Otherwise, validate as model ID from registry + const modelInfo = getLocalModelById(trimmed); + if (!modelInfo) { + return `Model ID '${trimmed}' not found in registry. Use a full file path to a .gguf file, or one of the registry IDs (e.g., llama-3.3-8b-q4, qwen-2.5-coder-7b-q4)`; + } + + return null; // Valid registry ID + }, + }, + setupInfo: { + title: 'Local Models Setup', + description: + 'Add custom GGUF models by ID (from registry) or absolute file path. Ensure node-llama-cpp is installed and GPU acceleration is configured.', + docsUrl: 'https://docs.dexto.ai/docs/guides/supported-llm-providers#local-models', + }, + }, + + vertex: { + displayName: 'Google Vertex AI', + description: 'Custom Vertex model IDs', + steps: [ + { + field: 'name', + label: 'Vertex Model ID', + placeholder: 'e.g., gemini-2.0-flash-exp, claude-4-5-sonnet@20250929', + required: true, + validate: validators.required('Model ID'), + }, + { ...DISPLAY_NAME_STEP, placeholder: 'e.g., Gemini 2.0 Flash Exp' }, + { + ...MAX_INPUT_TOKENS_STEP, + placeholder: 'e.g., 1048576 (leave blank for default)', + }, + ], + buildModel: (values, provider) => { + const model: CustomModel = { + name: values.name || '', + provider, + }; + if (values.displayName?.trim()) { + model.displayName = values.displayName.trim(); + } + if (values.maxInputTokens?.trim()) { + model.maxInputTokens = parseInt(values.maxInputTokens, 10); + } + return model; + }, + setupInfo: { + title: 'Google Vertex AI Setup', + description: + 'Vertex AI uses Google Cloud Application Default Credentials (ADC). Set GOOGLE_VERTEX_PROJECT and optionally GOOGLE_VERTEX_LOCATION. Run: gcloud auth application-default login', + docsUrl: 'https://docs.dexto.ai/docs/guides/supported-llm-providers#google-vertex-ai', + }, + }, + + dexto: { + displayName: 'Dexto', + description: 'Access 100+ models with Dexto credits', + steps: [ + { + field: 'name', + label: 'Model ID (OpenRouter format)', + placeholder: 'e.g., anthropic/claude-sonnet-4.5, openai/gpt-5.2', + required: true, + validate: validators.slashFormat, + }, + { ...DISPLAY_NAME_STEP, placeholder: 'e.g., Claude 4.5 Sonnet via Dexto' }, + REASONING_EFFORT_STEP, + // No API key step - Dexto uses OAuth login (DEXTO_API_KEY from auth.json) + ], + buildModel: (values, provider) => { + const model: CustomModel = { + name: values.name || '', + provider, + }; + if (values.displayName?.trim()) { + model.displayName = values.displayName.trim(); + } + if (values.reasoningEffort?.trim()) { + model.reasoningEffort = + values.reasoningEffort.toLowerCase() as CustomModel['reasoningEffort']; + } + return model; + }, + asyncValidation: { + field: 'name', + validate: async (modelId: string) => { + // Reuse OpenRouter validation since Dexto uses OpenRouter model IDs + let status = lookupOpenRouterModel(modelId); + + // If cache is stale/empty, try to refresh + if (status === 'unknown') { + try { + await refreshOpenRouterModelCache(); + status = lookupOpenRouterModel(modelId); + } catch { + // Network failed - allow the model (graceful degradation) + return null; + } + } + + if (status === 'invalid') { + return `Model '${modelId}' not found. Dexto uses OpenRouter model IDs - check https://openrouter.ai/models`; + } + + return null; + }, + }, + setupInfo: { + title: 'Dexto Setup', + description: + 'Add OpenRouter-format models that use your Dexto credits. Requires login: run `dexto login` first.', + docsUrl: 'https://openrouter.ai/models', + }, + }, +}; + +/** + * Get provider config by provider type. + */ +export function getProviderConfig(provider: CustomModelProvider): ProviderConfig { + return PROVIDER_CONFIGS[provider]; +} + +/** + * Get display label for provider selection menu. + * Format: "DisplayName (description)" + */ +export function getProviderLabel(provider: CustomModelProvider): string { + const config = PROVIDER_CONFIGS[provider]; + return `${config.displayName} (${config.description})`; +} + +/** + * Get all available provider types. + * Filters out 'dexto' when the feature flag is disabled. + */ +export function getAvailableProviders(): CustomModelProvider[] { + const dextoEnabled = isDextoAuthEnabled(); + return CUSTOM_MODEL_PROVIDERS.filter((provider) => provider !== 'dexto' || dextoEnabled); +} + +/** + * Check if a provider has async validation. + */ +export function hasAsyncValidation(provider: CustomModelProvider): boolean { + return !!PROVIDER_CONFIGS[provider].asyncValidation; +} + +/** + * Run async validation for a provider's field if applicable. + */ +export async function runAsyncValidation( + provider: CustomModelProvider, + field: string, + value: string +): Promise<string | null> { + const config = PROVIDER_CONFIGS[provider]; + if (config.asyncValidation && config.asyncValidation.field === field) { + return config.asyncValidation.validate(value); + } + return null; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/ApiKeyStep.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/ApiKeyStep.tsx new file mode 100644 index 00000000..fd61ad42 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/ApiKeyStep.tsx @@ -0,0 +1,47 @@ +/** + * ApiKeyStep Component + * Renders API key status and help text for providers that support API keys. + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import { getProviderKeyStatus } from '@dexto/agent-management'; +import type { LLMProvider } from '@dexto/core'; +import type { CustomModelProvider } from '@dexto/agent-management'; + +interface ApiKeyStepProps { + /** The provider type */ + provider: CustomModelProvider; +} + +/** + * Displays API key status for the current provider. + * Shows whether the key is already configured or needs to be set. + */ +export function ApiKeyStep({ provider }: ApiKeyStepProps): React.ReactElement { + const keyStatus = getProviderKeyStatus(provider as LLMProvider); + + if (keyStatus.hasApiKey) { + return <Text color="green">✓ {keyStatus.envVar} already set, press Enter to skip</Text>; + } + + return <Text color="yellowBright">No {keyStatus.envVar} configured</Text>; +} + +/** + * Get the env var name for a provider's API key. + */ +export function getProviderEnvVar(provider: CustomModelProvider): string { + const keyStatus = getProviderKeyStatus(provider as LLMProvider); + return keyStatus.envVar; +} + +/** + * Check if a provider has an API key configured. + */ +export function hasApiKeyConfigured(provider: CustomModelProvider): boolean { + const keyStatus = getProviderKeyStatus(provider as LLMProvider); + return keyStatus.hasApiKey; +} + +export default ApiKeyStep; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/ProviderSelector.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/ProviderSelector.tsx new file mode 100644 index 00000000..75ea28b0 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/ProviderSelector.tsx @@ -0,0 +1,64 @@ +/** + * ProviderSelector Component + * Renders the provider selection screen with navigation. + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { CustomModelProvider } from '@dexto/agent-management'; +import { getProviderLabel, getAvailableProviders } from '../provider-config.js'; + +interface ProviderSelectorProps { + /** Currently highlighted provider index */ + selectedIndex: number; + /** Whether we're editing an existing model */ + isEditing: boolean; +} + +/** + * Renders a list of available providers with the current selection highlighted. + */ +export function ProviderSelector({ + selectedIndex, + isEditing, +}: ProviderSelectorProps): React.ReactElement { + const providers = getAvailableProviders(); + + return ( + <Box + flexDirection="column" + borderStyle="round" + borderColor="green" + paddingX={1} + marginTop={1} + > + <Box marginBottom={1}> + <Text bold color="green"> + {isEditing ? 'Edit Custom Model' : 'Add Custom Model'} + </Text> + </Box> + + <Text bold>Select Provider:</Text> + + <Box flexDirection="column" marginTop={1}> + {providers.map((provider, index) => ( + <Box key={provider}> + <Text + color={index === selectedIndex ? 'cyan' : 'gray'} + bold={index === selectedIndex} + > + {index === selectedIndex ? '❯ ' : ' '} + {getProviderLabel(provider)} + </Text> + </Box> + ))} + </Box> + + <Box marginTop={1}> + <Text color="gray">↑↓ navigate • Enter select • Esc cancel</Text> + </Box> + </Box> + ); +} + +export default ProviderSelector; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/SetupInfoBanner.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/SetupInfoBanner.tsx new file mode 100644 index 00000000..e451da4e --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/SetupInfoBanner.tsx @@ -0,0 +1,36 @@ +/** + * SetupInfoBanner Component + * Displays setup information for providers that need special configuration. + */ + +import React from 'react'; +import { Box, Text } from 'ink'; + +interface SetupInfoBannerProps { + /** Banner title */ + title: string; + /** Description text */ + description: string; + /** Optional documentation URL */ + docsUrl?: string | undefined; +} + +/** + * Displays a setup info banner with title, description, and optional docs link. + * Used for providers like Bedrock that require special configuration. + */ +export function SetupInfoBanner({ + title, + description, + docsUrl, +}: SetupInfoBannerProps): React.ReactElement { + return ( + <Box flexDirection="column" marginBottom={1}> + <Text color="blue">ℹ {title}</Text> + <Text color="gray">{description}</Text> + {docsUrl && <Text color="gray">Setup guide: {docsUrl}</Text>} + </Box> + ); +} + +export default SetupInfoBanner; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/WizardStepInput.tsx b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/WizardStepInput.tsx new file mode 100644 index 00000000..24caccef --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/WizardStepInput.tsx @@ -0,0 +1,80 @@ +/** + * WizardStepInput Component + * Renders a single wizard step with label, placeholder, input, and error display. + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { WizardStep } from '../types.js'; + +interface WizardStepInputProps { + /** Current step configuration */ + step: WizardStep; + /** Current input value */ + currentInput: string; + /** Error message (if any) */ + error: string | null; + /** Whether validation is in progress */ + isValidating: boolean; + /** Whether saving is in progress */ + isSaving: boolean; + /** Optional additional content to render after placeholder (e.g., API key status) */ + additionalContent?: React.ReactNode; +} + +/** + * Renders a wizard step with: + * - Step label and placeholder text + * - Text input field with cursor + * - Error message (if any) + * - Validation/saving indicators + */ +export function WizardStepInput({ + step, + currentInput, + error, + isValidating, + isSaving, + additionalContent, +}: WizardStepInputProps): React.ReactElement { + return ( + <> + {/* Current step prompt */} + <Box flexDirection="column"> + <Text bold>{step.label}:</Text> + <Text color="gray">{step.placeholder}</Text> + {additionalContent} + </Box> + + {/* Input field */} + <Box marginTop={1}> + <Text color="cyan">> </Text> + <Text>{currentInput}</Text> + <Text color="cyan">_</Text> + </Box> + + {/* Error message */} + {error && ( + <Box marginTop={1}> + <Text color="red">{error}</Text> + </Box> + )} + + {/* Validating indicator */} + {isValidating && ( + <Box marginTop={1}> + <Text color="yellowBright">Validating model...</Text> + </Box> + )} + + {/* Saving indicator */} + {isSaving && ( + <Box marginTop={1}> + <Text color="yellowBright">Saving...</Text> + </Box> + )} + </> + ); +} + +export default WizardStepInput; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/index.ts b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/index.ts new file mode 100644 index 00000000..85ff86b1 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/shared/index.ts @@ -0,0 +1,8 @@ +/** + * Shared components for the CustomModelWizard. + */ + +export { ApiKeyStep, getProviderEnvVar, hasApiKeyConfigured } from './ApiKeyStep.js'; +export { SetupInfoBanner } from './SetupInfoBanner.js'; +export { WizardStepInput } from './WizardStepInput.js'; +export { ProviderSelector } from './ProviderSelector.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/types.ts b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/types.ts new file mode 100644 index 00000000..996ba33f --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/overlays/custom-model-wizard/types.ts @@ -0,0 +1,113 @@ +/** + * Shared types for the CustomModelWizard component architecture. + */ + +import type { CustomModel, CustomModelProvider } from '@dexto/agent-management'; +import type { Key } from '../../../hooks/useInputOrchestrator.js'; + +/** + * A single step in the wizard flow. + */ +export interface WizardStep { + field: string; + label: string; + placeholder: string; + required: boolean; + validate?: (value: string) => string | null; + /** + * Optional condition to determine if this step should be shown. + * Takes the current accumulated values and returns true if the step should be shown. + * If omitted, the step is always shown. + */ + condition?: (values: Record<string, string>) => boolean; +} + +/** + * Props passed to each provider-specific wizard component. + */ +export interface ProviderWizardProps { + /** Current accumulated values from all steps */ + values: Record<string, string>; + /** Current step index within the provider's steps */ + currentStep: number; + /** Current text input value */ + currentInput: string; + /** Validation error message (if any) */ + error: string | null; + /** Whether async validation is in progress */ + isValidating: boolean; + /** Whether the model is being saved */ + isSaving: boolean; + /** Whether we're editing an existing model */ + isEditing: boolean; +} + +/** + * Handle returned by provider components for input handling. + */ +export interface ProviderWizardHandle { + /** Handle keyboard input. Returns true if handled. */ + handleInput: (input: string, key: Key) => boolean; + /** Get the steps for this provider */ + getSteps: () => WizardStep[]; + /** Get current step config */ + getCurrentStepConfig: () => WizardStep | undefined; +} + +/** + * Configuration for a custom model provider. + */ +export interface ProviderConfig { + /** Display name shown in provider selector */ + displayName: string; + /** Short description of the provider */ + description: string; + /** Wizard steps for this provider */ + steps: WizardStep[]; + /** Build a CustomModel from accumulated values */ + buildModel: (values: Record<string, string>, provider: CustomModelProvider) => CustomModel; + /** Optional async validation for specific fields */ + asyncValidation?: { + field: string; + validate: (value: string) => Promise<string | null>; + }; + /** Whether this provider needs a setup info banner */ + setupInfo?: { + title: string; + description: string; + docsUrl?: string; + }; +} + +/** + * Common validation functions + */ +export const validators = { + required: (label: string) => (v: string) => (v.trim() ? null : `${label} is required`), + + url: (v: string) => { + if (!v.trim()) return 'URL is required'; + try { + const url = new URL(v); + if (!['http:', 'https:'].includes(url.protocol)) { + return 'URL must use http:// or https://'; + } + return null; + } catch { + return 'Invalid URL format'; + } + }, + + positiveNumber: (v: string) => { + if (!v.trim()) return null; // Optional field + const num = parseInt(v, 10); + if (isNaN(num) || num <= 0) return 'Must be a positive number'; + return null; + }, + + slashFormat: (v: string) => { + if (!v.trim()) return 'Model ID is required'; + if (!v.includes('/')) return 'Must use format: provider/model-name'; + return null; + }, +}; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/renderers/DiffRenderer.tsx b/dexto/packages/cli/src/cli/ink-cli/components/renderers/DiffRenderer.tsx new file mode 100644 index 00000000..80c045f0 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/renderers/DiffRenderer.tsx @@ -0,0 +1,147 @@ +/** + * DiffRenderer Component + * + * Renders unified diff output with colored lines and line numbers. + * Used for edit_file and write_file (overwrite) tool results in message list. + * Matches the approval preview UX style. + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { DiffDisplayData } from '@dexto/core'; +import { makeRelativePath } from '../../utils/messageFormatting.js'; +import { + parseUnifiedDiff, + findLinePairs, + computeWordDiff, + getLineNumWidth, + DiffLine, + HunkSeparator, +} from './diff-shared.js'; + +interface DiffRendererProps { + /** Diff display data from tool result */ + data: DiffDisplayData; + /** Maximum lines to display before truncating */ + maxLines?: number; +} + +/** + * Renders unified diff with colored lines, line numbers, and word-level highlighting. + * Matches the approval preview UX style. No truncation by default. + */ +export function DiffRenderer({ data, maxLines = Infinity }: DiffRendererProps) { + const { unified, filename, additions, deletions } = data; + const hunks = parseUnifiedDiff(unified); + + // Calculate max line number for width + let maxLineNum = 1; + for (const hunk of hunks) { + for (const line of hunk.lines) { + maxLineNum = Math.max(maxLineNum, line.lineNum); + } + } + const lineNumWidth = getLineNumWidth(maxLineNum); + + // Find line pairs for word-level diff + const allLinePairs = hunks.map((hunk) => findLinePairs(hunk.lines)); + + // Count total display lines and apply truncation + let totalLines = 0; + for (const hunk of hunks) { + totalLines += hunk.lines.length; + } + + const shouldTruncate = totalLines > maxLines; + let linesRendered = 0; + + return ( + <Box flexDirection="column"> + {/* Header */} + <Box> + <Text color="gray">{' ⎿ '}</Text> + <Text>{makeRelativePath(filename)}</Text> + <Text color="green"> +{additions}</Text> + <Text color="red"> -{deletions}</Text> + </Box> + + {/* Diff content - paddingLeft keeps backgrounds bounded within container */} + <Box flexDirection="column" paddingLeft={2}> + {hunks.map((hunk, hunkIndex) => { + if (shouldTruncate && linesRendered >= maxLines) { + return null; + } + + const linePairs = allLinePairs[hunkIndex]!; + const processedIndices = new Set<number>(); + + return ( + <React.Fragment key={hunkIndex}> + {hunkIndex > 0 && <HunkSeparator />} + + {hunk.lines.map((line, lineIndex) => { + if (shouldTruncate && linesRendered >= maxLines) { + return null; + } + + if (processedIndices.has(lineIndex)) { + return null; + } + + const pair = linePairs.get(lineIndex); + if (pair) { + processedIndices.add(lineIndex + 1); + + if (shouldTruncate && linesRendered + 2 > maxLines) { + linesRendered = maxLines; + return null; + } + + linesRendered += 2; + + const { oldParts, newParts } = computeWordDiff( + pair.del.content, + pair.add.content + ); + + return ( + <React.Fragment key={lineIndex}> + <DiffLine + type="deletion" + lineNum={pair.del.lineNum} + lineNumWidth={lineNumWidth} + content={pair.del.content} + wordDiffParts={oldParts} + /> + <DiffLine + type="addition" + lineNum={pair.add.lineNum} + lineNumWidth={lineNumWidth} + content={pair.add.content} + wordDiffParts={newParts} + /> + </React.Fragment> + ); + } + + linesRendered++; + + return ( + <DiffLine + key={lineIndex} + type={line.type} + lineNum={line.lineNum} + lineNumWidth={lineNumWidth} + content={line.content} + /> + ); + })} + </React.Fragment> + ); + })} + + {shouldTruncate && <Text color="gray">... +{totalLines - maxLines} lines</Text>} + </Box> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/renderers/FilePreviewRenderer.tsx b/dexto/packages/cli/src/cli/ink-cli/components/renderers/FilePreviewRenderer.tsx new file mode 100644 index 00000000..a3e1bcd1 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/renderers/FilePreviewRenderer.tsx @@ -0,0 +1,202 @@ +/** + * File Preview Renderers + * + * Preview renderers for edit_file and write_file approval prompts. + * - DiffPreview: for edit_file and write_file (overwrite) + * - CreateFilePreview: for write_file (new file) + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { DiffDisplayData, FileDisplayData } from '@dexto/core'; +import { makeRelativePath } from '../../utils/messageFormatting.js'; +import { + parseUnifiedDiff, + findLinePairs, + computeWordDiff, + getLineNumWidth, + formatLineNum, + DiffLine, + HunkSeparator, +} from './diff-shared.js'; + +// ============================================================================= +// DiffPreview Component +// ============================================================================= + +interface DiffPreviewProps { + data: DiffDisplayData; + /** Header text: "Edit file" or "Overwrite file" */ + headerType: 'edit' | 'overwrite'; +} + +/** + * Enhanced diff preview for edit_file and write_file (overwrite) approval + * Shows full diff with line backgrounds, hunk collapsing, and word-level highlighting + */ +export function DiffPreview({ data, headerType }: DiffPreviewProps) { + const { unified, filename } = data; + const hunks = parseUnifiedDiff(unified); + + // Calculate max line number for width + let maxLineNum = 1; + for (const hunk of hunks) { + for (const line of hunk.lines) { + maxLineNum = Math.max(maxLineNum, line.lineNum); + } + } + const lineNumWidth = getLineNumWidth(maxLineNum); + + // Find line pairs for word-level diff + const allLinePairs = hunks.map((hunk) => findLinePairs(hunk.lines)); + + const headerText = headerType === 'edit' ? 'Edit file' : 'Overwrite file'; + + return ( + <Box flexDirection="column" marginBottom={1}> + {/* Header - standalone line */} + <Box marginBottom={0}> + <Text color="cyan" bold> + {headerText} + </Text> + </Box> + + {/* Box containing filename and diff content */} + <Box flexDirection="column" borderStyle="round" borderColor="gray" paddingX={1}> + {/* Filename */} + <Box marginBottom={0}> + <Text>{makeRelativePath(filename)}</Text> + </Box> + {hunks.map((hunk, hunkIndex) => { + const linePairs = allLinePairs[hunkIndex]!; + const processedIndices = new Set<number>(); + + return ( + <React.Fragment key={hunkIndex}> + {/* Hunk separator (except for first hunk) */} + {hunkIndex > 0 && <HunkSeparator />} + + {hunk.lines.map((line, lineIndex) => { + // Skip if already processed as part of a pair + if (processedIndices.has(lineIndex)) { + return null; + } + + // Check if this is part of a deletion/addition pair + const pair = linePairs.get(lineIndex); + if (pair) { + // Mark the addition line as processed + processedIndices.add(lineIndex + 1); + + // Compute word-level diff + const { oldParts, newParts } = computeWordDiff( + pair.del.content, + pair.add.content + ); + + return ( + <React.Fragment key={lineIndex}> + <DiffLine + type="deletion" + lineNum={pair.del.lineNum} + lineNumWidth={lineNumWidth} + content={pair.del.content} + wordDiffParts={oldParts} + /> + <DiffLine + type="addition" + lineNum={pair.add.lineNum} + lineNumWidth={lineNumWidth} + content={pair.add.content} + wordDiffParts={newParts} + /> + </React.Fragment> + ); + } + + // Regular line (no word-level diff) + return ( + <DiffLine + key={lineIndex} + type={line.type} + lineNum={line.lineNum} + lineNumWidth={lineNumWidth} + content={line.content} + /> + ); + })} + </React.Fragment> + ); + })} + </Box> + </Box> + ); +} + +// ============================================================================= +// CreateFilePreview Component +// ============================================================================= + +interface CreateFilePreviewProps { + data: FileDisplayData; + /** Custom header text (defaults to "Create file") */ + header?: string; +} + +/** + * Preview for write_file (new file creation) or plan review + * Shows full file content with line numbers, white text + */ +export function CreateFilePreview({ data, header = 'Create file' }: CreateFilePreviewProps) { + const { path, content, lineCount } = data; + + if (!content) { + // Fallback if content not provided + return ( + <Box flexDirection="column" marginBottom={1}> + <Box marginBottom={0}> + <Text color="cyan" bold> + {header} + </Text> + </Box> + <Box flexDirection="column" borderStyle="round" borderColor="gray" paddingX={1}> + <Text>{makeRelativePath(path)}</Text> + {lineCount && <Text color="gray">{lineCount} lines</Text>} + </Box> + </Box> + ); + } + + const lines = content.split('\n'); + const lineNumWidth = getLineNumWidth(lines.length); + + return ( + <Box flexDirection="column" marginBottom={1}> + {/* Header - standalone line */} + <Box marginBottom={0}> + <Text color="cyan" bold> + {header} + </Text> + </Box> + + {/* Box containing filename and file content */} + <Box flexDirection="column" borderStyle="round" borderColor="gray" paddingX={1}> + {/* Filename */} + <Box marginBottom={0}> + <Text>{makeRelativePath(path)}</Text> + </Box> + + {/* File content */} + {lines.map((line, index) => ( + <Box key={index}> + <Text color="gray">{formatLineNum(index + 1, lineNumWidth)}</Text> + <Text> + {' '} + {line} + </Text> + </Box> + ))} + </Box> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/renderers/FileRenderer.tsx b/dexto/packages/cli/src/cli/ink-cli/components/renderers/FileRenderer.tsx new file mode 100644 index 00000000..6706ae3f --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/renderers/FileRenderer.tsx @@ -0,0 +1,58 @@ +/** + * FileRenderer Component + * + * Renders file operation status. + * - Read: "Read N lines" + * - Write/Create: "Wrote N lines to filename" + */ + +import React from 'react'; +import { Text } from 'ink'; +import type { FileDisplayData } from '@dexto/core'; + +interface FileRendererProps { + /** File display data from tool result */ + data: FileDisplayData; +} + +/** + * Renders file operation status. + * Uses ⎿ character for continuation lines. + */ +export function FileRenderer({ data }: FileRendererProps) { + const { operation, lineCount } = data; + + // Format based on operation type + if (operation === 'read') { + // Format: "Read N lines" + const lineText = lineCount !== undefined ? `${lineCount} lines` : 'file'; + return ( + <Text color="gray"> + {' ⎿ '}Read {lineText} + </Text> + ); + } + + // For write/create operations + if (operation === 'write' || operation === 'create') { + const lineText = lineCount !== undefined ? `${lineCount} lines` : 'content'; + return ( + <Text color="gray"> + {' ⎿ '}Wrote {lineText} + </Text> + ); + } + + // Delete operation + if (operation === 'delete') { + return <Text color="gray">{' ⎿ '}Deleted file</Text>; + } + + // Fallback + return ( + <Text color="gray"> + {' ⎿ '} + {operation} + </Text> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/renderers/GenericRenderer.tsx b/dexto/packages/cli/src/cli/ink-cli/components/renderers/GenericRenderer.tsx new file mode 100644 index 00000000..cbe512ef --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/renderers/GenericRenderer.tsx @@ -0,0 +1,70 @@ +/** + * GenericRenderer Component + * + * Fallback renderer for unknown tools and MCP tools. + * Renders content[] as plain text with truncation. + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { ContentPart } from '@dexto/core'; + +interface GenericRendererProps { + /** Content parts from SanitizedToolResult */ + content: ContentPart[]; + /** Maximum lines to display before truncating */ + maxLines?: number; +} + +/** + * Renders tool result content as plain text. + * Used as fallback for tools without specific display data. + * Uses ⎿ character for continuation lines. + */ +export function GenericRenderer({ content, maxLines = 15 }: GenericRendererProps) { + // Extract text from content parts + const text = content + .filter((part): part is { type: 'text'; text: string } => part.type === 'text') + .map((part) => part.text) + .join('\n'); + + if (!text) { + return null; + } + + const lines = text.split('\n').filter((line) => line.trim().length > 0); + const displayLines = lines.slice(0, maxLines); + const truncated = lines.length > maxLines; + + // For single line results, show inline + if (lines.length === 1 && lines[0]) { + const line = lines[0]; + return ( + <Text color="gray"> + {' ⎿ '} + {line.slice(0, 80)} + {line.length > 80 ? '...' : ''} + </Text> + ); + } + + return ( + <Box flexDirection="column"> + <Text color="gray"> + {' ⎿ '} + {displayLines.length} lines + </Text> + {displayLines.map((line, i) => ( + <Text key={i} color="gray" wrap="truncate"> + {' ⎿ '} + {line} + </Text> + ))} + {truncated && ( + <Text color="gray"> + {' ⎿ '}... {lines.length - maxLines} more lines + </Text> + )} + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/renderers/SearchRenderer.tsx b/dexto/packages/cli/src/cli/ink-cli/components/renderers/SearchRenderer.tsx new file mode 100644 index 00000000..ac86d57b --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/renderers/SearchRenderer.tsx @@ -0,0 +1,67 @@ +/** + * SearchRenderer Component + * + * Renders search results with file:line format. + * Used for grep_content and glob_files tool results. + */ + +import React from 'react'; +import path from 'path'; +import { Box, Text } from 'ink'; +import type { SearchDisplayData } from '@dexto/core'; + +/** + * Convert absolute path to relative path from cwd + */ +function toRelativePath(absolutePath: string): string { + const cwd = process.cwd(); + const relative = path.relative(cwd, absolutePath); + if (relative === '') return '.'; + if (!relative.startsWith('..') && !path.isAbsolute(relative)) { + return relative; + } + return absolutePath; +} + +interface SearchRendererProps { + /** Search display data from tool result */ + data: SearchDisplayData; + /** Maximum matches to display before truncating */ + maxMatches?: number; +} + +/** + * Renders search results with file paths and line numbers. + * Uses ⎿ character for continuation lines. + */ +export function SearchRenderer({ data, maxMatches = 5 }: SearchRendererProps) { + const { pattern, matches, totalMatches, truncated: dataTruncated } = data; + const displayMatches = matches.slice(0, maxMatches); + const truncated = dataTruncated || matches.length > maxMatches; + + return ( + <Box flexDirection="column"> + {/* Summary header */} + <Text color="gray"> + {' ⎿ '} + {totalMatches} match{totalMatches !== 1 ? 'es' : ''} for "{pattern}" + {truncated && ' (truncated)'} + </Text> + + {/* Match results - file paths only for clean output */} + {displayMatches.map((match, i) => ( + <Text key={i} color="gray" wrap="truncate"> + {' ⎿ '} + {toRelativePath(match.file)} + {match.line > 0 && `:${match.line}`} + </Text> + ))} + + {matches.length > maxMatches && ( + <Text color="gray"> + {' ⎿ '}... {matches.length - maxMatches} more + </Text> + )} + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/renderers/ShellRenderer.tsx b/dexto/packages/cli/src/cli/ink-cli/components/renderers/ShellRenderer.tsx new file mode 100644 index 00000000..b286a866 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/renderers/ShellRenderer.tsx @@ -0,0 +1,64 @@ +/** + * ShellRenderer Component + * + * Renders shell command output. + * Shows actual stdout/stderr, limited to 5 lines with "+N lines" truncation. + */ + +import React from 'react'; +import { Box, Text } from 'ink'; +import type { ShellDisplayData } from '@dexto/core'; + +interface ShellRendererProps { + /** Shell display data from tool result */ + data: ShellDisplayData; + /** Maximum lines to display before truncating (default: 10) */ + maxLines?: number; +} + +/** + * Renders shell command result with output. + * Uses ⎿ character for continuation lines. + * Shows just the output, "(No content)" for empty results. + */ +export function ShellRenderer({ data, maxLines = 5 }: ShellRendererProps) { + // Use stdout from display data, fall back to stderr if no stdout + const output = data.stdout || data.stderr || ''; + + const lines = output.split('\n').filter((line) => line.length > 0); + const displayLines = lines.slice(0, maxLines); + const truncatedCount = lines.length - maxLines; + + // No output - show "(No content)" + if (lines.length === 0) { + return <Text color="gray">{' ⎿ '}(No content)</Text>; + } + + // Single line output - show inline + if (lines.length === 1 && lines[0]) { + return ( + <Text color="gray"> + {' ⎿ '} + {lines[0]} + </Text> + ); + } + + // Multi-line output + // TODO: Add ctrl+o expansion to show full output + return ( + <Box flexDirection="column"> + {displayLines.map((line, i) => ( + <Text key={i} color="gray" wrap="truncate"> + {i === 0 ? ' ⎿ ' : ' '} + {line} + </Text> + ))} + {truncatedCount > 0 && ( + <Text color="gray"> + {' '}+{truncatedCount} lines + </Text> + )} + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/renderers/diff-shared.tsx b/dexto/packages/cli/src/cli/ink-cli/components/renderers/diff-shared.tsx new file mode 100644 index 00000000..3f551e7a --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/renderers/diff-shared.tsx @@ -0,0 +1,251 @@ +/** + * Shared Diff Components and Utilities + * + * Common code for DiffRenderer and FilePreviewRenderer. + * Handles unified diff parsing, word-level diff highlighting, and line rendering. + */ + +import React from 'react'; +import { Box, Text, useStdout } from 'ink'; +import { diffWords } from 'diff'; + +// ============================================================================= +// Types +// ============================================================================= + +export interface ParsedHunk { + oldStart: number; + newStart: number; + lines: ParsedLine[]; +} + +export interface ParsedLine { + type: 'context' | 'addition' | 'deletion'; + content: string; + lineNum: number; +} + +export interface WordDiffPart { + value: string; + added?: boolean; + removed?: boolean; +} + +// ============================================================================= +// Diff Parsing +// ============================================================================= + +/** + * Parse unified diff into structured hunks + */ +export function parseUnifiedDiff(unified: string): ParsedHunk[] { + const lines = unified.split('\n'); + const hunks: ParsedHunk[] = []; + let currentHunk: ParsedHunk | null = null; + let oldLine = 0; + let newLine = 0; + + for (const line of lines) { + if (line.startsWith('---') || line.startsWith('+++') || line.startsWith('Index:')) { + continue; + } + + const hunkMatch = line.match(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/); + if (hunkMatch) { + if (currentHunk) { + hunks.push(currentHunk); + } + oldLine = parseInt(hunkMatch[1]!, 10); + newLine = parseInt(hunkMatch[3]!, 10); + currentHunk = { + oldStart: oldLine, + newStart: newLine, + lines: [], + }; + continue; + } + + if (!currentHunk) continue; + + if (line.startsWith('+')) { + currentHunk.lines.push({ + type: 'addition', + content: line.slice(1), + lineNum: newLine++, + }); + } else if (line.startsWith('-')) { + currentHunk.lines.push({ + type: 'deletion', + content: line.slice(1), + lineNum: oldLine++, + }); + } else if (line.startsWith(' ') || line === '') { + currentHunk.lines.push({ + type: 'context', + content: line.startsWith(' ') ? line.slice(1) : line, + lineNum: newLine, + }); + oldLine++; + newLine++; + } + } + + if (currentHunk) { + hunks.push(currentHunk); + } + + return hunks; +} + +/** + * Find paired deletion/addition lines for word-level diff + */ +export function findLinePairs( + lines: ParsedLine[] +): Map<number, { del: ParsedLine; add: ParsedLine }> { + const pairs = new Map<number, { del: ParsedLine; add: ParsedLine }>(); + + for (let i = 0; i < lines.length - 1; i++) { + const current = lines[i]!; + const next = lines[i + 1]!; + + if (current.type === 'deletion' && next.type === 'addition') { + pairs.set(i, { del: current, add: next }); + } + } + + return pairs; +} + +/** + * Compute word-level diff between two strings + */ +export function computeWordDiff( + oldStr: string, + newStr: string +): { oldParts: WordDiffPart[]; newParts: WordDiffPart[] } { + const changes = diffWords(oldStr, newStr); + + const oldParts: WordDiffPart[] = []; + const newParts: WordDiffPart[] = []; + + for (const change of changes) { + if (change.added) { + newParts.push({ value: change.value, added: true }); + } else if (change.removed) { + oldParts.push({ value: change.value, removed: true }); + } else { + oldParts.push({ value: change.value }); + newParts.push({ value: change.value }); + } + } + + return { oldParts, newParts }; +} + +// ============================================================================= +// Line Number Formatting +// ============================================================================= + +export function getLineNumWidth(maxLineNum: number): number { + return Math.max(3, String(maxLineNum).length); +} + +export function formatLineNum(num: number, width: number): string { + return String(num).padStart(width, ' '); +} + +// ============================================================================= +// Diff Line Component +// ============================================================================= + +interface DiffLineProps { + type: 'context' | 'addition' | 'deletion'; + lineNum: number; + lineNumWidth: number; + content: string; + wordDiffParts?: WordDiffPart[]; +} + +/** + * Render a single diff line with true 2-column layout. + * Column 1: Fixed-width gutter (line number + symbol) + * Column 2: Fixed-width content area (remaining terminal width) + * This prevents content/backgrounds from leaking beyond boundaries. + */ +export function DiffLine({ type, lineNum, lineNumWidth, content, wordDiffParts }: DiffLineProps) { + const { stdout } = useStdout(); + const terminalWidth = stdout?.columns ?? 80; + + const lineNumStr = formatLineNum(lineNum, lineNumWidth); + // Gutter: line number + space + symbol + space (e.g., " 42 - ") + const gutterWidth = lineNumWidth + 3; + + // Content column width - remaining space after gutter and padding + // Account for padding (2 chars from DiffRenderer's paddingLeft) + const contentWidth = Math.max(20, terminalWidth - gutterWidth - 4); + + // Get colors based on type + const getColors = () => { + switch (type) { + case 'deletion': + return { bg: '#662222', fg: 'white', symbol: '-', symbolColor: 'red' }; + case 'addition': + return { bg: '#224466', fg: 'white', symbol: '+', symbolColor: 'blue' }; + default: + return { bg: undefined, fg: undefined, symbol: ' ', symbolColor: undefined }; + } + }; + + const colors = getColors(); + + // Render content with word-level diff highlighting if available + const renderContent = () => { + if (wordDiffParts && wordDiffParts.length > 0) { + return wordDiffParts.map((part, i) => { + if (type === 'deletion' && part.removed) { + return ( + <Text key={i} backgroundColor="#882222"> + {part.value} + </Text> + ); + } else if (type === 'addition' && part.added) { + return ( + <Text key={i} backgroundColor="#224488"> + {part.value} + </Text> + ); + } + return <Text key={i}>{part.value}</Text>; + }); + } + return content; + }; + + return ( + <Box> + {/* Column 1: Fixed-width gutter */} + <Box width={gutterWidth} flexShrink={0}> + <Text color="gray">{lineNumStr}</Text> + <Text color={colors.symbolColor as any}> {colors.symbol} </Text> + </Box> + {/* Column 2: Fixed-width content - text wraps within this boundary */} + <Box width={contentWidth} flexShrink={0}> + <Text backgroundColor={colors.bg as any} color={colors.fg as any} wrap="wrap"> + {renderContent()} + </Text> + </Box> + </Box> + ); +} + +/** + * Hunk separator component + */ +export function HunkSeparator() { + return ( + <Box> + <Text color="gray">...</Text> + </Box> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/renderers/index.tsx b/dexto/packages/cli/src/cli/ink-cli/components/renderers/index.tsx new file mode 100644 index 00000000..a3aed645 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/renderers/index.tsx @@ -0,0 +1,72 @@ +/** + * Tool Result Renderers + * + * Dispatch component that renders tool results based on display type. + * Uses discriminated union from ToolDisplayData for type-safe rendering. + */ + +import React from 'react'; +import type { ContentPart, ToolDisplayData } from '@dexto/core'; +import { DiffRenderer } from './DiffRenderer.js'; +import { ShellRenderer } from './ShellRenderer.js'; +import { SearchRenderer } from './SearchRenderer.js'; +import { FileRenderer } from './FileRenderer.js'; +import { GenericRenderer } from './GenericRenderer.js'; + +// Re-export individual renderers for direct use +export { DiffRenderer } from './DiffRenderer.js'; +export { ShellRenderer } from './ShellRenderer.js'; +export { SearchRenderer } from './SearchRenderer.js'; +export { FileRenderer } from './FileRenderer.js'; +export { GenericRenderer } from './GenericRenderer.js'; + +// File preview renderers for approval prompts (full content, no truncation) +export { DiffPreview, CreateFilePreview } from './FilePreviewRenderer.js'; + +interface ToolResultRendererProps { + /** Display data from SanitizedToolResult.meta.display */ + display?: ToolDisplayData; + /** Content parts from SanitizedToolResult.content */ + content: ContentPart[]; + /** Maximum lines/matches to display */ + maxLines?: number; +} + +/** + * Renders tool results based on display type. + * Falls back to GenericRenderer for unknown types or missing display data. + * Each renderer uses its own default limits - only pass maxLines to override. + */ +export function ToolResultRenderer({ display, content, maxLines }: ToolResultRendererProps) { + // Default to generic if no display data + const displayData = display ?? { type: 'generic' as const }; + + switch (displayData.type) { + case 'diff': + return ( + <DiffRenderer data={displayData} {...(maxLines !== undefined && { maxLines })} /> + ); + + case 'shell': + return ( + <ShellRenderer data={displayData} {...(maxLines !== undefined && { maxLines })} /> + ); + + case 'search': + return ( + <SearchRenderer + data={displayData} + {...(maxLines !== undefined && { maxMatches: maxLines })} + /> + ); + + case 'file': + return <FileRenderer data={displayData} />; + + case 'generic': + default: + return ( + <GenericRenderer content={content} {...(maxLines !== undefined && { maxLines })} /> + ); + } +} diff --git a/dexto/packages/cli/src/cli/ink-cli/components/shared/MarkdownText.tsx b/dexto/packages/cli/src/cli/ink-cli/components/shared/MarkdownText.tsx new file mode 100644 index 00000000..a8061ae0 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/shared/MarkdownText.tsx @@ -0,0 +1,635 @@ +/** + * MarkdownText Component + * + * Renders markdown text with terminal-appropriate styling. + * Handles both inline markdown (bold, code, italic) and block elements (headers, code blocks, lists). + * + * Uses wrap-ansi for proper word wrapping to avoid mid-word splits. + * Streaming-safe: incomplete markdown tokens won't match regex patterns, + * so they render as plain text until complete. + */ + +import React, { memo, useMemo } from 'react'; +import { Text, Box, useStdout } from 'ink'; +import chalk from 'chalk'; +import wrapAnsi from 'wrap-ansi'; +import stringWidth from 'string-width'; +import { highlight, supportsLanguage } from 'cli-highlight'; + +// ============================================================================ +// Inline Markdown Parsing +// ============================================================================ + +interface InlineSegment { + type: 'text' | 'bold' | 'code' | 'italic' | 'strikethrough' | 'link' | 'url'; + content: string; + url?: string; // For links +} + +/** + * Parse inline markdown and return segments. + * Uses a single regex to find all inline patterns, processes in order. + */ +function parseInlineMarkdown(text: string): InlineSegment[] { + // Early return for plain text without markdown indicators + if (!/[*_~`\[<]|https?:\/\//.test(text)) { + return [{ type: 'text', content: text }]; + } + + const segments: InlineSegment[] = []; + let lastIndex = 0; + + // Combined regex for all inline patterns + // Order matters: longer/more specific patterns first + const inlineRegex = + /(\*\*[^*]+\*\*|__[^_]+__|`[^`]+`|\*[^*]+\*|_[^_]+_|~~[^~]+~~|\[[^\]]+\]\([^)]+\)|https?:\/\/[^\s<]+)/g; + + let match; + while ((match = inlineRegex.exec(text)) !== null) { + // Add text before the match + if (match.index > lastIndex) { + segments.push({ type: 'text', content: text.slice(lastIndex, match.index) }); + } + + const fullMatch = match[0]; + + // Determine the type and extract content + if (fullMatch.startsWith('**') && fullMatch.endsWith('**') && fullMatch.length > 4) { + // Bold: **text** + segments.push({ type: 'bold', content: fullMatch.slice(2, -2) }); + } else if (fullMatch.startsWith('__') && fullMatch.endsWith('__') && fullMatch.length > 4) { + // Bold: __text__ + segments.push({ type: 'bold', content: fullMatch.slice(2, -2) }); + } else if (fullMatch.startsWith('`') && fullMatch.endsWith('`') && fullMatch.length > 2) { + // Inline code: `code` + segments.push({ type: 'code', content: fullMatch.slice(1, -1) }); + } else if (fullMatch.startsWith('~~') && fullMatch.endsWith('~~') && fullMatch.length > 4) { + // Strikethrough: ~~text~~ + segments.push({ type: 'strikethrough', content: fullMatch.slice(2, -2) }); + } else if (fullMatch.startsWith('*') && fullMatch.endsWith('*') && fullMatch.length > 2) { + // Italic: *text* + segments.push({ type: 'italic', content: fullMatch.slice(1, -1) }); + } else if (fullMatch.startsWith('_') && fullMatch.endsWith('_') && fullMatch.length > 2) { + // Italic: _text_ + segments.push({ type: 'italic', content: fullMatch.slice(1, -1) }); + } else if ( + fullMatch.startsWith('[') && + fullMatch.includes('](') && + fullMatch.endsWith(')') + ) { + // Link: [text](url) + const linkMatch = fullMatch.match(/\[([^\]]+)\]\(([^)]+)\)/); + if (linkMatch && linkMatch[1] && linkMatch[2]) { + segments.push({ type: 'link', content: linkMatch[1], url: linkMatch[2] }); + } else { + segments.push({ type: 'text', content: fullMatch }); + } + } else if (/^https?:\/\//.test(fullMatch)) { + // Raw URL + segments.push({ type: 'url', content: fullMatch }); + } else { + // Fallback: render as plain text + segments.push({ type: 'text', content: fullMatch }); + } + + lastIndex = inlineRegex.lastIndex; + } + + // Add remaining text after last match + if (lastIndex < text.length) { + segments.push({ type: 'text', content: text.slice(lastIndex) }); + } + + return segments; +} + +// ============================================================================ +// ANSI String Conversion (for wrap-ansi compatibility) +// ============================================================================ + +/** + * Convert parsed markdown segments to an ANSI-escaped string. + * This allows wrap-ansi to properly handle styled text while word-wrapping. + */ +function segmentsToAnsi(segments: InlineSegment[], defaultColor: string): string { + const colorFn = getChalkColor(defaultColor); + + return segments + .map((segment) => { + switch (segment.type) { + case 'bold': + return colorFn.bold(segment.content); + case 'code': + return chalk.cyan(segment.content); + case 'italic': + return chalk.gray(segment.content); + case 'strikethrough': + return colorFn.strikethrough(segment.content); + case 'link': + return colorFn(segment.content) + chalk.blue(` (${segment.url})`); + case 'url': + return chalk.blue(segment.content); + default: + return colorFn(segment.content); + } + }) + .join(''); +} + +/** + * Get chalk color function from color name + */ +function getChalkColor(color: string): typeof chalk { + switch (color) { + case 'white': + return chalk.white; + case 'gray': + return chalk.gray; + case 'blue': + return chalk.blue; + case 'cyan': + return chalk.cyan; + case 'green': + return chalk.green; + case 'yellow': + return chalk.rgb(255, 165, 0); + case 'orange': + return chalk.rgb(255, 165, 0); + case 'red': + return chalk.red; + case 'magenta': + return chalk.green; + default: + return chalk.white; + } +} + +// ============================================================================ +// Wrapped Paragraph Component (uses wrap-ansi for proper word wrapping) +// ============================================================================ + +interface WrappedParagraphProps { + text: string; + defaultColor: string; + bulletPrefix?: string; + isFirstParagraph?: boolean; +} + +/** + * Renders a paragraph with proper word wrapping using wrap-ansi. + * Handles bullet prefix with continuation line indentation. + */ +const WrappedParagraphInternal: React.FC<WrappedParagraphProps> = ({ + text, + defaultColor, + bulletPrefix, + isFirstParagraph = false, +}) => { + const { stdout } = useStdout(); + const terminalWidth = stdout?.columns ?? 80; + + const wrappedLines = useMemo(() => { + // Parse markdown and convert to ANSI string + const segments = parseInlineMarkdown(text); + const ansiString = segmentsToAnsi(segments, defaultColor); + + // Calculate available width - always account for bullet indent since all lines get it + const prefixWidth = bulletPrefix ? stringWidth(bulletPrefix) : 0; + const availableWidth = Math.max(20, terminalWidth - prefixWidth); + + // Word-wrap the ANSI string + const wrapped = wrapAnsi(ansiString, availableWidth, { + hard: false, // Don't break in the middle of words + wordWrap: true, // Enable word wrapping + trim: false, // Don't trim whitespace + }); + + return wrapped.split('\n'); + }, [text, defaultColor, bulletPrefix, isFirstParagraph, terminalWidth]); + + // Calculate indent for continuation lines (spaces to align with first line content) + // All lines get indentation when bulletPrefix is provided, for consistent left margin + const indentSpaces = bulletPrefix ? ' '.repeat(stringWidth(bulletPrefix)) : ''; + + return ( + <> + {wrappedLines.map((line, i) => { + const isFirstLine = i === 0; + // First line of first paragraph gets bullet, all other lines get space indent + const prefix = + isFirstLine && isFirstParagraph && bulletPrefix ? bulletPrefix : indentSpaces; + + return ( + <Box key={i}> + <Text> + {prefix} + {line} + </Text> + </Box> + ); + })} + </> + ); +}; + +const WrappedParagraph = memo(WrappedParagraphInternal); + +// ============================================================================ +// Legacy RenderInline (for headers and other non-wrapped content) +// ============================================================================ + +interface RenderInlineProps { + text: string; + defaultColor?: string; + /** Apply bold styling to all text segments (used for headers) */ + bold?: boolean; +} + +/** + * Renders inline markdown segments with appropriate styling. + * Used for headers and other content that doesn't need word wrapping. + */ +const RenderInlineInternal: React.FC<RenderInlineProps> = ({ + text, + defaultColor = 'white', + bold: baseBold = false, +}) => { + const segments = parseInlineMarkdown(text); + + return ( + <> + {segments.map((segment, i) => { + switch (segment.type) { + case 'bold': + return ( + <Text key={i} bold color={defaultColor}> + {segment.content} + </Text> + ); + case 'code': + return ( + <Text key={i} bold={baseBold} color="cyan"> + {segment.content} + </Text> + ); + case 'italic': + return ( + <Text key={i} bold={baseBold} color="gray"> + {segment.content} + </Text> + ); + case 'strikethrough': + return ( + <Text key={i} bold={baseBold} strikethrough color={defaultColor}> + {segment.content} + </Text> + ); + case 'link': + return ( + <Text key={i} bold={baseBold} color={defaultColor}> + {segment.content} + <Text color="blue"> ({segment.url})</Text> + </Text> + ); + case 'url': + return ( + <Text key={i} bold={baseBold} color="blue"> + {segment.content} + </Text> + ); + default: + return ( + <Text key={i} bold={baseBold} color={defaultColor}> + {segment.content} + </Text> + ); + } + })} + </> + ); +}; + +const RenderInline = memo(RenderInlineInternal); + +// ============================================================================ +// Block-level Markdown Rendering +// ============================================================================ + +interface MarkdownTextProps { + children: string; + /** Default text color */ + color?: string; + /** Optional prefix for first line (e.g., "⏺ " for assistant messages) */ + bulletPrefix?: string; +} + +/** + * Main MarkdownText component. + * Handles block-level elements (headers, code blocks, lists) and delegates + * paragraph rendering to WrappedParagraph for proper word wrapping. + */ +const MarkdownTextInternal: React.FC<MarkdownTextProps> = ({ + children, + color = 'white', + bulletPrefix, +}) => { + if (!children) return null; + + const defaultColor = color; + const lines = children.split('\n'); + let isFirstContentLine = true; // Track first actual content for bullet prefix + + // Regex patterns for block elements + const headerRegex = /^(#{1,6})\s+(.*)$/; + const codeFenceRegex = /^(`{3,}|~{3,})(\w*)$/; + const ulItemRegex = /^(\s*)([-*+])\s+(.*)$/; + const olItemRegex = /^(\s*)(\d+)\.\s+(.*)$/; + // CommonMark allows spaces between HR characters (e.g., "- - -" or "* * *") + const hrRegex = /^(\s*[-*_]\s*){3,}$/; + + const blocks: React.ReactNode[] = []; + let inCodeBlock = false; + let codeBlockLines: string[] = []; + let codeBlockLang = ''; + let codeBlockFence = ''; + + lines.forEach((line, index) => { + const key = `line-${index}`; + const trimmedLine = line.trim(); + + // Handle code block state + if (inCodeBlock) { + const fenceMatch = trimmedLine.match(codeFenceRegex); + // Per CommonMark spec, closing fence must use same character and be at least as long + if ( + fenceMatch && + fenceMatch[1] && + codeBlockFence[0] && + fenceMatch[1][0] === codeBlockFence[0] && + fenceMatch[1].length >= codeBlockFence.length + ) { + // End of code block + blocks.push( + <RenderCodeBlock key={key} lines={codeBlockLines} language={codeBlockLang} /> + ); + inCodeBlock = false; + codeBlockLines = []; + codeBlockLang = ''; + codeBlockFence = ''; + } else { + codeBlockLines.push(line); + } + return; + } + + // Check for code fence start + const codeFenceMatch = trimmedLine.match(codeFenceRegex); + if (codeFenceMatch && codeFenceMatch[1]) { + inCodeBlock = true; + codeBlockFence = codeFenceMatch[1]; + codeBlockLang = codeFenceMatch[2] || ''; + return; + } + + // Headers + const headerMatch = line.match(headerRegex); + if (headerMatch && headerMatch[1] && headerMatch[2] !== undefined) { + const level = headerMatch[1].length; + const headerText = headerMatch[2]; + blocks.push(<RenderHeader key={key} level={level} text={headerText} />); + return; + } + + // Horizontal rule + if (hrRegex.test(trimmedLine)) { + blocks.push( + <Box key={key}> + <Text color="gray">{'─'.repeat(40)}</Text> + </Box> + ); + return; + } + + // Unordered list + const ulMatch = line.match(ulItemRegex); + if (ulMatch && ulMatch[1] !== undefined && ulMatch[3] !== undefined) { + const indent = ulMatch[1].length; + const itemText = ulMatch[3]; + blocks.push( + <RenderListItem + key={key} + indent={indent} + marker="-" + text={itemText} + defaultColor={defaultColor} + /> + ); + return; + } + + // Ordered list + const olMatch = line.match(olItemRegex); + if (olMatch && olMatch[1] !== undefined && olMatch[2] && olMatch[3] !== undefined) { + const indent = olMatch[1].length; + const number = olMatch[2]; + const itemText = olMatch[3]; + blocks.push( + <RenderListItem + key={key} + indent={indent} + marker={`${number}.`} + text={itemText} + defaultColor={defaultColor} + /> + ); + return; + } + + // Empty line - add spacing + if (trimmedLine.length === 0) { + blocks.push(<Box key={key} height={1} />); + return; + } + + // Regular paragraph line - use WrappedParagraph for proper word wrapping + const usePrefix = isFirstContentLine && bulletPrefix; + blocks.push( + <WrappedParagraph + key={key} + text={line} + defaultColor={defaultColor} + {...(bulletPrefix && { bulletPrefix })} + isFirstParagraph={usePrefix ? true : false} + /> + ); + if (usePrefix) { + isFirstContentLine = false; + } + }); + + // Handle unclosed code block (streaming case) + if (inCodeBlock && codeBlockLines.length > 0) { + blocks.push( + <RenderCodeBlock + key="code-pending" + lines={codeBlockLines} + language={codeBlockLang} + isPending + /> + ); + } + + // Wrap in column layout + return <Box flexDirection="column">{blocks}</Box>; +}; + +// ============================================================================ +// Helper Components +// ============================================================================ + +interface RenderHeaderProps { + level: number; + text: string; +} + +const RenderHeaderInternal: React.FC<RenderHeaderProps> = ({ level, text }) => { + // Color based on header level + const headerColors: Record<number, string> = { + 1: 'blue', + 2: 'cyan', + 3: 'white', + 4: 'gray', + 5: 'gray', + 6: 'gray', + }; + const headerColor = headerColors[level] || 'white'; + + return ( + <Box marginTop={level <= 2 ? 1 : 0}> + <RenderInline text={text} defaultColor={headerColor} bold /> + </Box> + ); +}; + +const RenderHeader = memo(RenderHeaderInternal); + +interface RenderListItemProps { + indent: number; + marker: string; + text: string; + defaultColor: string; +} + +/** + * List item with proper word wrapping using wrap-ansi. + * Continuation lines are indented to align with the first line content. + */ +const RenderListItemInternal: React.FC<RenderListItemProps> = ({ + indent, + marker, + text, + defaultColor, +}) => { + const { stdout } = useStdout(); + const terminalWidth = stdout?.columns ?? 80; + + const paddingLeft = Math.floor(indent / 2); + const markerWithSpace = `${marker} `; + const markerWidth = stringWidth(markerWithSpace); + + const wrappedLines = useMemo(() => { + // Parse markdown and convert to ANSI string + const segments = parseInlineMarkdown(text); + const ansiString = segmentsToAnsi(segments, defaultColor); + + // Available width = terminal - padding - marker + const availableWidth = Math.max(20, terminalWidth - paddingLeft - markerWidth); + + // Word-wrap the ANSI string + const wrapped = wrapAnsi(ansiString, availableWidth, { + hard: false, + wordWrap: true, + trim: false, + }); + + return wrapped.split('\n'); + }, [text, defaultColor, terminalWidth, paddingLeft, markerWidth]); + + const continuationIndent = ' '.repeat(markerWidth); + + return ( + <Box paddingLeft={paddingLeft} flexDirection="column"> + {wrappedLines.map((line, i) => ( + <Box key={i} flexDirection="row"> + <Text color={defaultColor}> + {i === 0 ? markerWithSpace : continuationIndent} + </Text> + <Text>{line}</Text> + </Box> + ))} + </Box> + ); +}; + +const RenderListItem = memo(RenderListItemInternal); + +interface RenderCodeBlockProps { + lines: string[]; + language: string; + isPending?: boolean; +} + +const RenderCodeBlockInternal: React.FC<RenderCodeBlockProps> = ({ + lines, + language, + isPending, +}) => { + // Memoize the highlighted code to avoid re-highlighting on every render + const highlightedCode = useMemo(() => { + const code = lines.join('\n'); + + // If we have a language and it's supported, use syntax highlighting + if (language && supportsLanguage(language)) { + try { + return highlight(code, { language, ignoreIllegals: true }); + } catch { + // Fall back to plain cyan if highlighting fails + return chalk.cyan(code); + } + } + + // If no language specified, try auto-detection + if (!language && code.trim()) { + try { + return highlight(code, { ignoreIllegals: true }); + } catch { + // Fall back to plain cyan if auto-detection fails + return chalk.cyan(code); + } + } + + // Fallback: plain cyan text + return chalk.cyan(code); + }, [lines, language]); + + return ( + <Box flexDirection="column" marginTop={1} marginBottom={1}> + {language && <Text color="gray">{language}</Text>} + <Box flexDirection="column" paddingLeft={1}> + <Text>{highlightedCode}</Text> + {isPending && <Text color="gray">...</Text>} + </Box> + </Box> + ); +}; + +const RenderCodeBlock = memo(RenderCodeBlockInternal); + +// ============================================================================ +// Exports +// ============================================================================ + +export const MarkdownText = memo(MarkdownTextInternal); + +// Also export inline renderer for use in other components +export { RenderInline, parseInlineMarkdown }; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/shared/VirtualizedList.tsx b/dexto/packages/cli/src/cli/ink-cli/components/shared/VirtualizedList.tsx new file mode 100644 index 00000000..425ecc08 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/shared/VirtualizedList.tsx @@ -0,0 +1,448 @@ +/** + * VirtualizedList Component + * Only renders items visible in the viewport for performance. + */ + +import { + useState, + useRef, + useLayoutEffect, + forwardRef, + useImperativeHandle, + useEffect, + useMemo, + useCallback, +} from 'react'; +import type React from 'react'; +import { type DOMElement, measureElement, Box } from 'ink'; +import { useBatchedScroll } from '../../hooks/useBatchedScroll.js'; + +export const SCROLL_TO_ITEM_END = Number.MAX_SAFE_INTEGER; + +type VirtualizedListProps<T> = { + data: T[]; + renderItem: (info: { item: T; index: number }) => React.ReactElement; + estimatedItemHeight: (index: number) => number; + keyExtractor: (item: T, index: number) => string; + initialScrollIndex?: number; + initialScrollOffsetInIndex?: number; + scrollbarThumbColor?: string; +}; + +export type VirtualizedListRef<T> = { + scrollBy: (delta: number) => void; + scrollTo: (offset: number) => void; + scrollToEnd: () => void; + scrollToIndex: (params: { index: number; viewOffset?: number; viewPosition?: number }) => void; + scrollToItem: (params: { item: T; viewOffset?: number; viewPosition?: number }) => void; + getScrollIndex: () => number; + getScrollState: () => { + scrollTop: number; + scrollHeight: number; + innerHeight: number; + }; +}; + +function findLastIndex<T>( + array: T[], + predicate: (value: T, index: number, obj: T[]) => unknown +): number { + for (let i = array.length - 1; i >= 0; i--) { + if (predicate(array[i]!, i, array)) { + return i; + } + } + return -1; +} + +function VirtualizedListInner<T>( + props: VirtualizedListProps<T>, + ref: React.Ref<VirtualizedListRef<T>> +) { + const { + data, + renderItem, + estimatedItemHeight, + keyExtractor, + initialScrollIndex, + initialScrollOffsetInIndex, + } = props; + const dataRef = useRef(data); + useEffect(() => { + dataRef.current = data; + }, [data]); + + const [scrollAnchor, setScrollAnchor] = useState(() => { + const scrollToEnd = + initialScrollIndex === SCROLL_TO_ITEM_END || + (typeof initialScrollIndex === 'number' && + initialScrollIndex >= data.length - 1 && + initialScrollOffsetInIndex === SCROLL_TO_ITEM_END); + + if (scrollToEnd) { + return { + index: data.length > 0 ? data.length - 1 : 0, + offset: SCROLL_TO_ITEM_END, + }; + } + + if (typeof initialScrollIndex === 'number') { + return { + index: Math.max(0, Math.min(data.length - 1, initialScrollIndex)), + offset: initialScrollOffsetInIndex ?? 0, + }; + } + + return { index: 0, offset: 0 }; + }); + + const [isStickingToBottom, setIsStickingToBottom] = useState(() => { + const scrollToEnd = + initialScrollIndex === SCROLL_TO_ITEM_END || + (typeof initialScrollIndex === 'number' && + initialScrollIndex >= data.length - 1 && + initialScrollOffsetInIndex === SCROLL_TO_ITEM_END); + return scrollToEnd; + }); + + const containerRef = useRef<DOMElement>(null); + const [containerHeight, setContainerHeight] = useState(0); + const itemRefs = useRef<Array<DOMElement | null>>([]); + const [heights, setHeights] = useState<number[]>([]); + const isInitialScrollSet = useRef(false); + + const { totalHeight, offsets } = useMemo(() => { + const offsets: number[] = [0]; + let totalHeight = 0; + for (let i = 0; i < data.length; i++) { + const height = heights[i] ?? estimatedItemHeight(i); + totalHeight += height; + offsets.push(totalHeight); + } + return { totalHeight, offsets }; + }, [heights, data, estimatedItemHeight]); + + useEffect(() => { + setHeights((prevHeights) => { + if (data.length === prevHeights.length) { + return prevHeights; + } + + const newHeights = [...prevHeights]; + if (data.length < prevHeights.length) { + newHeights.length = data.length; + } else { + for (let i = prevHeights.length; i < data.length; i++) { + newHeights[i] = estimatedItemHeight(i); + } + } + return newHeights; + }); + }, [data, estimatedItemHeight]); + + // Calculate visible range + const scrollableContainerHeight = containerRef.current + ? Math.round(measureElement(containerRef.current).height) + : containerHeight; + + const getAnchorForScrollTop = useCallback( + (scrollTop: number, offsets: number[]): { index: number; offset: number } => { + const index = findLastIndex(offsets, (offset) => offset <= scrollTop); + if (index === -1) { + return { index: 0, offset: 0 }; + } + return { index, offset: scrollTop - offsets[index]! }; + }, + [] + ); + + const scrollTop = useMemo(() => { + const offset = offsets[scrollAnchor.index]; + if (typeof offset !== 'number') { + return 0; + } + + let rawScrollTop: number; + if (scrollAnchor.offset === SCROLL_TO_ITEM_END) { + const itemHeight = heights[scrollAnchor.index] ?? 0; + rawScrollTop = offset + itemHeight - scrollableContainerHeight; + } else { + rawScrollTop = offset + scrollAnchor.offset; + } + + // Clamp to valid range - negative scrollTop causes content to vanish! + const maxScroll = Math.max(0, totalHeight - scrollableContainerHeight); + return Math.max(0, Math.min(maxScroll, rawScrollTop)); + }, [scrollAnchor, offsets, heights, scrollableContainerHeight, totalHeight]); + + const startIndex = Math.max(0, findLastIndex(offsets, (offset) => offset <= scrollTop) - 1); + const endIndexOffset = offsets.findIndex( + (offset) => offset > scrollTop + scrollableContainerHeight + ); + const endIndex = + endIndexOffset === -1 ? data.length - 1 : Math.min(data.length - 1, endIndexOffset); + + // Measure container and items + useLayoutEffect(() => { + if (containerRef.current) { + const height = Math.round(measureElement(containerRef.current).height); + if (containerHeight !== height) { + setContainerHeight(height); + } + } + + let newHeights: number[] | null = null; + for (let i = startIndex; i <= endIndex; i++) { + const itemRef = itemRefs.current[i]; + if (itemRef) { + const height = Math.round(measureElement(itemRef).height); + if (height !== heights[i]) { + if (!newHeights) { + newHeights = [...heights]; + } + newHeights[i] = height; + } + } + } + if (newHeights) { + setHeights(newHeights); + } + }); + + // Auto-scroll to bottom when new items added + const prevDataLength = useRef(data.length); + const prevTotalHeight = useRef(totalHeight); + const prevScrollTop = useRef(scrollTop); + const prevContainerHeight = useRef(scrollableContainerHeight); + + useLayoutEffect(() => { + const contentPreviouslyFit = prevTotalHeight.current <= prevContainerHeight.current; + const wasScrolledToBottomPixels = + prevScrollTop.current >= prevTotalHeight.current - prevContainerHeight.current - 1; + const wasAtBottom = contentPreviouslyFit || wasScrolledToBottomPixels; + + if (wasAtBottom && scrollTop >= prevScrollTop.current) { + setIsStickingToBottom(true); + } + + const listGrew = data.length > prevDataLength.current; + const containerChanged = prevContainerHeight.current !== scrollableContainerHeight; + + if ( + (listGrew && (isStickingToBottom || wasAtBottom)) || + (isStickingToBottom && containerChanged) + ) { + setScrollAnchor({ + index: data.length > 0 ? data.length - 1 : 0, + offset: SCROLL_TO_ITEM_END, + }); + if (!isStickingToBottom) { + setIsStickingToBottom(true); + } + } else if ( + (scrollAnchor.index >= data.length || + scrollTop > totalHeight - scrollableContainerHeight) && + data.length > 0 + ) { + const newScrollTop = Math.max(0, totalHeight - scrollableContainerHeight); + setScrollAnchor(getAnchorForScrollTop(newScrollTop, offsets)); + } else if (data.length === 0) { + setScrollAnchor({ index: 0, offset: 0 }); + } + + prevDataLength.current = data.length; + prevTotalHeight.current = totalHeight; + prevScrollTop.current = scrollTop; + prevContainerHeight.current = scrollableContainerHeight; + }, [ + data.length, + totalHeight, + scrollTop, + scrollableContainerHeight, + scrollAnchor.index, + getAnchorForScrollTop, + offsets, + isStickingToBottom, + ]); + + // Handle initial scroll position + useLayoutEffect(() => { + if ( + isInitialScrollSet.current || + offsets.length <= 1 || + totalHeight <= 0 || + containerHeight <= 0 + ) { + return; + } + + if (typeof initialScrollIndex === 'number') { + const scrollToEnd = + initialScrollIndex === SCROLL_TO_ITEM_END || + (initialScrollIndex >= data.length - 1 && + initialScrollOffsetInIndex === SCROLL_TO_ITEM_END); + + if (scrollToEnd) { + setScrollAnchor({ + index: data.length - 1, + offset: SCROLL_TO_ITEM_END, + }); + setIsStickingToBottom(true); + isInitialScrollSet.current = true; + return; + } + + const index = Math.max(0, Math.min(data.length - 1, initialScrollIndex)); + const offset = initialScrollOffsetInIndex ?? 0; + const newScrollTop = (offsets[index] ?? 0) + offset; + const clampedScrollTop = Math.max( + 0, + Math.min(totalHeight - scrollableContainerHeight, newScrollTop) + ); + setScrollAnchor(getAnchorForScrollTop(clampedScrollTop, offsets)); + isInitialScrollSet.current = true; + } + }, [ + initialScrollIndex, + initialScrollOffsetInIndex, + offsets, + totalHeight, + containerHeight, + getAnchorForScrollTop, + data.length, + scrollableContainerHeight, + ]); + + const topSpacerHeight = offsets[startIndex] ?? 0; + const bottomSpacerHeight = totalHeight - (offsets[endIndex + 1] ?? totalHeight); + + const renderedItems = []; + for (let i = startIndex; i <= endIndex; i++) { + const item = data[i]; + if (item) { + renderedItems.push( + <Box + key={keyExtractor(item, i)} + width="100%" + ref={(el) => { + itemRefs.current[i] = el; + }} + > + {renderItem({ item, index: i })} + </Box> + ); + } + } + + const { getScrollTop, setPendingScrollTop } = useBatchedScroll(scrollTop); + + useImperativeHandle( + ref, + () => ({ + scrollBy: (delta: number) => { + if (delta < 0) { + setIsStickingToBottom(false); + } + const currentScrollTop = getScrollTop(); + const newScrollTop = Math.max( + 0, + Math.min(totalHeight - scrollableContainerHeight, currentScrollTop + delta) + ); + setPendingScrollTop(newScrollTop); + setScrollAnchor(getAnchorForScrollTop(newScrollTop, offsets)); + }, + scrollTo: (offset: number) => { + setIsStickingToBottom(false); + const newScrollTop = Math.max( + 0, + Math.min(totalHeight - scrollableContainerHeight, offset) + ); + setPendingScrollTop(newScrollTop); + setScrollAnchor(getAnchorForScrollTop(newScrollTop, offsets)); + }, + scrollToEnd: () => { + setIsStickingToBottom(true); + if (data.length > 0) { + setScrollAnchor({ + index: data.length - 1, + offset: SCROLL_TO_ITEM_END, + }); + } + }, + scrollToIndex: ({ index, viewOffset = 0, viewPosition = 0 }) => { + setIsStickingToBottom(false); + const offset = offsets[index]; + if (offset !== undefined) { + const newScrollTop = Math.max( + 0, + Math.min( + totalHeight - scrollableContainerHeight, + offset - viewPosition * scrollableContainerHeight + viewOffset + ) + ); + setPendingScrollTop(newScrollTop); + setScrollAnchor(getAnchorForScrollTop(newScrollTop, offsets)); + } + }, + scrollToItem: ({ item, viewOffset = 0, viewPosition = 0 }) => { + setIsStickingToBottom(false); + const index = data.indexOf(item); + if (index !== -1) { + const offset = offsets[index]; + if (offset !== undefined) { + const newScrollTop = Math.max( + 0, + Math.min( + totalHeight - scrollableContainerHeight, + offset - viewPosition * scrollableContainerHeight + viewOffset + ) + ); + setPendingScrollTop(newScrollTop); + setScrollAnchor(getAnchorForScrollTop(newScrollTop, offsets)); + } + } + }, + getScrollIndex: () => scrollAnchor.index, + getScrollState: () => ({ + scrollTop: getScrollTop(), + scrollHeight: totalHeight, + innerHeight: containerHeight, + }), + }), + [ + offsets, + scrollAnchor, + totalHeight, + getAnchorForScrollTop, + data, + scrollableContainerHeight, + getScrollTop, + setPendingScrollTop, + containerHeight, + ] + ); + + return ( + <Box + ref={containerRef} + overflowY="scroll" + overflowX="hidden" + scrollTop={scrollTop} + scrollbarThumbColor={props.scrollbarThumbColor ?? 'gray'} + width="100%" + height="100%" + flexDirection="column" + paddingRight={1} + > + <Box flexShrink={0} width="100%" flexDirection="column"> + <Box height={topSpacerHeight} flexShrink={0} /> + {renderedItems} + <Box height={bottomSpacerHeight} flexShrink={0} /> + </Box> + </Box> + ); +} + +export const VirtualizedList = forwardRef(VirtualizedListInner) as <T>( + props: VirtualizedListProps<T> & { ref?: React.Ref<VirtualizedListRef<T>> } +) => React.ReactElement; diff --git a/dexto/packages/cli/src/cli/ink-cli/components/shared/text-buffer.ts b/dexto/packages/cli/src/cli/ink-cli/components/shared/text-buffer.ts new file mode 100644 index 00000000..880cf3a3 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/components/shared/text-buffer.ts @@ -0,0 +1,1772 @@ +/** + * Text Buffer - Core text editing buffer with visual line wrapping + */ + +import { useState, useCallback, useEffect, useMemo, useReducer } from 'react'; +import { + toCodePoints, + cpLen, + cpSlice, + stripUnsafeCharacters, + getCachedStringWidth, + isWordCharStrict, + isWhitespace, + isWordCharWithCombining, + isDifferentScript, +} from '../../utils/textUtils.js'; +import type { Key } from '../../hooks/useKeypress.js'; + +export type Direction = + | 'left' + | 'right' + | 'up' + | 'down' + | 'wordLeft' + | 'wordRight' + | 'home' + | 'end'; + +// Find next word start within a line, starting from col +export const findNextWordStartInLine = (line: string, col: number): number | null => { + const chars = toCodePoints(line); + let i = col; + + if (i >= chars.length) return null; + + const currentChar = chars[i]; + + // Skip current word/sequence based on character type + if (isWordCharStrict(currentChar)) { + while (i < chars.length && isWordCharWithCombining(chars[i])) { + // Check for script boundary - if next character is from different script, stop here + if ( + i + 1 < chars.length && + isWordCharStrict(chars[i + 1]) && + isDifferentScript(chars[i], chars[i + 1]) + ) { + i++; // Include current character + break; // Stop at script boundary + } + i++; + } + } else if (!isWhitespace(currentChar)) { + while (i < chars.length && !isWordCharStrict(chars[i]) && !isWhitespace(chars[i])) { + i++; + } + } + + // Skip whitespace + while (i < chars.length && isWhitespace(chars[i])) { + i++; + } + + return i < chars.length ? i : null; +}; + +// Find previous word start within a line +export const findPrevWordStartInLine = (line: string, col: number): number | null => { + const chars = toCodePoints(line); + let i = col; + + if (i <= 0) return null; + + i--; + + // Skip whitespace moving backwards + while (i >= 0 && isWhitespace(chars[i])) { + i--; + } + + if (i < 0) return null; + + if (isWordCharStrict(chars[i])) { + // We're in a word, move to its beginning + while (i >= 0 && isWordCharStrict(chars[i])) { + // Check for script boundary - if previous character is from different script, stop here + if ( + i - 1 >= 0 && + isWordCharStrict(chars[i - 1]) && + isDifferentScript(chars[i], chars[i - 1]) + ) { + return i; // Return current position at script boundary + } + i--; + } + return i + 1; + } else { + // We're in punctuation, move to its beginning + while (i >= 0 && !isWordCharStrict(chars[i]) && !isWhitespace(chars[i])) { + i--; + } + return i + 1; + } +}; + +// Find word end within a line +export const findWordEndInLine = (line: string, col: number): number | null => { + const chars = toCodePoints(line); + let i = col; + + // If we're already at the end of a word (including punctuation sequences), advance to next word + // This includes both regular word endings and script boundaries + const atEndOfWordChar = + i < chars.length && + isWordCharWithCombining(chars[i]) && + (i + 1 >= chars.length || + !isWordCharWithCombining(chars[i + 1]) || + (isWordCharStrict(chars[i]) && + i + 1 < chars.length && + isWordCharStrict(chars[i + 1]) && + isDifferentScript(chars[i], chars[i + 1]))); + + const atEndOfPunctuation = + i < chars.length && + !isWordCharWithCombining(chars[i]) && + !isWhitespace(chars[i]) && + (i + 1 >= chars.length || + isWhitespace(chars[i + 1]) || + isWordCharWithCombining(chars[i + 1])); + + if (atEndOfWordChar || atEndOfPunctuation) { + // We're at the end of a word or punctuation sequence, move forward to find next word + i++; + // Skip whitespace to find next word or punctuation + while (i < chars.length && isWhitespace(chars[i])) { + i++; + } + } + + // If we're not on a word character, find the next word or punctuation sequence + if (i < chars.length && !isWordCharWithCombining(chars[i])) { + // Skip whitespace to find next word or punctuation + while (i < chars.length && isWhitespace(chars[i])) { + i++; + } + } + + // Move to end of current word (including combining marks, but stop at script boundaries) + let foundWord = false; + let lastBaseCharPos = -1; + + if (i < chars.length && isWordCharWithCombining(chars[i])) { + // Handle word characters + while (i < chars.length && isWordCharWithCombining(chars[i])) { + foundWord = true; + + // Track the position of the last base character (not combining mark) + if (isWordCharStrict(chars[i])) { + lastBaseCharPos = i; + } + + // Check if next character is from a different script (word boundary) + if ( + i + 1 < chars.length && + isWordCharStrict(chars[i + 1]) && + isDifferentScript(chars[i], chars[i + 1]) + ) { + i++; // Include current character + if (isWordCharStrict(chars[i - 1])) { + lastBaseCharPos = i - 1; + } + break; // Stop at script boundary + } + + i++; + } + } else if (i < chars.length && !isWhitespace(chars[i])) { + // Handle punctuation sequences (like ████) + while (i < chars.length && !isWordCharStrict(chars[i]) && !isWhitespace(chars[i])) { + foundWord = true; + lastBaseCharPos = i; + i++; + } + } + + // Only return a position if we actually found a word + // Return the position of the last base character, not combining marks + if (foundWord && lastBaseCharPos >= col) { + return lastBaseCharPos; + } + + return null; +}; + +// Initialize segmenter for word boundary detection +const segmenter = new Intl.Segmenter(undefined, { granularity: 'word' }); + +function findPrevWordBoundary(line: string, cursorCol: number): number { + const codePoints = toCodePoints(line); + // Convert cursorCol (CP index) to string index + const prefix = codePoints.slice(0, cursorCol).join(''); + const cursorIdx = prefix.length; + + let targetIdx = 0; + + for (const seg of segmenter.segment(line)) { + // We want the last word start strictly before the cursor. + // If we've reached or passed the cursor, we stop. + if (seg.index >= cursorIdx) break; + + if (seg.isWordLike) { + targetIdx = seg.index; + } + } + + return toCodePoints(line.slice(0, targetIdx)).length; +} + +function findNextWordBoundary(line: string, cursorCol: number): number { + const codePoints = toCodePoints(line); + const prefix = codePoints.slice(0, cursorCol).join(''); + const cursorIdx = prefix.length; + + let targetIdx = line.length; + + for (const seg of segmenter.segment(line)) { + const segEnd = seg.index + seg.segment.length; + + if (segEnd > cursorIdx) { + if (seg.isWordLike) { + targetIdx = segEnd; + break; + } + } + } + + return toCodePoints(line.slice(0, targetIdx)).length; +} + +// Find next word across lines +export const findNextWordAcrossLines = ( + lines: string[], + cursorRow: number, + cursorCol: number, + searchForWordStart: boolean +): { row: number; col: number } | null => { + // First try current line + const currentLine = lines[cursorRow] || ''; + const colInCurrentLine = searchForWordStart + ? findNextWordStartInLine(currentLine, cursorCol) + : findWordEndInLine(currentLine, cursorCol); + + if (colInCurrentLine !== null) { + return { row: cursorRow, col: colInCurrentLine }; + } + + // Search subsequent lines + for (let row = cursorRow + 1; row < lines.length; row++) { + const line = lines[row] || ''; + const chars = toCodePoints(line); + + // For empty lines, if we haven't found any words yet, return the empty line + if (chars.length === 0) { + // Check if there are any words in remaining lines + let hasWordsInLaterLines = false; + for (let laterRow = row + 1; laterRow < lines.length; laterRow++) { + const laterLine = lines[laterRow] || ''; + const laterChars = toCodePoints(laterLine); + let firstNonWhitespace = 0; + while ( + firstNonWhitespace < laterChars.length && + isWhitespace(laterChars[firstNonWhitespace]) + ) { + firstNonWhitespace++; + } + if (firstNonWhitespace < laterChars.length) { + hasWordsInLaterLines = true; + break; + } + } + + // If no words in later lines, return the empty line + if (!hasWordsInLaterLines) { + return { row, col: 0 }; + } + continue; + } + + // Find first non-whitespace + let firstNonWhitespace = 0; + while (firstNonWhitespace < chars.length && isWhitespace(chars[firstNonWhitespace])) { + firstNonWhitespace++; + } + + if (firstNonWhitespace < chars.length) { + if (searchForWordStart) { + return { row, col: firstNonWhitespace }; + } else { + // For word end, find the end of the first word + const endCol = findWordEndInLine(line, firstNonWhitespace); + if (endCol !== null) { + return { row, col: endCol }; + } + } + } + } + + return null; +}; + +// Find previous word across lines +export const findPrevWordAcrossLines = ( + lines: string[], + cursorRow: number, + cursorCol: number +): { row: number; col: number } | null => { + // First try current line + const currentLine = lines[cursorRow] || ''; + const colInCurrentLine = findPrevWordStartInLine(currentLine, cursorCol); + + if (colInCurrentLine !== null) { + return { row: cursorRow, col: colInCurrentLine }; + } + + // Search previous lines + for (let row = cursorRow - 1; row >= 0; row--) { + const line = lines[row] || ''; + const chars = toCodePoints(line); + + if (chars.length === 0) continue; + + // Find last word start + let lastWordStart = chars.length; + while (lastWordStart > 0 && isWhitespace(chars[lastWordStart - 1])) { + lastWordStart--; + } + + if (lastWordStart > 0) { + // Find start of this word + const wordStart = findPrevWordStartInLine(line, lastWordStart); + if (wordStart !== null) { + return { row, col: wordStart }; + } + } + } + + return null; +}; + +// Helper functions for vim line operations +export const getPositionFromOffsets = (startOffset: number, endOffset: number, lines: string[]) => { + let offset = 0; + let startRow = 0; + let startCol = 0; + let endRow = 0; + let endCol = 0; + + // Find start position + for (let i = 0; i < lines.length; i++) { + const lineLength = lines[i]!.length + 1; // +1 for newline + if (offset + lineLength > startOffset) { + startRow = i; + startCol = startOffset - offset; + break; + } + offset += lineLength; + } + + // Find end position + offset = 0; + for (let i = 0; i < lines.length; i++) { + const lineLength = lines[i]!.length + (i < lines.length - 1 ? 1 : 0); // +1 for newline except last line + if (offset + lineLength >= endOffset) { + endRow = i; + endCol = endOffset - offset; + break; + } + offset += lineLength; + } + + return { startRow, startCol, endRow, endCol }; +}; + +export const getLineRangeOffsets = (startRow: number, lineCount: number, lines: string[]) => { + let startOffset = 0; + + // Calculate start offset + for (let i = 0; i < startRow; i++) { + startOffset += lines[i]!.length + 1; // +1 for newline + } + + // Calculate end offset + let endOffset = startOffset; + for (let i = 0; i < lineCount; i++) { + const lineIndex = startRow + i; + if (lineIndex < lines.length) { + endOffset += lines[lineIndex]!.length; + if (lineIndex < lines.length - 1) { + endOffset += 1; // +1 for newline + } + } + } + + return { startOffset, endOffset }; +}; + +export const replaceRangeInternal = ( + state: TextBufferState, + startRow: number, + startCol: number, + endRow: number, + endCol: number, + text: string +): TextBufferState => { + const currentLine = (row: number) => state.lines[row] || ''; + const currentLineLen = (row: number) => cpLen(currentLine(row)); + const clamp = (value: number, min: number, max: number) => Math.min(Math.max(value, min), max); + + if ( + startRow > endRow || + (startRow === endRow && startCol > endCol) || + startRow < 0 || + startCol < 0 || + endRow >= state.lines.length || + (endRow < state.lines.length && endCol > currentLineLen(endRow)) + ) { + return state; // Invalid range + } + + const newLines = [...state.lines]; + + const sCol = clamp(startCol, 0, currentLineLen(startRow)); + const eCol = clamp(endCol, 0, currentLineLen(endRow)); + + const prefix = cpSlice(currentLine(startRow), 0, sCol); + const suffix = cpSlice(currentLine(endRow), eCol); + + const normalisedReplacement = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + const replacementParts = normalisedReplacement.split('\n'); + + // The combined first line of the new text + const firstLine = prefix + replacementParts[0]; + + if (replacementParts.length === 1) { + // No newlines in replacement: combine prefix, replacement, and suffix on one line. + newLines.splice(startRow, endRow - startRow + 1, firstLine + suffix); + } else { + // Newlines in replacement: create new lines. + const lastLine = replacementParts[replacementParts.length - 1] + suffix; + const middleLines = replacementParts.slice(1, -1); + newLines.splice(startRow, endRow - startRow + 1, firstLine, ...middleLines, lastLine); + } + + const finalCursorRow = startRow + replacementParts.length - 1; + const finalCursorCol = + (replacementParts.length > 1 ? 0 : sCol) + + cpLen(replacementParts[replacementParts.length - 1]!); + + return { + ...state, + lines: newLines, + cursorRow: Math.min(Math.max(finalCursorRow, 0), newLines.length - 1), + cursorCol: Math.max(0, Math.min(finalCursorCol, cpLen(newLines[finalCursorRow] || ''))), + preferredCol: null, + }; +}; + +export interface Viewport { + height: number; + width: number; +} + +function clamp(v: number, min: number, max: number): number { + return v < min ? min : v > max ? max : v; +} + +/* ────────────────────────────────────────────────────────────────────────── */ + +interface UseTextBufferProps { + initialText?: string; + initialCursorOffset?: number; + viewport: Viewport; + onChange?: (text: string) => void; + inputFilter?: (text: string) => string; + singleLine?: boolean; +} + +interface UndoHistoryEntry { + lines: string[]; + cursorRow: number; + cursorCol: number; +} + +function calculateInitialCursorPosition(initialLines: string[], offset: number): [number, number] { + let remainingChars = offset; + let row = 0; + while (row < initialLines.length) { + const lineLength = cpLen(initialLines[row]!); + // Add 1 for the newline character (except for the last line) + const totalCharsInLineAndNewline = lineLength + (row < initialLines.length - 1 ? 1 : 0); + + if (remainingChars <= lineLength) { + // Cursor is on this line + return [row, remainingChars]; + } + remainingChars -= totalCharsInLineAndNewline; + row++; + } + // Offset is beyond the text, place cursor at the end of the last line + if (initialLines.length > 0) { + const lastRow = initialLines.length - 1; + return [lastRow, cpLen(initialLines[lastRow]!)]; + } + return [0, 0]; // Default for empty text +} + +export function offsetToLogicalPos(text: string, offset: number): [number, number] { + let row = 0; + let col = 0; + let currentOffset = 0; + + if (offset === 0) return [0, 0]; + + const lines = text.split('\n'); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + const lineLength = cpLen(line); + const lineLengthWithNewline = lineLength + (i < lines.length - 1 ? 1 : 0); + + if (offset <= currentOffset + lineLength) { + // Check against lineLength first + row = i; + col = offset - currentOffset; + return [row, col]; + } else if (offset <= currentOffset + lineLengthWithNewline) { + // Check if offset is the newline itself + row = i; + col = lineLength; // Position cursor at the end of the current line content + // If the offset IS the newline, and it's not the last line, advance to next line, col 0 + if (offset === currentOffset + lineLengthWithNewline && i < lines.length - 1) { + return [i + 1, 0]; + } + return [row, col]; // Otherwise, it's at the end of the current line content + } + currentOffset += lineLengthWithNewline; + } + + // If offset is beyond the text length, place cursor at the end of the last line + // or [0,0] if text is empty + if (lines.length > 0) { + row = lines.length - 1; + col = cpLen(lines[row]!); + } else { + row = 0; + col = 0; + } + return [row, col]; +} + +/** + * Converts logical row/col position to absolute text offset + * Inverse operation of offsetToLogicalPos + */ +export function logicalPosToOffset(lines: string[], row: number, col: number): number { + let offset = 0; + + // Clamp row to valid range + const actualRow = Math.min(row, lines.length - 1); + + // Add lengths of all lines before the target row + for (let i = 0; i < actualRow; i++) { + offset += cpLen(lines[i]!) + 1; // +1 for newline + } + + // Add column offset within the target row + if (actualRow >= 0 && actualRow < lines.length) { + offset += Math.min(col, cpLen(lines[actualRow]!)); + } + + return offset; +} + +export interface VisualLayout { + visualLines: string[]; + // For each logical line, an array of [visualLineIndex, startColInLogical] + logicalToVisualMap: Array<Array<[number, number]>>; + // For each visual line, its [logicalLineIndex, startColInLogical] + visualToLogicalMap: Array<[number, number]>; +} + +// Calculates the visual wrapping of lines and the mapping between logical and visual coordinates. +// This is an expensive operation and should be memoized. +function calculateLayout(logicalLines: string[], viewportWidth: number): VisualLayout { + const visualLines: string[] = []; + const logicalToVisualMap: Array<Array<[number, number]>> = []; + const visualToLogicalMap: Array<[number, number]> = []; + + logicalLines.forEach((logLine, logIndex) => { + logicalToVisualMap[logIndex] = []; + if (logLine.length === 0) { + // Handle empty logical line + logicalToVisualMap[logIndex].push([visualLines.length, 0]); + visualToLogicalMap.push([logIndex, 0]); + visualLines.push(''); + } else { + // Non-empty logical line + let currentPosInLogLine = 0; // Tracks position within the current logical line (code point index) + const codePointsInLogLine = toCodePoints(logLine); + + while (currentPosInLogLine < codePointsInLogLine.length) { + let currentChunk = ''; + let currentChunkVisualWidth = 0; + let numCodePointsInChunk = 0; + let lastWordBreakPoint = -1; // Index in codePointsInLogLine for word break + let numCodePointsAtLastWordBreak = 0; + + // Iterate through code points to build the current visual line (chunk) + for (let i = currentPosInLogLine; i < codePointsInLogLine.length; i++) { + const char = codePointsInLogLine[i]!; + const charVisualWidth = getCachedStringWidth(char); + + if (currentChunkVisualWidth + charVisualWidth > viewportWidth) { + // Character would exceed viewport width + if ( + lastWordBreakPoint !== -1 && + numCodePointsAtLastWordBreak > 0 && + currentPosInLogLine + numCodePointsAtLastWordBreak < i + ) { + // We have a valid word break point to use, and it's not the start of the current segment + currentChunk = codePointsInLogLine + .slice( + currentPosInLogLine, + currentPosInLogLine + numCodePointsAtLastWordBreak + ) + .join(''); + numCodePointsInChunk = numCodePointsAtLastWordBreak; + } else { + // No word break, or word break is at the start of this potential chunk, or word break leads to empty chunk. + // Hard break: take characters up to viewportWidth, or just the current char if it alone is too wide. + if (numCodePointsInChunk === 0 && charVisualWidth > viewportWidth) { + // Single character is wider than viewport, take it anyway + currentChunk = char; + numCodePointsInChunk = 1; + } else if ( + numCodePointsInChunk === 0 && + charVisualWidth <= viewportWidth + ) { + // This case should ideally be caught by the next iteration if the char fits. + // If it doesn't fit (because currentChunkVisualWidth was already > 0 from a previous char that filled the line), + // then numCodePointsInChunk would not be 0. + // This branch means the current char *itself* doesn't fit an empty line, which is handled by the above. + // If we are here, it means the loop should break and the current chunk (which is empty) is finalized. + } + } + break; // Break from inner loop to finalize this chunk + } + + currentChunk += char; + currentChunkVisualWidth += charVisualWidth; + numCodePointsInChunk++; + + // Check for word break opportunity (space) + if (char === ' ') { + lastWordBreakPoint = i; // Store code point index of the space + // Store the state *before* adding the space, if we decide to break here. + numCodePointsAtLastWordBreak = numCodePointsInChunk - 1; // Chars *before* the space + } + } + + // If the inner loop completed without breaking (i.e., remaining text fits) + // or if the loop broke but numCodePointsInChunk is still 0 (e.g. first char too wide for empty line) + if ( + numCodePointsInChunk === 0 && + currentPosInLogLine < codePointsInLogLine.length + ) { + // This can happen if the very first character considered for a new visual line is wider than the viewport. + // In this case, we take that single character. + const firstChar = codePointsInLogLine[currentPosInLogLine]!; + currentChunk = firstChar; + numCodePointsInChunk = 1; // Ensure we advance + } + + // If after everything, numCodePointsInChunk is still 0 but we haven't processed the whole logical line, + // it implies an issue, like viewportWidth being 0 or less. Avoid infinite loop. + if ( + numCodePointsInChunk === 0 && + currentPosInLogLine < codePointsInLogLine.length + ) { + // Force advance by one character to prevent infinite loop if something went wrong + currentChunk = codePointsInLogLine[currentPosInLogLine]!; + numCodePointsInChunk = 1; + } + + logicalToVisualMap[logIndex].push([visualLines.length, currentPosInLogLine]); + visualToLogicalMap.push([logIndex, currentPosInLogLine]); + visualLines.push(currentChunk); + + const logicalStartOfThisChunk = currentPosInLogLine; + currentPosInLogLine += numCodePointsInChunk; + + // If the chunk processed did not consume the entire logical line, + // and the character immediately following the chunk is a space, + // advance past this space as it acted as a delimiter for word wrapping. + if ( + logicalStartOfThisChunk + numCodePointsInChunk < codePointsInLogLine.length && + currentPosInLogLine < codePointsInLogLine.length && // Redundant if previous is true, but safe + codePointsInLogLine[currentPosInLogLine] === ' ' + ) { + currentPosInLogLine++; + } + } + } + }); + + // If the entire logical text was empty, ensure there's one empty visual line. + if (logicalLines.length === 0 || (logicalLines.length === 1 && logicalLines[0] === '')) { + if (visualLines.length === 0) { + visualLines.push(''); + if (!logicalToVisualMap[0]) logicalToVisualMap[0] = []; + logicalToVisualMap[0].push([0, 0]); + visualToLogicalMap.push([0, 0]); + } + } + + return { + visualLines, + logicalToVisualMap, + visualToLogicalMap, + }; +} + +// Calculates the visual cursor position based on a pre-calculated layout. +// This is a lightweight operation. +function calculateVisualCursorFromLayout( + layout: VisualLayout, + logicalCursor: [number, number] +): [number, number] { + const { logicalToVisualMap, visualLines } = layout; + const [logicalRow, logicalCol] = logicalCursor; + + const segmentsForLogicalLine = logicalToVisualMap[logicalRow]; + + if (!segmentsForLogicalLine || segmentsForLogicalLine.length === 0) { + // This can happen for an empty document. + return [0, 0]; + } + + // Find the segment where the logical column fits. + // The segments are sorted by startColInLogical. + let targetSegmentIndex = segmentsForLogicalLine.findIndex(([, startColInLogical], index) => { + const nextStartColInLogical = + index + 1 < segmentsForLogicalLine.length + ? segmentsForLogicalLine[index + 1]![1] + : Infinity; + return logicalCol >= startColInLogical && logicalCol < nextStartColInLogical; + }); + + // If not found, it means the cursor is at the end of the logical line. + if (targetSegmentIndex === -1) { + if (logicalCol === 0) { + targetSegmentIndex = 0; + } else { + targetSegmentIndex = segmentsForLogicalLine.length - 1; + } + } + + const [visualRow, startColInLogical] = segmentsForLogicalLine[targetSegmentIndex]!; + const visualCol = logicalCol - startColInLogical; + + // The visual column should not exceed the length of the visual line. + const clampedVisualCol = Math.min(visualCol, cpLen(visualLines[visualRow] ?? '')); + + return [visualRow, clampedVisualCol]; +} + +// --- Start of reducer logic --- + +export interface TextBufferState { + lines: string[]; + cursorRow: number; + cursorCol: number; + preferredCol: number | null; // This is the logical character offset in the visual line + undoStack: UndoHistoryEntry[]; + redoStack: UndoHistoryEntry[]; + viewportWidth: number; + viewportHeight: number; + visualLayout: VisualLayout; +} + +const historyLimit = 100; + +export const pushUndo = (currentState: TextBufferState): TextBufferState => { + const snapshot = { + lines: [...currentState.lines], + cursorRow: currentState.cursorRow, + cursorCol: currentState.cursorCol, + }; + const newStack = [...currentState.undoStack, snapshot]; + if (newStack.length > historyLimit) { + newStack.shift(); + } + return { ...currentState, undoStack: newStack, redoStack: [] }; +}; + +export type TextBufferAction = + | { type: 'set_text'; payload: string; pushToUndo?: boolean } + | { type: 'insert'; payload: string } + | { type: 'backspace' } + | { + type: 'move'; + payload: { + dir: Direction; + }; + } + | { + type: 'set_cursor'; + payload: { + cursorRow: number; + cursorCol: number; + preferredCol: number | null; + }; + } + | { type: 'delete' } + | { type: 'delete_word_left' } + | { type: 'delete_word_right' } + | { type: 'kill_line_right' } + | { type: 'kill_line_left' } + | { type: 'undo' } + | { type: 'redo' } + | { + type: 'replace_range'; + payload: { + startRow: number; + startCol: number; + endRow: number; + endCol: number; + text: string; + }; + } + | { type: 'move_to_offset'; payload: { offset: number } } + | { type: 'create_undo_snapshot' } + | { type: 'set_viewport'; payload: { width: number; height: number } }; + +export interface TextBufferOptions { + inputFilter?: ((text: string) => string) | undefined; + singleLine?: boolean | undefined; +} + +function textBufferReducerLogic( + state: TextBufferState, + action: TextBufferAction, + options: TextBufferOptions = {} +): TextBufferState { + const pushUndoLocal = pushUndo; + + const currentLine = (r: number): string => state.lines[r] ?? ''; + const currentLineLen = (r: number): number => cpLen(currentLine(r)); + + switch (action.type) { + case 'set_text': { + let nextState = state; + if (action.pushToUndo !== false) { + nextState = pushUndoLocal(state); + } + const newContentLines = action.payload.replace(/\r\n?/g, '\n').split('\n'); + const lines = newContentLines.length === 0 ? [''] : newContentLines; + const lastNewLineIndex = lines.length - 1; + return { + ...nextState, + lines, + cursorRow: lastNewLineIndex, + cursorCol: cpLen(lines[lastNewLineIndex] ?? ''), + preferredCol: null, + }; + } + + case 'insert': { + // Validate payload before pushing undo to avoid orphaned undo entries + let payload = action.payload; + if (options.singleLine) { + payload = payload.replace(/[\r\n]/g, ''); + } + if (options.inputFilter) { + payload = options.inputFilter(payload); + } + + if (payload.length === 0) { + return state; + } + + // Now push undo since we know we'll make a change + const nextState = pushUndoLocal(state); + const newLines = [...nextState.lines]; + let newCursorRow = nextState.cursorRow; + let newCursorCol = nextState.cursorCol; + + const currentLine = (r: number) => newLines[r] ?? ''; + + const str = stripUnsafeCharacters(payload.replace(/\r\n/g, '\n').replace(/\r/g, '\n')); + const parts = str.split('\n'); + const lineContent = currentLine(newCursorRow); + const before = cpSlice(lineContent, 0, newCursorCol); + const after = cpSlice(lineContent, newCursorCol); + + if (parts.length > 1) { + newLines[newCursorRow] = before + parts[0]; + const remainingParts = parts.slice(1); + const lastPartOriginal = remainingParts.pop() ?? ''; + newLines.splice(newCursorRow + 1, 0, ...remainingParts); + newLines.splice(newCursorRow + parts.length - 1, 0, lastPartOriginal + after); + newCursorRow = newCursorRow + parts.length - 1; + newCursorCol = cpLen(lastPartOriginal); + } else { + newLines[newCursorRow] = before + parts[0]! + after; + newCursorCol = cpLen(before) + cpLen(parts[0]!); + } + + return { + ...nextState, + lines: newLines, + cursorRow: newCursorRow, + cursorCol: newCursorCol, + preferredCol: null, + }; + } + + case 'backspace': { + // Early return before pushing undo to avoid orphaned undo entries + if (state.cursorCol === 0 && state.cursorRow === 0) return state; + + const nextState = pushUndoLocal(state); + const newLines = [...nextState.lines]; + let newCursorRow = nextState.cursorRow; + let newCursorCol = nextState.cursorCol; + + const currentLine = (r: number) => newLines[r] ?? ''; + + if (newCursorCol > 0) { + const lineContent = currentLine(newCursorRow); + newLines[newCursorRow] = + cpSlice(lineContent, 0, newCursorCol - 1) + cpSlice(lineContent, newCursorCol); + newCursorCol--; + } else if (newCursorRow > 0) { + const prevLineContent = currentLine(newCursorRow - 1); + const currentLineContentVal = currentLine(newCursorRow); + const newCol = cpLen(prevLineContent); + newLines[newCursorRow - 1] = prevLineContent + currentLineContentVal; + newLines.splice(newCursorRow, 1); + newCursorRow--; + newCursorCol = newCol; + } + + return { + ...nextState, + lines: newLines, + cursorRow: newCursorRow, + cursorCol: newCursorCol, + preferredCol: null, + }; + } + + case 'set_viewport': { + const { width, height } = action.payload; + if (width === state.viewportWidth && height === state.viewportHeight) { + return state; + } + return { + ...state, + viewportWidth: width, + viewportHeight: height, + }; + } + + case 'move': { + const { dir } = action.payload; + const { cursorRow, cursorCol, lines, visualLayout, preferredCol } = state; + + // Visual movements + if ( + dir === 'left' || + dir === 'right' || + dir === 'up' || + dir === 'down' || + dir === 'home' || + dir === 'end' + ) { + const visualCursor = calculateVisualCursorFromLayout(visualLayout, [ + cursorRow, + cursorCol, + ]); + const { visualLines, visualToLogicalMap } = visualLayout; + + let newVisualRow = visualCursor[0]; + let newVisualCol = visualCursor[1]; + let newPreferredCol = preferredCol; + + const currentVisLineLen = cpLen(visualLines[newVisualRow] ?? ''); + + switch (dir) { + case 'left': + newPreferredCol = null; + if (newVisualCol > 0) { + newVisualCol--; + } else if (newVisualRow > 0) { + newVisualRow--; + newVisualCol = cpLen(visualLines[newVisualRow] ?? ''); + } + break; + case 'right': + newPreferredCol = null; + if (newVisualCol < currentVisLineLen) { + newVisualCol++; + } else if (newVisualRow < visualLines.length - 1) { + newVisualRow++; + newVisualCol = 0; + } + break; + case 'up': + if (newVisualRow > 0) { + if (newPreferredCol === null) newPreferredCol = newVisualCol; + newVisualRow--; + newVisualCol = clamp( + newPreferredCol, + 0, + cpLen(visualLines[newVisualRow] ?? '') + ); + } + break; + case 'down': + if (newVisualRow < visualLines.length - 1) { + if (newPreferredCol === null) newPreferredCol = newVisualCol; + newVisualRow++; + newVisualCol = clamp( + newPreferredCol, + 0, + cpLen(visualLines[newVisualRow] ?? '') + ); + } + break; + case 'home': + newPreferredCol = null; + newVisualCol = 0; + break; + case 'end': + newPreferredCol = null; + newVisualCol = currentVisLineLen; + break; + default: + return state; + } + + if (visualToLogicalMap[newVisualRow]) { + const [logRow, logStartCol] = visualToLogicalMap[newVisualRow]!; + return { + ...state, + cursorRow: logRow, + cursorCol: clamp(logStartCol + newVisualCol, 0, cpLen(lines[logRow] ?? '')), + preferredCol: newPreferredCol, + }; + } + return state; + } + + // Logical movements + switch (dir) { + case 'wordLeft': { + if (cursorCol === 0 && cursorRow === 0) return state; + + let newCursorRow = cursorRow; + let newCursorCol = cursorCol; + + if (cursorCol === 0) { + newCursorRow--; + newCursorCol = cpLen(lines[newCursorRow] ?? ''); + } else { + const lineContent = lines[cursorRow] ?? ''; + newCursorCol = findPrevWordBoundary(lineContent, cursorCol); + } + return { + ...state, + cursorRow: newCursorRow, + cursorCol: newCursorCol, + preferredCol: null, + }; + } + case 'wordRight': { + const lineContent = lines[cursorRow] ?? ''; + if (cursorRow === lines.length - 1 && cursorCol === cpLen(lineContent)) { + return state; + } + + let newCursorRow = cursorRow; + let newCursorCol = cursorCol; + const lineLen = cpLen(lineContent); + + if (cursorCol >= lineLen) { + newCursorRow++; + newCursorCol = 0; + } else { + newCursorCol = findNextWordBoundary(lineContent, cursorCol); + } + return { + ...state, + cursorRow: newCursorRow, + cursorCol: newCursorCol, + preferredCol: null, + }; + } + default: + return state; + } + } + + case 'set_cursor': { + return { + ...state, + ...action.payload, + }; + } + + case 'delete': { + const { cursorRow, cursorCol, lines } = state; + const lineContent = currentLine(cursorRow); + if (cursorCol < currentLineLen(cursorRow)) { + const nextState = pushUndoLocal(state); + const newLines = [...nextState.lines]; + newLines[cursorRow] = + cpSlice(lineContent, 0, cursorCol) + cpSlice(lineContent, cursorCol + 1); + return { + ...nextState, + lines: newLines, + preferredCol: null, + }; + } else if (cursorRow < lines.length - 1) { + const nextState = pushUndoLocal(state); + const nextLineContent = currentLine(cursorRow + 1); + const newLines = [...nextState.lines]; + newLines[cursorRow] = lineContent + nextLineContent; + newLines.splice(cursorRow + 1, 1); + return { + ...nextState, + lines: newLines, + preferredCol: null, + }; + } + return state; + } + + case 'delete_word_left': { + const { cursorRow, cursorCol } = state; + if (cursorCol === 0 && cursorRow === 0) return state; + + const nextState = pushUndoLocal(state); + const newLines = [...nextState.lines]; + let newCursorRow = cursorRow; + let newCursorCol = cursorCol; + + if (newCursorCol > 0) { + const lineContent = currentLine(newCursorRow); + const prevWordStart = findPrevWordStartInLine(lineContent, newCursorCol); + const start = prevWordStart === null ? 0 : prevWordStart; + newLines[newCursorRow] = + cpSlice(lineContent, 0, start) + cpSlice(lineContent, newCursorCol); + newCursorCol = start; + } else { + // Act as a backspace + const prevLineContent = currentLine(cursorRow - 1); + const currentLineContentVal = currentLine(cursorRow); + const newCol = cpLen(prevLineContent); + newLines[cursorRow - 1] = prevLineContent + currentLineContentVal; + newLines.splice(cursorRow, 1); + newCursorRow--; + newCursorCol = newCol; + } + + return { + ...nextState, + lines: newLines, + cursorRow: newCursorRow, + cursorCol: newCursorCol, + preferredCol: null, + }; + } + + case 'delete_word_right': { + const { cursorRow, cursorCol, lines } = state; + const lineContent = currentLine(cursorRow); + const lineLen = cpLen(lineContent); + + if (cursorCol >= lineLen && cursorRow === lines.length - 1) { + return state; + } + + const nextState = pushUndoLocal(state); + const newLines = [...nextState.lines]; + + if (cursorCol >= lineLen) { + // Act as a delete, joining with the next line + const nextLineContent = currentLine(cursorRow + 1); + newLines[cursorRow] = lineContent + nextLineContent; + newLines.splice(cursorRow + 1, 1); + } else { + const nextWordStart = findNextWordStartInLine(lineContent, cursorCol); + const end = nextWordStart === null ? lineLen : nextWordStart; + newLines[cursorRow] = + cpSlice(lineContent, 0, cursorCol) + cpSlice(lineContent, end); + } + + return { + ...nextState, + lines: newLines, + preferredCol: null, + }; + } + + case 'kill_line_right': { + const { cursorRow, cursorCol, lines } = state; + const lineContent = currentLine(cursorRow); + if (cursorCol < currentLineLen(cursorRow)) { + const nextState = pushUndoLocal(state); + const newLines = [...nextState.lines]; + newLines[cursorRow] = cpSlice(lineContent, 0, cursorCol); + return { + ...nextState, + lines: newLines, + }; + } else if (cursorRow < lines.length - 1) { + // Act as a delete + const nextState = pushUndoLocal(state); + const nextLineContent = currentLine(cursorRow + 1); + const newLines = [...nextState.lines]; + newLines[cursorRow] = lineContent + nextLineContent; + newLines.splice(cursorRow + 1, 1); + return { + ...nextState, + lines: newLines, + preferredCol: null, + }; + } + return state; + } + + case 'kill_line_left': { + const { cursorRow, cursorCol } = state; + if (cursorCol > 0) { + const nextState = pushUndoLocal(state); + const lineContent = currentLine(cursorRow); + const newLines = [...nextState.lines]; + newLines[cursorRow] = cpSlice(lineContent, cursorCol); + return { + ...nextState, + lines: newLines, + cursorCol: 0, + preferredCol: null, + }; + } + return state; + } + + case 'undo': { + const stateToRestore = state.undoStack[state.undoStack.length - 1]; + if (!stateToRestore) return state; + + const currentSnapshot = { + lines: [...state.lines], + cursorRow: state.cursorRow, + cursorCol: state.cursorCol, + }; + return { + ...state, + ...stateToRestore, + undoStack: state.undoStack.slice(0, -1), + redoStack: [...state.redoStack, currentSnapshot], + }; + } + + case 'redo': { + const stateToRestore = state.redoStack[state.redoStack.length - 1]; + if (!stateToRestore) return state; + + const currentSnapshot = { + lines: [...state.lines], + cursorRow: state.cursorRow, + cursorCol: state.cursorCol, + }; + return { + ...state, + ...stateToRestore, + redoStack: state.redoStack.slice(0, -1), + undoStack: [...state.undoStack, currentSnapshot], + }; + } + + case 'replace_range': { + const { startRow, startCol, endRow, endCol, text } = action.payload; + const nextState = pushUndoLocal(state); + return replaceRangeInternal(nextState, startRow, startCol, endRow, endCol, text); + } + + case 'move_to_offset': { + const { offset } = action.payload; + const [newRow, newCol] = offsetToLogicalPos(state.lines.join('\n'), offset); + return { + ...state, + cursorRow: newRow, + cursorCol: newCol, + preferredCol: null, + }; + } + + case 'create_undo_snapshot': { + return pushUndoLocal(state); + } + + default: + return state; + } +} + +export function textBufferReducer( + state: TextBufferState, + action: TextBufferAction, + options: TextBufferOptions = {} +): TextBufferState { + const newState = textBufferReducerLogic(state, action, options); + + if (newState.lines !== state.lines || newState.viewportWidth !== state.viewportWidth) { + return { + ...newState, + visualLayout: calculateLayout(newState.lines, newState.viewportWidth), + }; + } + + return newState; +} + +// --- End of reducer logic --- + +export function useTextBuffer({ + initialText = '', + initialCursorOffset = 0, + viewport, + onChange, + inputFilter, + singleLine = false, +}: UseTextBufferProps): TextBuffer { + const initialState = useMemo((): TextBufferState => { + const lines = initialText.split('\n'); + const [initialCursorRow, initialCursorCol] = calculateInitialCursorPosition( + lines.length === 0 ? [''] : lines, + initialCursorOffset + ); + const visualLayout = calculateLayout(lines.length === 0 ? [''] : lines, viewport.width); + return { + lines: lines.length === 0 ? [''] : lines, + cursorRow: initialCursorRow, + cursorCol: initialCursorCol, + preferredCol: null, + undoStack: [], + redoStack: [], + viewportWidth: viewport.width, + viewportHeight: viewport.height, + visualLayout, + }; + }, [initialText, initialCursorOffset, viewport.width, viewport.height]); + + const [state, dispatch] = useReducer( + (s: TextBufferState, a: TextBufferAction) => + textBufferReducer(s, a, { inputFilter, singleLine }), + initialState + ); + const { lines, cursorRow, cursorCol, preferredCol, visualLayout } = state; + + const text = useMemo(() => lines.join('\n'), [lines]); + + const visualCursor = useMemo( + () => calculateVisualCursorFromLayout(visualLayout, [cursorRow, cursorCol]), + [visualLayout, cursorRow, cursorCol] + ); + + const { visualLines, visualToLogicalMap } = visualLayout; + + const [visualScrollRow, setVisualScrollRow] = useState<number>(0); + + useEffect(() => { + if (onChange) { + onChange(text); + } + }, [text, onChange]); + + useEffect(() => { + dispatch({ + type: 'set_viewport', + payload: { width: viewport.width, height: viewport.height }, + }); + }, [viewport.width, viewport.height]); + + // Update visual scroll (vertical) + useEffect(() => { + const { height } = viewport; + const totalVisualLines = visualLines.length; + const maxScrollStart = Math.max(0, totalVisualLines - height); + let newVisualScrollRow = visualScrollRow; + + if (visualCursor[0] < visualScrollRow) { + newVisualScrollRow = visualCursor[0]; + } else if (visualCursor[0] >= visualScrollRow + height) { + newVisualScrollRow = visualCursor[0] - height + 1; + } + + // When the number of visual lines shrinks (e.g., after widening the viewport), + // ensure scroll never starts beyond the last valid start so we can render a full window. + newVisualScrollRow = clamp(newVisualScrollRow, 0, maxScrollStart); + + if (newVisualScrollRow !== visualScrollRow) { + setVisualScrollRow(newVisualScrollRow); + } + }, [visualCursor, visualScrollRow, viewport, visualLines.length]); + + const insert = useCallback( + (ch: string, { paste: _paste = false }: { paste?: boolean } = {}): void => { + if (!singleLine && /[\n\r]/.test(ch)) { + dispatch({ type: 'insert', payload: ch }); + return; + } + + let currentText = ''; + for (const char of toCodePoints(ch)) { + if (char.codePointAt(0) === 127) { + if (currentText.length > 0) { + dispatch({ type: 'insert', payload: currentText }); + currentText = ''; + } + dispatch({ type: 'backspace' }); + } else { + currentText += char; + } + } + if (currentText.length > 0) { + dispatch({ type: 'insert', payload: currentText }); + } + }, + [singleLine] + ); + + const newline = useCallback((): void => { + if (singleLine) { + return; + } + dispatch({ type: 'insert', payload: '\n' }); + }, [singleLine]); + + const backspace = useCallback((): void => { + dispatch({ type: 'backspace' }); + }, []); + + const del = useCallback((): void => { + dispatch({ type: 'delete' }); + }, []); + + const move = useCallback( + (dir: Direction): void => { + dispatch({ type: 'move', payload: { dir } }); + }, + [dispatch] + ); + + const undo = useCallback((): void => { + dispatch({ type: 'undo' }); + }, []); + + const redo = useCallback((): void => { + dispatch({ type: 'redo' }); + }, []); + + const setText = useCallback((newText: string): void => { + dispatch({ type: 'set_text', payload: newText }); + }, []); + + const deleteWordLeft = useCallback((): void => { + dispatch({ type: 'delete_word_left' }); + }, []); + + const deleteWordRight = useCallback((): void => { + dispatch({ type: 'delete_word_right' }); + }, []); + + const killLineRight = useCallback((): void => { + dispatch({ type: 'kill_line_right' }); + }, []); + + const killLineLeft = useCallback((): void => { + dispatch({ type: 'kill_line_left' }); + }, []); + + const setViewport = useCallback((width: number, height: number): void => { + dispatch({ type: 'set_viewport', payload: { width, height } }); + }, []); + + const handleInput = useCallback( + (key: Key): void => { + const { sequence: input } = key; + + if (key.paste) { + // Do not do any other processing on pastes so ensure we handle them + // before all other cases. + insert(input, { paste: key.paste }); + return; + } + + if ( + !singleLine && + (key.name === 'return' || input === '\r' || input === '\n' || input === '\\r') // VSCode terminal represents shift + enter this way + ) + newline(); + else if (key.name === 'left' && !key.meta && !key.ctrl) move('left'); + else if (key.ctrl && key.name === 'b') move('left'); + else if (key.name === 'right' && !key.meta && !key.ctrl) move('right'); + else if (key.ctrl && key.name === 'f') move('right'); + else if (key.name === 'up') move('up'); + else if (key.name === 'down') move('down'); + else if ((key.ctrl || key.meta) && key.name === 'left') move('wordLeft'); + else if (key.meta && key.name === 'b') move('wordLeft'); + else if ((key.ctrl || key.meta) && key.name === 'right') move('wordRight'); + else if (key.meta && key.name === 'f') move('wordRight'); + else if (key.name === 'home') move('home'); + else if (key.ctrl && key.name === 'a') move('home'); + else if (key.name === 'end') move('end'); + else if (key.ctrl && key.name === 'e') move('end'); + else if (key.ctrl && key.name === 'w') deleteWordLeft(); + else if ((key.meta || key.ctrl) && (key.name === 'backspace' || input === '\x7f')) + deleteWordLeft(); + else if ((key.meta || key.ctrl) && key.name === 'delete') deleteWordRight(); + else if (key.name === 'backspace' || input === '\x7f' || (key.ctrl && key.name === 'h')) + backspace(); + else if (key.name === 'delete' || (key.ctrl && key.name === 'd')) del(); + else if (key.ctrl && !key.shift && key.name === 'z') undo(); + else if (key.ctrl && key.shift && key.name === 'z') redo(); + else if (key.insertable) { + insert(input, { paste: key.paste }); + } + }, + [ + newline, + move, + deleteWordLeft, + deleteWordRight, + backspace, + del, + insert, + undo, + redo, + singleLine, + ] + ); + + const renderedVisualLines = useMemo( + () => visualLines.slice(visualScrollRow, visualScrollRow + viewport.height), + [visualLines, visualScrollRow, viewport.height] + ); + + const replaceRange = useCallback( + ( + startRow: number, + startCol: number, + endRow: number, + endCol: number, + text: string + ): void => { + dispatch({ + type: 'replace_range', + payload: { startRow, startCol, endRow, endCol, text }, + }); + }, + [] + ); + + const replaceRangeByOffset = useCallback( + (startOffset: number, endOffset: number, replacementText: string): void => { + const [startRow, startCol] = offsetToLogicalPos(text, startOffset); + const [endRow, endCol] = offsetToLogicalPos(text, endOffset); + replaceRange(startRow, startCol, endRow, endCol, replacementText); + }, + [text, replaceRange] + ); + + const moveToOffset = useCallback((offset: number): void => { + dispatch({ type: 'move_to_offset', payload: { offset } }); + }, []); + + const moveToVisualPosition = useCallback( + (visRow: number, visCol: number): void => { + const { visualLines, visualToLogicalMap } = visualLayout; + // Clamp visRow to valid range + const clampedVisRow = Math.max(0, Math.min(visRow, visualLines.length - 1)); + const visualLine = visualLines[clampedVisRow] || ''; + + if (visualToLogicalMap[clampedVisRow]) { + const [logRow, logStartCol] = visualToLogicalMap[clampedVisRow]; + + const codePoints = toCodePoints(visualLine); + let currentVisX = 0; + let charOffset = 0; + + for (const char of codePoints) { + const charWidth = getCachedStringWidth(char); + // If the click is within this character + if (visCol < currentVisX + charWidth) { + // Check if we clicked the second half of a wide character + if (charWidth > 1 && visCol >= currentVisX + charWidth / 2) { + charOffset++; + } + break; + } + currentVisX += charWidth; + charOffset++; + } + + // Clamp charOffset to length + charOffset = Math.min(charOffset, codePoints.length); + + const newCursorRow = logRow; + const newCursorCol = logStartCol + charOffset; + + dispatch({ + type: 'set_cursor', + payload: { + cursorRow: newCursorRow, + cursorCol: newCursorCol, + preferredCol: charOffset, + }, + }); + } + }, + [visualLayout] + ); + + const getOffset = useCallback( + (): number => logicalPosToOffset(lines, cursorRow, cursorCol), + [lines, cursorRow, cursorCol] + ); + + const returnValue: TextBuffer = useMemo( + () => ({ + lines, + text, + cursor: [cursorRow, cursorCol], + preferredCol, + + allVisualLines: visualLines, + viewportVisualLines: renderedVisualLines, + visualCursor, + visualScrollRow, + visualToLogicalMap, + + setText, + insert, + newline, + backspace, + del, + move, + undo, + redo, + replaceRange, + replaceRangeByOffset, + moveToOffset, + getOffset, + moveToVisualPosition, + deleteWordLeft, + deleteWordRight, + killLineRight, + killLineLeft, + handleInput, + setViewport, + }), + [ + lines, + text, + cursorRow, + cursorCol, + preferredCol, + visualLines, + renderedVisualLines, + visualCursor, + visualScrollRow, + visualToLogicalMap, + setText, + insert, + newline, + backspace, + del, + move, + undo, + redo, + replaceRange, + replaceRangeByOffset, + moveToOffset, + getOffset, + moveToVisualPosition, + deleteWordLeft, + deleteWordRight, + killLineRight, + killLineLeft, + handleInput, + setViewport, + ] + ); + return returnValue; +} + +export interface TextBuffer { + // State + lines: string[]; // Logical lines + text: string; + cursor: [number, number]; // Logical cursor [row, col] + /** + * When the user moves the caret vertically we try to keep their original + * horizontal column even when passing through shorter lines. We remember + * that *preferred* column in this field while the user is still travelling + * vertically. Any explicit horizontal movement resets the preference. + */ + preferredCol: number | null; + + // Visual state (handles wrapping) + allVisualLines: string[]; // All visual lines for the current text and viewport width. + viewportVisualLines: string[]; // The subset of visual lines to be rendered based on visualScrollRow and viewport.height + visualCursor: [number, number]; // Visual cursor [row, col] relative to the start of all visualLines + visualScrollRow: number; // Scroll position for visual lines (index of the first visible visual line) + /** + * For each visual line (by absolute index in allVisualLines) provides a tuple + * [logicalLineIndex, startColInLogical] that maps where that visual line + * begins within the logical buffer. Indices are code-point based. + */ + visualToLogicalMap: Array<[number, number]>; + + // Actions + + /** Replaces the entire buffer content with the provided text. Undoable. */ + setText: (text: string) => void; + /** Insert a single character or string. */ + insert: (ch: string, opts?: { paste?: boolean }) => void; + newline: () => void; + backspace: () => void; + del: () => void; + move: (dir: Direction) => void; + undo: () => void; + redo: () => void; + /** Replaces the text within the specified range with new text. */ + replaceRange: ( + startRow: number, + startCol: number, + endRow: number, + endCol: number, + text: string + ) => void; + /** Delete the word to the left of the caret. */ + deleteWordLeft: () => void; + /** Delete the word to the right of the caret. */ + deleteWordRight: () => void; + /** Deletes text from the cursor to the end of the current line. */ + killLineRight: () => void; + /** Deletes text from the start of the current line to the cursor. */ + killLineLeft: () => void; + /** High level "handleInput" – receives what Ink gives us. */ + handleInput: (key: Key) => void; + + replaceRangeByOffset: (startOffset: number, endOffset: number, replacementText: string) => void; + getOffset: () => number; + moveToOffset(offset: number): void; + moveToVisualPosition(visualRow: number, visualCol: number): void; + /** Update the viewport dimensions. */ + setViewport: (width: number, height: number) => void; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/constants/processingPhrases.ts b/dexto/packages/cli/src/cli/ink-cli/constants/processingPhrases.ts new file mode 100644 index 00000000..43131ecb --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/constants/processingPhrases.ts @@ -0,0 +1,97 @@ +/** + * Processing phrases for the status bar + * These cycle during processing to provide entertaining feedback + */ + +export const processingPhrases: string[] = [ + // pop culture + 'I want to be the very best…', + 'I will be the next hokage…', + 'Shinzou Sasageyo…', + 'My soldiers! Rage!…', + 'May the force be with you…', + 'Why so serious?…', + "That's what she said …", + 'Winter is coming…', + "It's over 9000!…", + "I'm Batman…", + "You can't handle the truth…", + 'We were on a break!…', + 'Bazinga!…', + "How you doin'?…", + "There's always money in the banana stand…", + 'I am the one who knocks…', + 'Yabba dabba doo!…', + 'The tribe has spoken…', + 'This is the Way…', + 'Plata o plomo…', + 'Yeah, science!…', + "They're minerals, Marie!…", + 'The North remembers…', + 'Life is like a box of chocolates…', + 'Avengers, assemble!', + 'I can do this all day…', + 'Elementary, my dear Watson…', + 'Identity theft is not a joke, Jim! …', + "I'm not superstitious, but I am a little stitious…", + 'Why waste time say lot word when few word do trick?…', + "You're a wizard, Harry…", + "I'll be back…", + 'Houston, we have a problem…', + 'Are you not entertained?…', + 'To infinity and beyond…', + 'Snakes. Why did it have to be snakes?…', + 'Hakuna matata…', + + // Playful + 'Let me cook…', + 'Manifesting greatness…', + 'Rizzing the huzz…', + 'Memeing…', + 'Outperforming other AI agents…', + 'Rolling with the squad…', + 'Incanting secret scripts…', + 'Making no mistakes…', + 'Making you rich…', + 'Farming easy points…', + 'Using 200+ IQ…', + 'Turning into Jarvis…', + 'Dextomaxxing…', + 'Zapping…', + 'Braining…', + 'Using all 3 brain cells…', + "I'm not lazy, I'm just on energy-saving mode…", + 'I came. I saw. I made it awkward…', + 'My boss told me to have a good day, so I went home…', + "I put the 'pro' in procrastination…", + 'Delulu is the solulu…', + 'Zombies eat brains. You are safe…', + //'Installing malware (just kidding)…', + + // Vines + 'Look at all those chickens…', + 'What are those!!…', + 'He needs some milk…', + 'Something came in the mail today…', + 'Road work ahead? I sure hope it does…', + 'Merry Chrysler…', + "I'm in me mum's car. Vroom vroom…", + 'Stop! I could have dropped my croissant!…', + 'That was legitness…', + 'Why are you running?…', + 'What da dog doin?…', + 'Can I pet that dawg?…', + 'And they were roommates!…', + + // Nerdy + 'Attention is all I need…', + 'Transformer powers activate…', +]; + +/** + * Get a random phrase from the list + */ +export function getRandomPhrase(): string { + const index = Math.floor(Math.random() * processingPhrases.length); + return processingPhrases[index] ?? 'Processing…'; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/constants/tips.ts b/dexto/packages/cli/src/cli/ink-cli/constants/tips.ts new file mode 100644 index 00000000..d3e596fe --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/constants/tips.ts @@ -0,0 +1,62 @@ +/** + * Informative tips for the status bar + * These are shown during processing to teach users about features + * + * Categories: + * - Command tips + * - Keyboard shortcut tips + * - Feature tips + */ + +export const tips: string[] = [ + // Command tips + 'Type /help to see all available commands…', + 'Use /model to switch between different AI models…', + 'Use /resume to load a previous conversation…', + 'Use /search to find messages across all sessions…', + 'Use /mcp to manage MCP servers…', + 'Use /tools to see all available tools…', + 'Use /prompts to browse, add and delete custom prompts…', + 'Use /log to change logging verbosity…', + 'Use /clear to clear the session context…', + 'Use /exit or /quit to close dexto…', + 'Use /docs to access documentation…', + 'Use /copy to copy the previous response…', + 'Use /shortcuts to see all available shortcuts…', + 'Use /sysprompt to see the current system prompt…', + 'Use /context to see the current token usage…', + 'Use /compact to summarize the current session…', + + // Keyboard shortcut tips + 'Press Escape to cancel the current request…', + 'Press Ctrl+C twice to exit dexto…', + 'Press Escape to close overlays and menus…', + 'Use Up/Down arrows to navigate command history…', + 'Press Enter to submit your message…', + 'Press Ctrl+T to show/hide the task list…', + 'Press Ctrl+R to search previous prompts…', + 'Press Shift+Enter to insert a new line…', + + // Feature tips + 'Start with ! to run bash commands directly…', + 'Paste copied images with Ctrl+V…', + 'Large pastes are automatically collapsed - press Ctrl+P to toggle…', + 'Use @ to reference files and resources…', + 'Type / to see available slash commands…', + 'MCP servers extend dexto with custom tools…', + 'Sessions are automatically saved for later…', + 'You can create custom commands with /prompts…', + 'You can submit messages while Dexto is processing…', + 'Use /stream to toggle streaming mode…', + + // Platform tips + 'On Mac, use Option+Up/Down to jump to start/end of input…', +]; + +/** + * Get a random tip from the list + */ +export function getRandomTip(): string { + const index = Math.floor(Math.random() * tips.length); + return tips[index] ?? 'Type /help to see available commands…'; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/containers/InputContainer.tsx b/dexto/packages/cli/src/cli/ink-cli/containers/InputContainer.tsx new file mode 100644 index 00000000..0a67ece1 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/containers/InputContainer.tsx @@ -0,0 +1,809 @@ +/** + * InputContainer Component + * Smart container for input area - handles submission and state + * + * Buffer is passed as prop from parent (useCLIState). + * No more ref chain - buffer can be accessed directly. + */ + +import React, { useCallback, useRef, useEffect, useImperativeHandle, forwardRef } from 'react'; +import type { DextoAgent, ContentPart, ImagePart, TextPart, QueuedMessage } from '@dexto/core'; +import { InputArea, type OverlayTrigger } from '../components/input/InputArea.js'; +import { InputService, processStream } from '../services/index.js'; +import { useSoundService } from '../contexts/index.js'; +import type { + Message, + UIState, + InputState, + SessionState, + PendingImage, + PastedBlock, + TodoItem, +} from '../state/types.js'; +import { createUserMessage } from '../utils/messageFormatting.js'; +import { generateMessageId } from '../utils/idGenerator.js'; +import type { ApprovalRequest } from '../components/ApprovalPrompt.js'; +import type { TextBuffer } from '../components/shared/text-buffer.js'; +import { capture } from '../../../analytics/index.js'; + +/** Type for pending session creation promise */ +type SessionCreationResult = { id: string }; + +/** Handle for imperative access to InputContainer */ +export interface InputContainerHandle { + /** Submit a command/message programmatically (bypasses overlay check) */ + submit: (text: string) => Promise<void>; +} + +interface InputContainerProps { + /** Text buffer (owned by useCLIState) */ + buffer: TextBuffer; + input: InputState; + ui: UIState; + session: SessionState; + approval: ApprovalRequest | null; + /** Queued messages waiting to be processed */ + queuedMessages: QueuedMessage[]; + setInput: React.Dispatch<React.SetStateAction<InputState>>; + setUi: React.Dispatch<React.SetStateAction<UIState>>; + setSession: React.Dispatch<React.SetStateAction<SessionState>>; + /** Setter for finalized messages (rendered in <Static>) */ + setMessages: React.Dispatch<React.SetStateAction<Message[]>>; + /** Setter for pending/streaming messages (rendered dynamically) */ + setPendingMessages: React.Dispatch<React.SetStateAction<Message[]>>; + /** Setter for dequeued buffer (user messages waiting to render after pending) */ + setDequeuedBuffer: React.Dispatch<React.SetStateAction<Message[]>>; + /** Setter for queued messages */ + setQueuedMessages: React.Dispatch<React.SetStateAction<QueuedMessage[]>>; + /** Setter for current approval request (for approval UI via processStream) */ + setApproval: React.Dispatch<React.SetStateAction<ApprovalRequest | null>>; + /** Setter for approval queue (for queued approvals via processStream) */ + setApprovalQueue: React.Dispatch<React.SetStateAction<ApprovalRequest[]>>; + /** Setter for todo items (for todo tool updates via processStream) */ + setTodos: React.Dispatch<React.SetStateAction<TodoItem[]>>; + agent: DextoAgent; + inputService: InputService; + /** Optional keyboard scroll handler (for alternate buffer mode) */ + onKeyboardScroll?: (direction: 'up' | 'down') => void; + /** Whether to stream chunks or wait for complete response (default: true) */ + useStreaming?: boolean; +} + +/** + * Smart container for input area + * Manages submission, history, and overlay triggers + */ +export const InputContainer = forwardRef<InputContainerHandle, InputContainerProps>( + function InputContainer( + { + buffer, + input, + ui, + session, + approval, + queuedMessages, + setInput, + setUi, + setSession, + setMessages, + setPendingMessages, + setDequeuedBuffer, + setQueuedMessages, + setApproval, + setApprovalQueue, + setTodos, + agent, + inputService, + onKeyboardScroll, + useStreaming = true, + }, + ref + ) { + // Track pending session creation to prevent race conditions + const sessionCreationPromiseRef = useRef<Promise<SessionCreationResult> | null>(null); + + // Sound notification service from context + const soundService = useSoundService(); + + // Ref to track autoApproveEdits so processStream can read latest value mid-stream + const autoApproveEditsRef = useRef(ui.autoApproveEdits); + useEffect(() => { + autoApproveEditsRef.current = ui.autoApproveEdits; + }, [ui.autoApproveEdits]); + + // Clear the session creation ref when session is cleared + useEffect(() => { + if (session.id === null) { + sessionCreationPromiseRef.current = null; + } + }, [session.id]); + + // Extract text content from ContentPart[] + const extractTextFromContent = useCallback((content: ContentPart[]): string => { + return content + .filter((part): part is TextPart => part.type === 'text') + .map((part) => part.text) + .join('\n'); + }, []); + + // Handle history navigation - set text directly on buffer + // Up arrow first edits queued messages (removes from queue), then navigates history + const handleHistoryNavigate = useCallback( + (direction: 'up' | 'down') => { + const { history, historyIndex, draftBeforeHistory } = input; + + if (direction === 'up') { + // First check if there are queued messages to edit + if (queuedMessages.length > 0 && session.id) { + // Get the last queued message + const lastQueued = queuedMessages[queuedMessages.length - 1]; + if (lastQueued) { + // Extract text content and put it in the input + const text = extractTextFromContent(lastQueued.content); + buffer.setText(text); + setInput((prev) => ({ ...prev, value: text })); + // Remove from queue (this will trigger the event and update queuedMessages state) + agent.removeQueuedMessage(session.id, lastQueued.id).catch(() => { + // Silently ignore errors - queue might have been cleared + }); + return; + } + } + + // Don't navigate history when processing (only queue editing is allowed) + if (ui.isProcessing) return; + + // No queued messages, navigate history + if (history.length === 0) return; + + let newIndex = historyIndex; + if (newIndex < 0) { + // First time pressing up - save current input as draft + const currentText = buffer.text; + setInput((prev) => ({ + ...prev, + draftBeforeHistory: currentText, + historyIndex: history.length - 1, + value: history[history.length - 1] || '', + })); + buffer.setText(history[history.length - 1] || ''); + return; + } else if (newIndex > 0) { + newIndex = newIndex - 1; + } else { + return; // Already at oldest + } + + const historyItem = history[newIndex] || ''; + buffer.setText(historyItem); + setInput((prev) => ({ ...prev, value: historyItem, historyIndex: newIndex })); + } else { + // Down - navigate history (queued messages don't affect down navigation) + // Don't navigate history when processing + if (ui.isProcessing) return; + if (historyIndex < 0) return; // Not navigating history + if (historyIndex < history.length - 1) { + const newIndex = historyIndex + 1; + const historyItem = history[newIndex] || ''; + buffer.setText(historyItem); + setInput((prev) => ({ + ...prev, + value: historyItem, + historyIndex: newIndex, + })); + } else { + // At newest history item, restore draft + buffer.setText(draftBeforeHistory); + setInput((prev) => ({ + ...prev, + value: draftBeforeHistory, + historyIndex: -1, + draftBeforeHistory: '', + })); + } + } + }, + [ + buffer, + input, + setInput, + queuedMessages, + session.id, + agent, + extractTextFromContent, + ui.isProcessing, + ] + ); + + // Handle overlay triggers + // Allow triggers while processing (for queuing), but not during approval + // IMPORTANT: Use functional updates to check prev.activeOverlay, not the closure value. + // This avoids race conditions when open/close happen in quick succession (React batching). + const handleTriggerOverlay = useCallback( + (trigger: OverlayTrigger) => { + if (approval) return; + + if (trigger === 'close') { + // Use functional update to check the ACTUAL current state, not stale closure + setUi((prev) => { + if ( + prev.activeOverlay === 'slash-autocomplete' || + prev.activeOverlay === 'resource-autocomplete' + ) { + return { + ...prev, + activeOverlay: 'none', + mcpWizardServerType: null, + }; + } + return prev; + }); + } else if (trigger === 'slash-autocomplete') { + setUi((prev) => ({ ...prev, activeOverlay: 'slash-autocomplete' })); + } else if (trigger === 'resource-autocomplete') { + setUi((prev) => ({ ...prev, activeOverlay: 'resource-autocomplete' })); + } + }, + [setUi, approval] + ); + + // Handle image paste from clipboard + const handleImagePaste = useCallback( + (image: PendingImage) => { + // Track image attachment analytics (only if session exists) + if (session.id) { + capture('dexto_image_attached', { + source: 'cli', + sessionId: session.id, + imageType: image.mimeType, + imageSizeBytes: Math.floor(image.data.length * 0.75), // Approx base64 decode + }); + } + + setInput((prev) => ({ + ...prev, + images: [...prev.images, image], + })); + }, + [setInput, session.id] + ); + + // Handle image removal (when placeholder is deleted from text) + const handleImageRemove = useCallback( + (imageId: string) => { + setInput((prev) => ({ + ...prev, + images: prev.images.filter((img) => img.id !== imageId), + })); + }, + [setInput] + ); + + // Handle new paste block creation (when large text is pasted) + const handlePasteBlock = useCallback( + (block: PastedBlock) => { + setInput((prev) => ({ + ...prev, + pastedBlocks: [...prev.pastedBlocks, block], + pasteCounter: Math.max(prev.pasteCounter, block.number), + })); + }, + [setInput] + ); + + // Handle paste block update (e.g., toggle collapse) + const handlePasteBlockUpdate = useCallback( + (blockId: string, updates: Partial<PastedBlock>) => { + setInput((prev) => ({ + ...prev, + pastedBlocks: prev.pastedBlocks.map((block) => + block.id === blockId ? { ...block, ...updates } : block + ), + })); + }, + [setInput] + ); + + // Handle paste block removal (when placeholder is deleted from text) + const handlePasteBlockRemove = useCallback( + (blockId: string) => { + setInput((prev) => ({ + ...prev, + pastedBlocks: prev.pastedBlocks.filter((block) => block.id !== blockId), + })); + }, + [setInput] + ); + + // Expand all collapsed paste blocks in a text string + const expandPasteBlocks = useCallback((text: string, blocks: PastedBlock[]): string => { + let result = text; + // Sort blocks by placeholder position descending to avoid offset issues + const sortedBlocks = [...blocks].sort((a, b) => { + const posA = result.indexOf(a.placeholder); + const posB = result.indexOf(b.placeholder); + return posB - posA; + }); + + for (const block of sortedBlocks) { + if (block.isCollapsed) { + // Replace placeholder with full text + result = result.replace(block.placeholder, block.fullText); + } + } + return result; + }, []); + + // Handle submission + // bypassOverlayCheck: skip the overlay check when called programmatically (e.g., from OverlayContainer) + const handleSubmit = useCallback( + async (value: string, bypassOverlayCheck = false) => { + // Expand all collapsed paste blocks before processing + const expandedValue = expandPasteBlocks(value, input.pastedBlocks); + const trimmed = expandedValue.trim(); + if (!trimmed) return; + + // Auto-queue when agent is processing + if (ui.isProcessing && session.id) { + // Build content parts for queueing + const content: ContentPart[] = [{ type: 'text', text: trimmed } as TextPart]; + // Add images if any + for (const img of input.images) { + content.push({ + type: 'image', + image: img.data, + mimeType: img.mimeType, + } as ImagePart); + } + + try { + await agent.queueMessage(session.id, { content }); + // Queued messages are displayed via QueuedMessagesDisplay component + // (state updated by message:queued event handler in useAgentEvents) + + // Clear input, update history, and clear images + buffer.setText(''); + setInput((prev) => { + const newHistory = + prev.history.length > 0 && + prev.history[prev.history.length - 1] === trimmed + ? prev.history + : [...prev.history, trimmed].slice(-100); + return { + ...prev, + value: '', + history: newHistory, + historyIndex: -1, + draftBeforeHistory: '', + images: [], + pastedBlocks: [], + }; + }); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `Failed to queue message: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + return; + } + + // Prevent double submission when autocomplete/selector is active + // Skip this check when called programmatically (e.g., from OverlayContainer prompt selection) + if ( + !bypassOverlayCheck && + ui.activeOverlay !== 'none' && + ui.activeOverlay !== 'approval' + ) { + return; + } + + // Capture images before clearing - we need them for the API call + const pendingImages = [...input.images]; + + // Create user message and add it to messages + const userMessage = createUserMessage(trimmed); + setMessages((prev) => [...prev, userMessage]); + + // Clear input directly on buffer and update history + buffer.setText(''); + setInput((prev) => { + const newHistory = + prev.history.length > 0 && prev.history[prev.history.length - 1] === trimmed + ? prev.history + : [...prev.history, trimmed].slice(-100); + return { + value: '', + history: newHistory, + historyIndex: -1, + draftBeforeHistory: '', + images: [], // Clear images on submit + pastedBlocks: [], // Clear paste blocks on submit + pasteCounter: prev.pasteCounter, // Keep counter for next session + }; + }); + + // Start processing + setUi((prev) => ({ + ...prev, + isProcessing: true, + isCancelling: false, + activeOverlay: 'none', + exitWarningShown: false, + exitWarningTimestamp: null, + })); + + // Parse and handle command or prompt + const parsed = inputService.parseInput(trimmed); + + // Check if this command should show an interactive overlay + if (parsed.type === 'command' && parsed.command) { + const { getCommandOverlay } = await import('../utils/commandOverlays.js'); + const overlay = getCommandOverlay(parsed.command, parsed.args || []); + if (overlay) { + setUi((prev) => ({ + ...prev, + isProcessing: false, + activeOverlay: overlay, + })); + return; + } + } + + if (parsed.type === 'command' && parsed.command) { + const { CommandService } = await import('../services/CommandService.js'); + const commandService = new CommandService(); + + try { + const result = await commandService.executeCommand( + parsed.command, + parsed.args || [], + agent, + session.id || undefined + ); + + if (result.type === 'output' && result.output) { + const output = result.output; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('command'), + role: 'system', + content: output, + timestamp: new Date(), + }, + ]); + } + + if (result.type === 'styled' && result.styled) { + const { fallbackText, styledType, styledData } = result.styled; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('command'), + role: 'system', + content: fallbackText, + timestamp: new Date(), + styledType, + styledData, + }, + ]); + } + + // Handle sendMessage - send through normal streaming flow + if (result.type === 'sendMessage' && result.messageToSend) { + let currentSessionId = session.id; + + if (!currentSessionId) { + if (sessionCreationPromiseRef.current) { + try { + const existingSession = + await sessionCreationPromiseRef.current; + currentSessionId = existingSession.id; + } catch { + sessionCreationPromiseRef.current = null; + } + } + + if (!currentSessionId) { + const sessionPromise = agent.createSession(); + sessionCreationPromiseRef.current = sessionPromise; + + const newSession = await sessionPromise; + currentSessionId = newSession.id; + setSession((prev) => ({ + ...prev, + id: currentSessionId, + hasActiveSession: true, + })); + } + } + + if (!currentSessionId) { + throw new Error('Failed to create or retrieve session'); + } + + // Send through normal streaming flow (matches WebUI pattern) + const iterator = await agent.stream( + result.messageToSend, + currentSessionId + ); + await processStream( + iterator, + { + setMessages, + setPendingMessages, + setDequeuedBuffer, + setUi, + setSession, + setQueuedMessages, + setApproval, + setApprovalQueue, + }, + { + useStreaming, + autoApproveEditsRef, + eventBus: agent.agentEventBus, + setTodos, + ...(soundService && { soundService }), + } + ); + return; // processStream handles UI state + } + + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Error: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } + } else { + try { + let currentSessionId = session.id; + + if (!currentSessionId) { + if (sessionCreationPromiseRef.current) { + try { + const existingSession = await sessionCreationPromiseRef.current; + currentSessionId = existingSession.id; + } catch { + sessionCreationPromiseRef.current = null; + } + } + + if (!currentSessionId) { + const sessionPromise = agent.createSession(); + sessionCreationPromiseRef.current = sessionPromise; + + const newSession = await sessionPromise; + currentSessionId = newSession.id; + setSession((prev) => ({ + ...prev, + id: currentSessionId, + hasActiveSession: true, + })); + } + } + + if (!currentSessionId) { + throw new Error('Failed to create or retrieve session'); + } + + const metadata = await agent.getSessionMetadata(currentSessionId); + const isFirstMessage = !metadata || metadata.messageCount <= 0; + + // Build content with images if any + let content: string | ContentPart[]; + + // Plan mode injection: prepend plan skill content on first message + // The <plan-mode> tags are filtered out by the message renderer so users + // don't see the instructions, but the LLM receives them. + // + // TODO: Consider dropping <plan-mode> content after plan is approved/disabled. + // Edge case: user may still need access to plan tools after approval for + // progress tracking (checking off tasks). Current approach keeps it simple + // by letting the content remain in context - it's not re-injected, just + // stays from the first message. + let messageText = trimmed; + if (ui.planModeActive && !ui.planModeInitialized) { + try { + const planSkill = await agent.resolvePrompt('plan', {}); + if (planSkill.text) { + messageText = `<plan-mode>\n${planSkill.text}\n</plan-mode>\n\n${trimmed}`; + // Mark plan mode as initialized after injection + setUi((prev) => ({ ...prev, planModeInitialized: true })); + } + } catch { + // Plan skill not found - continue without injection + // This can happen if the plan-tools plugin is not enabled + } + } + + if (pendingImages.length > 0) { + // Build multimodal content parts + const parts: ContentPart[] = []; + + // Add text part first (with potential plan-mode injection) + parts.push({ type: 'text', text: messageText } as TextPart); + + // Add image parts + for (const img of pendingImages) { + parts.push({ + type: 'image', + image: img.data, + mimeType: img.mimeType, + } as ImagePart); + } + + content = parts; + } else { + content = messageText; + } + + // Get current LLM config for analytics + const llmConfig = agent.getCurrentLLMConfig(); + + // Track message sent analytics + capture('dexto_message_sent', { + source: 'cli', + sessionId: currentSessionId, + provider: llmConfig.provider, + model: llmConfig.model, + hasImage: pendingImages.length > 0, + hasFile: false, + messageLength: trimmed.length, + }); + + // Use streaming API and process events directly + const iterator = await agent.stream(content, currentSessionId); + await processStream( + iterator, + { + setMessages, + setPendingMessages, + setDequeuedBuffer, + setUi, + setSession, + setQueuedMessages, + setApproval, + setApprovalQueue, + }, + { + useStreaming, + autoApproveEditsRef, + eventBus: agent.agentEventBus, + setTodos, + ...(soundService && { soundService }), + } + ); + + if (isFirstMessage) { + agent.generateSessionTitle(currentSessionId).catch(() => { + // Title generation is non-critical - silently ignore failures + }); + } + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Error: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } + } + }, + [ + buffer, + input.images, + input.pastedBlocks, + expandPasteBlocks, + setInput, + setUi, + setMessages, + setPendingMessages, + setDequeuedBuffer, + setQueuedMessages, + setSession, + agent, + inputService, + ui.isProcessing, + ui.activeOverlay, + ui.planModeActive, + ui.planModeInitialized, + session.id, + useStreaming, + soundService, + ] + ); + + // Determine if input should be active (not blocked by approval/overlay/history search) + // Input stays active for filter-type overlays (so user can keep typing to filter) + // Disable for approval prompts, overlays with their own text input, and history search mode + const overlaysWithOwnInput = [ + 'mcp-custom-wizard', + 'custom-model-wizard', + 'api-key-input', + 'search', + 'tool-browser', + 'prompt-add-wizard', + 'model-selector', + 'export-wizard', + 'marketplace-add', + ]; + const hasOverlayWithOwnInput = overlaysWithOwnInput.includes(ui.activeOverlay); + const isHistorySearchActive = ui.historySearch.isActive; + const isInputActive = !approval && !hasOverlayWithOwnInput && !isHistorySearchActive; + const isInputDisabled = !!approval || hasOverlayWithOwnInput || isHistorySearchActive; + // Allow submit when: + // - no overlay active + // - approval active + // Note: slash-autocomplete handles its own Enter key (either executes command or submits raw text) + const shouldHandleSubmit = ui.activeOverlay === 'none' || ui.activeOverlay === 'approval'; + // Allow history navigation when not blocked by approval/overlay + // Allow during processing so users can browse previous prompts while agent runs + const canNavigateHistory = !approval && ui.activeOverlay === 'none'; + + const placeholder = approval + ? 'Approval required above...' + : 'Type your message or /help for commands'; + + // Expose submit method for external use (e.g., from OverlayContainer) + // Pass bypassOverlayCheck=true since programmatic calls should skip the overlay check + useImperativeHandle(ref, () => ({ + submit: (text: string) => handleSubmit(text, true), + })); + + return ( + <InputArea + buffer={buffer} + onSubmit={shouldHandleSubmit ? handleSubmit : () => {}} + isDisabled={isInputDisabled} + isActive={isInputActive} + placeholder={placeholder} + onHistoryNavigate={canNavigateHistory ? handleHistoryNavigate : undefined} + onTriggerOverlay={handleTriggerOverlay} + onKeyboardScroll={onKeyboardScroll} + imageCount={input.images.length} + onImagePaste={handleImagePaste} + images={input.images} + onImageRemove={handleImageRemove} + pastedBlocks={input.pastedBlocks} + onPasteBlock={handlePasteBlock} + onPasteBlockUpdate={handlePasteBlockUpdate} + onPasteBlockRemove={handlePasteBlockRemove} + highlightQuery={ui.historySearch.isActive ? ui.historySearch.query : undefined} + /> + ); + } +); diff --git a/dexto/packages/cli/src/cli/ink-cli/containers/OverlayContainer.tsx b/dexto/packages/cli/src/cli/ink-cli/containers/OverlayContainer.tsx new file mode 100644 index 00000000..0faa2d86 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/containers/OverlayContainer.tsx @@ -0,0 +1,2442 @@ +/** + * OverlayContainer Component + * Smart container for managing all overlays (selectors, autocomplete, approval) + */ + +import React, { useCallback, useRef, useImperativeHandle, forwardRef, useState } from 'react'; +import { Box } from 'ink'; +import type { DextoAgent, McpServerConfig, McpServerStatus, McpServerType } from '@dexto/core'; +import type { TextBuffer } from '../components/shared/text-buffer.js'; +import type { Key } from '../hooks/useInputOrchestrator.js'; +import { ApprovalStatus, DenialReason, isUserMessage } from '@dexto/core'; +import type { Message, UIState, InputState, SessionState } from '../state/types.js'; +import { + ApprovalPrompt, + type ApprovalPromptHandle, + type ApprovalRequest, +} from '../components/ApprovalPrompt.js'; +import { + SlashCommandAutocomplete, + type SlashCommandAutocompleteHandle, +} from '../components/SlashCommandAutocomplete.js'; +import ResourceAutocomplete, { + type ResourceAutocompleteHandle, +} from '../components/ResourceAutocomplete.js'; +import ModelSelectorRefactored, { + type ModelSelectorHandle, +} from '../components/overlays/ModelSelectorRefactored.js'; +import SessionSelectorRefactored, { + type SessionSelectorHandle, +} from '../components/overlays/SessionSelectorRefactored.js'; +import LogLevelSelector, { + type LogLevelSelectorHandle, +} from '../components/overlays/LogLevelSelector.js'; +import StreamSelector, { + type StreamSelectorHandle, +} from '../components/overlays/StreamSelector.js'; +import ToolBrowser, { type ToolBrowserHandle } from '../components/overlays/ToolBrowser.js'; +import McpServerList, { + type McpServerListHandle, + type McpServerListAction, +} from '../components/overlays/McpServerList.js'; +import McpServerActions, { + type McpServerActionsHandle, + type McpServerAction, +} from '../components/overlays/McpServerActions.js'; +import McpAddChoice, { + type McpAddChoiceHandle, + type McpAddChoiceType, +} from '../components/overlays/McpAddChoice.js'; +import McpAddSelector, { + type McpAddSelectorHandle, + type McpAddResult, +} from '../components/overlays/McpAddSelector.js'; +import SessionSubcommandSelector, { + type SessionSubcommandSelectorHandle, + type SessionAction, +} from '../components/overlays/SessionSubcommandSelector.js'; +import McpCustomTypeSelector, { + type McpCustomTypeSelectorHandle, +} from '../components/overlays/McpCustomTypeSelector.js'; +import McpCustomWizard, { + type McpCustomWizardHandle, + type McpCustomConfig, +} from '../components/overlays/McpCustomWizard.js'; +import CustomModelWizard, { + type CustomModelWizardHandle, +} from '../components/overlays/CustomModelWizard.js'; +import type { CustomModel, ListedPlugin } from '@dexto/agent-management'; +import ApiKeyInput, { type ApiKeyInputHandle } from '../components/overlays/ApiKeyInput.js'; +import SearchOverlay, { type SearchOverlayHandle } from '../components/overlays/SearchOverlay.js'; +import PromptList, { + type PromptListHandle, + type PromptListAction, +} from '../components/overlays/PromptList.js'; +import PromptAddChoice, { + type PromptAddChoiceHandle, + type PromptAddChoiceResult, +} from '../components/overlays/PromptAddChoice.js'; +import PromptAddWizard, { + type PromptAddWizardHandle, + type NewPromptData, +} from '../components/overlays/PromptAddWizard.js'; +import PromptDeleteSelector, { + type PromptDeleteSelectorHandle, + type DeletablePrompt, +} from '../components/overlays/PromptDeleteSelector.js'; +import SessionRenameOverlay, { + type SessionRenameOverlayHandle, +} from '../components/overlays/SessionRenameOverlay.js'; +import ContextStatsOverlay, { + type ContextStatsOverlayHandle, +} from '../components/overlays/ContextStatsOverlay.js'; +import ExportWizard, { type ExportWizardHandle } from '../components/overlays/ExportWizard.js'; +import PluginManager, { + type PluginManagerHandle, + type PluginAction, +} from '../components/overlays/PluginManager.js'; +import PluginList, { type PluginListHandle } from '../components/overlays/PluginList.js'; +import PluginActions, { + type PluginActionsHandle, + type PluginActionResult, +} from '../components/overlays/PluginActions.js'; +import MarketplaceBrowser, { + type MarketplaceBrowserHandle, + type MarketplaceBrowserAction, +} from '../components/overlays/MarketplaceBrowser.js'; +import MarketplaceAddPrompt, { + type MarketplaceAddPromptHandle, +} from '../components/overlays/MarketplaceAddPrompt.js'; +import type { PromptAddScope } from '../state/types.js'; +import type { PromptInfo, ResourceMetadata, LLMProvider, SearchResult } from '@dexto/core'; +import type { LogLevel } from '@dexto/core'; +import { DextoValidationError, LLMErrorCode } from '@dexto/core'; +import { InputService } from '../services/InputService.js'; +import { createUserMessage, convertHistoryToUIMessages } from '../utils/messageFormatting.js'; +import { generateMessageId } from '../utils/idGenerator.js'; +import { capture } from '../../../analytics/index.js'; + +export interface OverlayContainerHandle { + handleInput: (input: string, key: Key) => boolean; +} + +interface OverlayContainerProps { + ui: UIState; + input: InputState; + session: SessionState; + approval: ApprovalRequest | null; + setInput: React.Dispatch<React.SetStateAction<InputState>>; + setUi: React.Dispatch<React.SetStateAction<UIState>>; + setSession: React.Dispatch<React.SetStateAction<SessionState>>; + setMessages: React.Dispatch<React.SetStateAction<Message[]>>; + setApproval: React.Dispatch<React.SetStateAction<ApprovalRequest | null>>; + setApprovalQueue: React.Dispatch<React.SetStateAction<ApprovalRequest[]>>; + agent: DextoAgent; + inputService: InputService; + buffer: TextBuffer; + /** Callback to refresh static content (clear terminal and force re-render) */ + refreshStatic?: () => void; + /** Callback to submit a prompt command through the normal streaming flow */ + onSubmitPromptCommand?: (commandText: string) => Promise<void>; +} + +/** + * Smart container for managing overlays + * Handles all modal interactions (selectors, autocomplete, approval) + */ +export const OverlayContainer = forwardRef<OverlayContainerHandle, OverlayContainerProps>( + function OverlayContainer( + { + ui, + input, + session, + approval, + setInput, + setUi, + setSession, + setMessages, + setApproval, + setApprovalQueue, + agent, + inputService, + buffer, + refreshStatic, + onSubmitPromptCommand, + }, + ref + ) { + const eventBus = agent.agentEventBus; + + // Refs to overlay components for input handling + const approvalRef = useRef<ApprovalPromptHandle>(null); + const slashAutocompleteRef = useRef<SlashCommandAutocompleteHandle>(null); + const resourceAutocompleteRef = useRef<ResourceAutocompleteHandle>(null); + const modelSelectorRef = useRef<ModelSelectorHandle>(null); + const sessionSelectorRef = useRef<SessionSelectorHandle>(null); + const logLevelSelectorRef = useRef<LogLevelSelectorHandle>(null); + const streamSelectorRef = useRef<StreamSelectorHandle>(null); + const toolBrowserRef = useRef<ToolBrowserHandle>(null); + const mcpServerListRef = useRef<McpServerListHandle>(null); + const mcpServerActionsRef = useRef<McpServerActionsHandle>(null); + const mcpAddChoiceRef = useRef<McpAddChoiceHandle>(null); + const mcpAddSelectorRef = useRef<McpAddSelectorHandle>(null); + const mcpCustomTypeSelectorRef = useRef<McpCustomTypeSelectorHandle>(null); + const mcpCustomWizardRef = useRef<McpCustomWizardHandle>(null); + const customModelWizardRef = useRef<CustomModelWizardHandle>(null); + const sessionSubcommandSelectorRef = useRef<SessionSubcommandSelectorHandle>(null); + const apiKeyInputRef = useRef<ApiKeyInputHandle>(null); + const searchOverlayRef = useRef<SearchOverlayHandle>(null); + const promptListRef = useRef<PromptListHandle>(null); + const promptAddChoiceRef = useRef<PromptAddChoiceHandle>(null); + const promptAddWizardRef = useRef<PromptAddWizardHandle>(null); + const promptDeleteSelectorRef = useRef<PromptDeleteSelectorHandle>(null); + const sessionRenameRef = useRef<SessionRenameOverlayHandle>(null); + const contextStatsRef = useRef<ContextStatsOverlayHandle>(null); + const exportWizardRef = useRef<ExportWizardHandle>(null); + const pluginManagerRef = useRef<PluginManagerHandle>(null); + const pluginListRef = useRef<PluginListHandle>(null); + const pluginActionsRef = useRef<PluginActionsHandle>(null); + const marketplaceBrowserRef = useRef<MarketplaceBrowserHandle>(null); + + // State for selected plugin (for plugin-actions overlay) + const [selectedPlugin, setSelectedPlugin] = useState<ListedPlugin | null>(null); + const marketplaceAddPromptRef = useRef<MarketplaceAddPromptHandle>(null); + + // Expose handleInput method via ref - routes to appropriate overlay + useImperativeHandle( + ref, + () => ({ + handleInput: (inputStr: string, key: Key): boolean => { + // Route to approval first (highest priority) + if (approval && approvalRef.current) { + return approvalRef.current.handleInput(inputStr, key); + } + + // Route to active overlay based on type + switch (ui.activeOverlay) { + case 'slash-autocomplete': + return ( + slashAutocompleteRef.current?.handleInput(inputStr, key) ?? false + ); + case 'resource-autocomplete': + return ( + resourceAutocompleteRef.current?.handleInput(inputStr, key) ?? false + ); + case 'model-selector': + return modelSelectorRef.current?.handleInput(inputStr, key) ?? false; + case 'session-selector': + return sessionSelectorRef.current?.handleInput(inputStr, key) ?? false; + case 'log-level-selector': + return logLevelSelectorRef.current?.handleInput(inputStr, key) ?? false; + case 'stream-selector': + return streamSelectorRef.current?.handleInput(inputStr, key) ?? false; + case 'tool-browser': + return toolBrowserRef.current?.handleInput(inputStr, key) ?? false; + case 'mcp-server-list': + return mcpServerListRef.current?.handleInput(inputStr, key) ?? false; + case 'mcp-server-actions': + return mcpServerActionsRef.current?.handleInput(inputStr, key) ?? false; + case 'mcp-add-choice': + return mcpAddChoiceRef.current?.handleInput(inputStr, key) ?? false; + case 'mcp-add-selector': + return mcpAddSelectorRef.current?.handleInput(inputStr, key) ?? false; + case 'mcp-custom-type-selector': + return ( + mcpCustomTypeSelectorRef.current?.handleInput(inputStr, key) ?? + false + ); + case 'mcp-custom-wizard': + return mcpCustomWizardRef.current?.handleInput(inputStr, key) ?? false; + case 'custom-model-wizard': + return ( + customModelWizardRef.current?.handleInput(inputStr, key) ?? false + ); + case 'session-subcommand-selector': + return ( + sessionSubcommandSelectorRef.current?.handleInput(inputStr, key) ?? + false + ); + case 'api-key-input': + return apiKeyInputRef.current?.handleInput(inputStr, key) ?? false; + case 'search': + return searchOverlayRef.current?.handleInput(inputStr, key) ?? false; + case 'prompt-list': + return promptListRef.current?.handleInput(inputStr, key) ?? false; + case 'prompt-add-choice': + return promptAddChoiceRef.current?.handleInput(inputStr, key) ?? false; + case 'prompt-add-wizard': + return promptAddWizardRef.current?.handleInput(inputStr, key) ?? false; + case 'prompt-delete-selector': + return ( + promptDeleteSelectorRef.current?.handleInput(inputStr, key) ?? false + ); + case 'session-rename': + return sessionRenameRef.current?.handleInput(inputStr, key) ?? false; + case 'context-stats': + return contextStatsRef.current?.handleInput(inputStr, key) ?? false; + case 'export-wizard': + return exportWizardRef.current?.handleInput(inputStr, key) ?? false; + case 'plugin-manager': + return pluginManagerRef.current?.handleInput(inputStr, key) ?? false; + case 'plugin-list': + return pluginListRef.current?.handleInput(inputStr, key) ?? false; + case 'plugin-actions': + return pluginActionsRef.current?.handleInput(inputStr, key) ?? false; + case 'marketplace-browser': + return ( + marketplaceBrowserRef.current?.handleInput(inputStr, key) ?? false + ); + case 'marketplace-add': + return ( + marketplaceAddPromptRef.current?.handleInput(inputStr, key) ?? false + ); + default: + return false; + } + }, + }), + [approval, ui.activeOverlay] + ); + + // NOTE: Automatic overlay detection removed to prevent infinite loop + // Overlays are now shown explicitly via setUi from InputContainer + // or from the main component's input detection logic + + // Helper: Complete approval and process queue + const completeApproval = useCallback(() => { + setApprovalQueue((queue) => { + if (queue.length > 0) { + // Show next approval from queue + const [next, ...rest] = queue; + setApproval(next!); + setUi((prev) => ({ ...prev, activeOverlay: 'approval' })); + return rest; + } else { + // No more approvals + setApproval(null); + setUi((prev) => ({ ...prev, activeOverlay: 'none' })); + return []; + } + }); + }, [setApproval, setApprovalQueue, setUi]); + + // Handle approval responses + const handleApprove = useCallback( + (options: { + rememberChoice?: boolean; + rememberPattern?: string; + formData?: Record<string, unknown>; + enableAcceptEditsMode?: boolean; + rememberDirectory?: boolean; + }) => { + if (!approval || !eventBus) return; + + // Enable "accept all edits" mode if requested + if (options.enableAcceptEditsMode) { + setUi((prev) => ({ ...prev, autoApproveEdits: true })); + } + + // Auto-disable plan mode when plan_create or plan_review is approved + // This signals the transition from planning phase to execution phase + const toolName = approval.metadata.toolName as string | undefined; + const isPlanTool = + toolName === 'plan_create' || + toolName === 'plan_review' || + toolName === 'custom--plan_create' || + toolName === 'custom--plan_review'; + if (isPlanTool) { + setUi((prev) => ({ + ...prev, + planModeActive: false, + planModeInitialized: false, + })); + } + + eventBus.emit('approval:response', { + approvalId: approval.approvalId, + status: ApprovalStatus.APPROVED, + sessionId: approval.sessionId, + data: { + rememberChoice: options.rememberChoice, + rememberPattern: options.rememberPattern, + formData: options.formData, + rememberDirectory: options.rememberDirectory, + }, + }); + + completeApproval(); + }, + [approval, eventBus, completeApproval, setUi] + ); + + const handleDeny = useCallback( + (feedback?: string) => { + if (!approval || !eventBus) return; + + // Include user feedback in the denial message if provided + const message = feedback + ? `User requested changes: ${feedback}` + : 'User denied the tool execution'; + + eventBus.emit('approval:response', { + approvalId: approval.approvalId, + status: ApprovalStatus.DENIED, + sessionId: approval.sessionId, + reason: DenialReason.USER_DENIED, + message, + }); + + completeApproval(); + }, + [approval, eventBus, completeApproval] + ); + + const handleCancelApproval = useCallback(() => { + if (!approval || !eventBus) return; + + eventBus.emit('approval:response', { + approvalId: approval.approvalId, + status: ApprovalStatus.CANCELLED, + sessionId: approval.sessionId, + reason: DenialReason.USER_CANCELLED, + message: 'User cancelled the approval request', + }); + + completeApproval(); + }, [approval, eventBus, completeApproval]); + + // Helper: Check if error is due to missing API key + const isApiKeyMissingError = (error: unknown): LLMProvider | null => { + if (error instanceof DextoValidationError) { + const apiKeyIssue = error.issues.find( + (issue) => issue.code === LLMErrorCode.API_KEY_MISSING + ); + if (apiKeyIssue && apiKeyIssue.context) { + // Extract provider from context + const context = apiKeyIssue.context as { provider?: string }; + if (context.provider) { + return context.provider as LLMProvider; + } + } + } + return null; + }; + + // Handle model selection + const handleModelSelect = useCallback( + async ( + provider: string, + model: string, + displayName?: string, + baseURL?: string, + reasoningEffort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' + ) => { + // Pre-check: Dexto provider requires OAuth login AND API key + // Check BEFORE closing the overlay so user can pick a different model + if (provider === 'dexto') { + try { + const { canUseDextoProvider } = await import('../../utils/dexto-setup.js'); + const canUse = await canUseDextoProvider(); + if (!canUse) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: + '❌ Cannot switch to Dexto model - authentication required. Run /login to authenticate.', + timestamp: new Date(), + }, + ]); + // Don't close the overlay - let user pick a different model + return; + } + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Failed to verify Dexto auth: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + // Don't close the overlay - let user pick a different model + return; + } + } + + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + try { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `🔄 Switching to ${displayName || model} (${provider})...`, + timestamp: new Date(), + }, + ]); + + await agent.switchLLM( + { provider: provider as LLMProvider, model, baseURL, reasoningEffort }, + session.id || undefined + ); + + // Update session state with display name (fallback to model ID) + setSession((prev) => ({ ...prev, modelName: displayName || model })); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Successfully switched to ${displayName || model} (${provider})`, + timestamp: new Date(), + }, + ]); + } catch (error) { + // Check if error is due to missing API key + const missingProvider = isApiKeyMissingError(error); + if (missingProvider) { + // Store pending model switch and show API key input + // Use missingProvider (from error) as the authoritative source + setUi((prev) => ({ + ...prev, + activeOverlay: 'api-key-input', + pendingModelSwitch: { + provider: missingProvider, + model, + ...(displayName && { displayName }), + }, + })); + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `🔑 API key required for ${provider}`, + timestamp: new Date(), + }, + ]); + return; + } + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Failed to switch model: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + }, + [setUi, setInput, setMessages, setSession, agent, session.id, buffer] + ); + + // State for editing custom model + const [editingModel, setEditingModel] = useState<CustomModel | null>(null); + + // Handle "Add custom model" from model selector + const handleAddCustomModel = useCallback(() => { + setEditingModel(null); + setUi((prev) => ({ ...prev, activeOverlay: 'custom-model-wizard' })); + }, [setUi]); + + // Handle "Edit custom model" from model selector + const handleEditCustomModel = useCallback( + (model: CustomModel) => { + setEditingModel(model); + setUi((prev) => ({ ...prev, activeOverlay: 'custom-model-wizard' })); + }, + [setUi] + ); + + // Handle custom model wizard completion + const handleCustomModelComplete = useCallback( + async (model: CustomModel) => { + const wasEditing = editingModel !== null; + setEditingModel(null); + setUi((prev) => ({ ...prev, activeOverlay: 'none' })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + if (wasEditing) { + // For edits, just show confirmation message + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Custom model "${model.displayName || model.name}" updated`, + timestamp: new Date(), + }, + ]); + } else { + // For new models, auto-switch to the newly created model + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Custom model "${model.displayName || model.name}" saved`, + timestamp: new Date(), + }, + ]); + + // Switch to the new model + await handleModelSelect( + model.provider, + model.name, + model.displayName, + model.baseURL + ); + } + }, + [setUi, setInput, setMessages, buffer, editingModel, handleModelSelect] + ); + + // Handle API key saved - retry the model switch + const handleApiKeySaved = useCallback( + async (meta: { provider: LLMProvider; envVar: string }) => { + const pending = ui.pendingModelSwitch; + if (!pending) { + // No pending switch, just close + setUi((prev) => ({ + ...prev, + activeOverlay: 'none', + pendingModelSwitch: null, + })); + return; + } + + setUi((prev) => ({ + ...prev, + activeOverlay: 'none', + pendingModelSwitch: null, + })); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ API key saved for ${meta.provider}`, + timestamp: new Date(), + }, + ]); + + // Retry the model switch + try { + const pendingDisplayName = pending.displayName || pending.model; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `🔄 Retrying switch to ${pendingDisplayName} (${pending.provider})...`, + timestamp: new Date(), + }, + ]); + + await agent.switchLLM( + { provider: pending.provider as LLMProvider, model: pending.model }, + session.id || undefined + ); + + // Update session state with display name (fallback to model ID) + setSession((prev) => ({ ...prev, modelName: pendingDisplayName })); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Successfully switched to ${pendingDisplayName} (${pending.provider})`, + timestamp: new Date(), + }, + ]); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Failed to switch model: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + }, + [ui.pendingModelSwitch, setUi, setMessages, setSession, agent, session.id] + ); + + // Handle API key input close (without saving) + const handleApiKeyClose = useCallback(() => { + setUi((prev) => ({ + ...prev, + activeOverlay: 'none', + pendingModelSwitch: null, + })); + }, [setUi]); + + // Handle search result selection - display the result context + const handleSearchResultSelect = useCallback( + (result: SearchResult) => { + setUi((prev) => ({ ...prev, activeOverlay: 'none' })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + // Display the selected search result as a system message + const roleLabel = + result.message.role === 'user' + ? '👤 User' + : result.message.role === 'assistant' + ? '🤖 Assistant' + : `📋 ${result.message.role}`; + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `🔍 Search Result from session ${result.sessionId.slice(0, 8)}:\n\n${roleLabel}:\n${result.context}`, + timestamp: new Date(), + }, + ]); + }, + [setUi, setInput, setMessages, buffer] + ); + + // Handle session selection + const handleSessionSelect = useCallback( + async (newSessionId: string) => { + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + try { + // Check if already on this session + if (newSessionId === session.id) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `ℹ️ Already using session ${newSessionId.slice(0, 8)}`, + timestamp: new Date(), + }, + ]); + return; + } + + // Track session switch analytics + capture('dexto_session_switched', { + source: 'cli', + fromSessionId: session.id || null, + toSessionId: newSessionId, + }); + + // Clear messages and session state before switching + setMessages([]); + setApproval(null); + setApprovalQueue([]); + setSession({ + id: newSessionId, + hasActiveSession: true, + modelName: session.modelName, + }); + + // Verify session exists + const sessionData = await agent.getSession(newSessionId); + if (!sessionData) { + throw new Error(`Session ${newSessionId} not found`); + } + + // Load session history + const history = await agent.getSessionHistory(newSessionId); + if (history && history.length > 0) { + const historyMessages = convertHistoryToUIMessages(history, newSessionId); + setMessages(historyMessages); + + // Extract user messages for input history (arrow up navigation) + const userInputHistory = history + .filter(isUserMessage) + .map((msg) => { + // Extract text content from user message + if (typeof msg.content === 'string') { + return msg.content; + } + // Handle array content (text parts) + if (Array.isArray(msg.content)) { + return msg.content + .filter( + (part): part is { type: 'text'; text: string } => + typeof part === 'object' && part.type === 'text' + ) + .map((part) => part.text) + .join('\n'); + } + return ''; + }) + .filter((text) => text.trim().length > 0); + + setInput((prev) => ({ + ...prev, + history: userInputHistory, + historyIndex: -1, + })); + } + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Switched to session ${newSessionId.slice(0, 8)}`, + timestamp: new Date(), + }, + ]); + + // Force Static component to re-render with the new history + refreshStatic?.(); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Failed to switch session: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + }, + [ + setUi, + setInput, + setMessages, + setApproval, + setApprovalQueue, + setSession, + agent, + session.id, + session.modelName, + buffer, + refreshStatic, + ] + ); + + // Handle slash command/prompt selection + const handlePromptSelect = useCallback( + async (prompt: PromptInfo) => { + // Use displayName for command text (user-friendly name without prefix) + const commandName = prompt.displayName || prompt.name; + const commandText = `/${commandName}`; + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + // Route prompts through InputContainer for streaming pipeline + if (onSubmitPromptCommand) { + await onSubmitPromptCommand(commandText); + return; + } + + // Fallback when callback not provided (shouldn't happen in normal usage) + // Show user message for the executed command + const userMessage = createUserMessage(commandText); + setMessages((prev) => [...prev, userMessage]); + + setUi((prev) => ({ ...prev, isProcessing: true, isCancelling: false })); + + const { CommandService } = await import('../services/CommandService.js'); + const commandService = new CommandService(); + + try { + // Use displayName to match the registered command name + const result = await commandService.executeCommand( + commandName, + [], + agent, + session.id || undefined + ); + + if (result.type === 'output' && result.output) { + const output = result.output; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('command'), + role: 'system', + content: output, + timestamp: new Date(), + }, + ]); + } + + if (result.type === 'styled' && result.styled) { + const { fallbackText, styledType, styledData } = result.styled; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('command'), + role: 'system', + content: fallbackText, + timestamp: new Date(), + styledType, + styledData, + }, + ]); + } + + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } + }, + [setUi, setInput, setMessages, agent, session.id, buffer, onSubmitPromptCommand] + ); + + // Handle loading command/prompt into input for editing (Tab key) + + const handleSystemCommandSelect = useCallback( + async (command: string) => { + // Check if this command has an interactive overlay + const { getCommandOverlayForSelect } = await import('../utils/commandOverlays.js'); + const overlay = getCommandOverlayForSelect(command); + if (overlay) { + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + setUi((prev) => ({ + ...prev, + activeOverlay: overlay, + mcpWizardServerType: null, + })); + return; + } + + const commandText = `/${command}`; + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + // Show user message for the executed command + const userMessage = createUserMessage(commandText); + setMessages((prev) => [...prev, userMessage]); + + setUi((prev) => ({ ...prev, isProcessing: true, isCancelling: false })); + + const { CommandService } = await import('../services/CommandService.js'); + const commandService = new CommandService(); + + try { + const result = await commandService.executeCommand( + command, + [], + agent, + session.id || undefined + ); + + if (result.type === 'output' && result.output) { + const output = result.output; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('command'), + role: 'system', + content: output, + timestamp: new Date(), + }, + ]); + } + + if (result.type === 'styled' && result.styled) { + const { fallbackText, styledType, styledData } = result.styled; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('command'), + role: 'system', + content: fallbackText, + timestamp: new Date(), + styledType, + styledData, + }, + ]); + } + + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } + }, + [setInput, setUi, setMessages, agent, session.id, buffer] + ); + + const handleLoadIntoInput = useCallback( + (text: string) => { + // Update both buffer (source of truth) and state + buffer.setText(text); + setInput((prev) => ({ ...prev, value: text })); + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + }, + [buffer, setInput, setUi] + ); + + // Handle resource selection + const handleResourceSelect = useCallback( + (resource: ResourceMetadata) => { + // Insert resource reference into input + const atIndex = input.value.lastIndexOf('@'); + if (atIndex >= 0) { + const before = input.value.slice(0, atIndex + 1); + const uriParts = resource.uri.split('/'); + const reference = + resource.name || uriParts[uriParts.length - 1] || resource.uri; + const newValue = `${before}${reference} `; + buffer.setText(newValue); + setInput((prev) => ({ ...prev, value: newValue })); + } + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + }, + [input.value, buffer, setInput, setUi] + ); + + const handleClose = useCallback(() => { + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + }, [setUi]); + + // Handle log level selection + const handleLogLevelSelect = useCallback( + (level: string) => { + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + // Set level on agent's logger (propagates to all child loggers via shared ref) + agent.logger.setLevel(level as LogLevel); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `📊 Log level set to: ${level}`, + timestamp: new Date(), + }, + ]); + }, + [setUi, setInput, setMessages, agent, buffer] + ); + + // Handle stream mode selection + const handleStreamSelect = useCallback( + (enabled: boolean) => { + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: enabled + ? '▶️ Streaming enabled - responses will appear as they are generated' + : '⏸️ Streaming disabled - responses will appear when complete', + timestamp: new Date(), + }, + ]); + }, + [setUi, setInput, setMessages, buffer] + ); + + // Handle MCP server list actions (select server or add new) + const handleMcpServerListAction = useCallback( + (action: McpServerListAction) => { + if (action.type === 'select-server') { + // Show server actions overlay + setUi((prev) => ({ + ...prev, + activeOverlay: 'mcp-server-actions', + selectedMcpServer: action.server, + })); + } else if (action.type === 'add-new') { + // Show add choice overlay + setUi((prev) => ({ + ...prev, + activeOverlay: 'mcp-add-choice', + })); + } + }, + [setUi] + ); + + // Handle MCP server actions (enable/disable/delete/back) + const handleMcpServerAction = useCallback( + async (action: McpServerAction) => { + const { server } = action; + + if (action.type === 'back') { + // Go back to server list + setUi((prev) => ({ + ...prev, + activeOverlay: 'mcp-server-list', + selectedMcpServer: null, + })); + return; + } + + // Close overlay and reset input for actual actions + setUi((prev) => ({ + ...prev, + activeOverlay: 'none', + selectedMcpServer: null, + mcpWizardServerType: null, + })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + if (action.type === 'enable' || action.type === 'disable') { + const newEnabled = action.type === 'enable'; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `${newEnabled ? '▶️' : '⏸️'} ${newEnabled ? 'Enabling' : 'Disabling'} ${server.name}...`, + timestamp: new Date(), + }, + ]); + + try { + // Enable or disable the server FIRST (before persisting) + // This ensures config only reflects successful state changes + if (newEnabled) { + try { + await agent.enableMcpServer(server.name); + } catch (connectError) { + // Connection failed - don't persist to config + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `⚠️ Failed to enable server: ${connectError instanceof Error ? connectError.message : String(connectError)}`, + timestamp: new Date(), + }, + ]); + return; + } + } else { + await agent.disableMcpServer(server.name); + } + + // Import persistence utilities + const { updateMcpServerField } = await import('@dexto/agent-management'); + + // Persist to config file AFTER successful enable/disable + await updateMcpServerField( + agent.getAgentFilePath(), + server.name, + 'enabled', + newEnabled + ); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ ${server.name} ${newEnabled ? 'enabled' : 'disabled'}`, + timestamp: new Date(), + }, + ]); + } catch (error) { + // Format error message with details if available + let errorMessage = error instanceof Error ? error.message : String(error); + if (error instanceof DextoValidationError && error.issues.length > 0) { + const issueDetails = error.issues + .map((i) => { + const path = i.path?.length ? `[${i.path.join('.')}] ` : ''; + return ` - ${path}${i.message}`; + }) + .join('\n'); + errorMessage = `Validation failed:\n${issueDetails}`; + } + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Failed to ${action.type} server: ${errorMessage}`, + timestamp: new Date(), + }, + ]); + } + } else if (action.type === 'delete') { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `🗑️ Deleting ${server.name}...`, + timestamp: new Date(), + }, + ]); + + try { + // Import persistence utilities + const { removeMcpServerFromConfig } = await import( + '@dexto/agent-management' + ); + + // Persist to config file using surgical removal + await removeMcpServerFromConfig(agent.getAgentFilePath(), server.name); + + // Also disconnect if connected + try { + await agent.removeMcpServer(server.name); + } catch { + // Ignore - server might not be connected + } + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Deleted ${server.name}`, + timestamp: new Date(), + }, + ]); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Failed to delete server: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + } + }, + [setUi, setInput, setMessages, agent, buffer] + ); + + // Handle MCP add choice (registry/custom/back) + const handleMcpAddChoice = useCallback( + (choice: McpAddChoiceType) => { + if (choice === 'back') { + // Go back to server list + setUi((prev) => ({ + ...prev, + activeOverlay: 'mcp-server-list', + })); + } else if (choice === 'registry') { + // Show registry selector (McpAddSelector) + setUi((prev) => ({ + ...prev, + activeOverlay: 'mcp-add-selector', + })); + } else if (choice === 'custom') { + // Show custom type selector + setUi((prev) => ({ + ...prev, + activeOverlay: 'mcp-custom-type-selector', + })); + } + }, + [setUi] + ); + + // Handle MCP add selection (presets only) + const handleMcpAddSelect = useCallback( + async (result: McpAddResult) => { + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + setUi((prev) => ({ ...prev, isProcessing: true, isCancelling: false })); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `🔌 Connecting to ${result.entry.name}...`, + timestamp: new Date(), + }, + ]); + + try { + const mcpConfig = result.entry.config as McpServerConfig; + await agent.addMcpServer(result.entry.id, mcpConfig); + + // Track MCP server connected analytics + capture('dexto_mcp_server_connected', { + source: 'cli', + serverName: result.entry.name, + transportType: mcpConfig.type, + }); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Connected to ${result.entry.name}`, + timestamp: new Date(), + }, + ]); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `❌ Failed to connect: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + }, + [setUi, setInput, setMessages, agent, buffer] + ); + + // Handle MCP custom type selection + const handleMcpCustomTypeSelect = useCallback( + (serverType: McpServerType) => { + setUi((prev) => ({ + ...prev, + mcpWizardServerType: serverType, + activeOverlay: 'mcp-custom-wizard', + })); + }, + [setUi] + ); + + // Handle MCP custom wizard completion + const handleMcpCustomWizardComplete = useCallback( + async (config: McpCustomConfig) => { + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + setUi((prev) => ({ ...prev, isProcessing: true, isCancelling: false })); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `🔌 Connecting to ${config.name}...`, + timestamp: new Date(), + }, + ]); + + try { + // Build the appropriate config based on server type + let serverConfig: McpServerConfig; + if (config.serverType === 'stdio') { + serverConfig = { + type: 'stdio', + command: config.command!, + args: config.args || [], + }; + } else if (config.serverType === 'http') { + serverConfig = { + type: 'http', + url: config.url!, + }; + } else { + // sse + serverConfig = { + type: 'sse', + url: config.url!, + }; + } + + await agent.addMcpServer(config.name, serverConfig); + + // Track MCP server connected analytics + capture('dexto_mcp_server_connected', { + source: 'cli', + serverName: config.name, + transportType: serverConfig.type, + }); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Connected to ${config.name}`, + timestamp: new Date(), + }, + ]); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `❌ Failed to connect: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + }, + [setUi, setInput, setMessages, agent, buffer] + ); + + // Handle plugin manager actions + const handlePluginManagerAction = useCallback( + (action: PluginAction) => { + if (action === 'list') { + setUi((prev) => ({ + ...prev, + activeOverlay: 'plugin-list', + })); + } else if (action === 'marketplace') { + setUi((prev) => ({ + ...prev, + activeOverlay: 'marketplace-browser', + })); + } + }, + [setUi] + ); + + // Handle plugin selection from plugin list + const handlePluginSelect = useCallback( + (plugin: ListedPlugin) => { + setSelectedPlugin(plugin); + setUi((prev) => ({ + ...prev, + activeOverlay: 'plugin-actions', + })); + }, + [setUi] + ); + + // Handle plugin actions (uninstall, back) + const handlePluginAction = useCallback( + async (action: PluginActionResult) => { + if (action.type === 'back') { + setSelectedPlugin(null); + setUi((prev) => ({ + ...prev, + activeOverlay: 'plugin-list', + })); + return; + } + + if (action.type === 'uninstall') { + setUi((prev) => ({ ...prev, activeOverlay: 'none', isProcessing: true })); + + try { + const { uninstallPlugin, reloadAgentConfigFromFile, enrichAgentConfig } = + await import('@dexto/agent-management'); + await uninstallPlugin(action.plugin.name); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `Plugin '${action.plugin.name}' has been uninstalled.`, + timestamp: new Date(), + }, + ]); + + // Refresh prompts to remove uninstalled plugin skills + try { + const newConfig = await reloadAgentConfigFromFile( + agent.getAgentFilePath() + ); + const enrichedConfig = enrichAgentConfig( + newConfig, + agent.getAgentFilePath() + ); + await agent.refreshPrompts(enrichedConfig.prompts); + } catch { + // Non-critical: prompts will refresh on next agent restart + } + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `Failed to uninstall plugin: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + + setSelectedPlugin(null); + setUi((prev) => ({ ...prev, isProcessing: false })); + } + }, + [setUi, setMessages, agent] + ); + + // Handle marketplace browser actions + const handleMarketplaceBrowserAction = useCallback( + async (action: MarketplaceBrowserAction) => { + if (action.type === 'add-marketplace') { + setUi((prev) => ({ ...prev, activeOverlay: 'marketplace-add' })); + } else if (action.type === 'plugin-installed') { + setUi((prev) => ({ ...prev, activeOverlay: 'none' })); + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Plugin '${action.pluginName}' installed from ${action.marketplace}`, + timestamp: new Date(), + }, + ]); + + // Refresh prompts to include new plugin skills + try { + const { reloadAgentConfigFromFile, enrichAgentConfig } = await import( + '@dexto/agent-management' + ); + const newConfig = await reloadAgentConfigFromFile(agent.getAgentFilePath()); + const enrichedConfig = enrichAgentConfig( + newConfig, + agent.getAgentFilePath() + ); + await agent.refreshPrompts(enrichedConfig.prompts); + } catch (error) { + // Non-critical: prompts will refresh on next agent restart + // Log but don't show error to user + } + } + }, + [setUi, setMessages, agent] + ); + + // Handle marketplace add completion + const handleMarketplaceAddComplete = useCallback( + (name: string, pluginCount: number) => { + setUi((prev) => ({ ...prev, activeOverlay: 'marketplace-browser' })); + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Marketplace '${name}' added (${pluginCount} plugins found)`, + timestamp: new Date(), + }, + ]); + }, + [setUi, setMessages] + ); + + // Handle session subcommand selection + const handleSessionSubcommandSelect = useCallback( + async (action: SessionAction) => { + if (action === 'switch') { + setInput((prev) => ({ ...prev, value: '/session switch' })); + setUi((prev) => ({ ...prev, activeOverlay: 'session-selector' })); + return; + } + + setUi((prev) => ({ ...prev, activeOverlay: 'none', mcpWizardServerType: null })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + setUi((prev) => ({ ...prev, isProcessing: true, isCancelling: false })); + + try { + const { CommandService } = await import('../services/CommandService.js'); + const commandService = new CommandService(); + const result = await commandService.executeCommand( + 'session', + [action], + agent, + session.id || undefined + ); + + if (result.type === 'output' && result.output) { + const output = result.output; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('command'), + role: 'system', + content: output, + timestamp: new Date(), + }, + ]); + } + if (result.type === 'styled' && result.styled) { + const { fallbackText, styledType, styledData } = result.styled; + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('command'), + role: 'system', + content: fallbackText, + timestamp: new Date(), + styledType, + styledData, + }, + ]); + } + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } + }, + [setInput, setUi, setMessages, agent, session.id, buffer] + ); + + // Handle prompt list actions (select/add/delete) + const handlePromptListAction = useCallback( + async (action: PromptListAction) => { + if (action.type === 'add-prompt') { + setUi((prev) => ({ + ...prev, + activeOverlay: 'prompt-add-choice', + })); + } else if (action.type === 'delete-prompt') { + setUi((prev) => ({ + ...prev, + activeOverlay: 'prompt-delete-selector', + })); + } else if (action.type === 'select-prompt') { + // Execute the prompt + const displayName = action.prompt.displayName || action.prompt.name; + const commandText = `/${displayName}`; + setUi((prev) => ({ + ...prev, + activeOverlay: 'none', + promptAddWizard: null, + })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + // Route through streaming pipeline + if (onSubmitPromptCommand) { + await onSubmitPromptCommand(commandText); + } + } + }, + [setUi, setInput, buffer, onSubmitPromptCommand] + ); + + // Handle prompt list load into input + const handlePromptLoadIntoInput = useCallback( + (text: string) => { + buffer.setText(text); + setInput((prev) => ({ ...prev, value: text })); + setUi((prev) => ({ + ...prev, + activeOverlay: 'none', + promptAddWizard: null, + })); + }, + [buffer, setInput, setUi] + ); + + // Handle prompt add choice (agent vs shared) + const handlePromptAddChoice = useCallback( + (choice: PromptAddChoiceResult) => { + if (choice === 'back') { + setUi((prev) => ({ + ...prev, + activeOverlay: 'prompt-list', + })); + } else { + setUi((prev) => ({ + ...prev, + activeOverlay: 'prompt-add-wizard', + promptAddWizard: { + scope: choice, + step: 'name', + name: '', + title: '', + description: '', + content: '', + }, + })); + } + }, + [setUi] + ); + + // Handle prompt add wizard completion + const handlePromptAddComplete = useCallback( + async (data: NewPromptData) => { + const scope = ui.promptAddWizard?.scope || 'agent'; + setUi((prev) => ({ + ...prev, + activeOverlay: 'none', + promptAddWizard: null, + })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `📝 Creating ${scope === 'shared' ? 'shared' : 'agent'} prompt "${data.name}"...`, + timestamp: new Date(), + }, + ]); + + try { + const { mkdir, writeFile } = await import('fs/promises'); + const { dirname, join } = await import('path'); + + // Validate prompt name to prevent path traversal + const SAFE_NAME_PATTERN = /^[a-z0-9][a-z0-9-_]*$/i; + if (!SAFE_NAME_PATTERN.test(data.name)) { + throw new Error( + `Invalid prompt name "${data.name}". Names must start with a letter or number and contain only letters, numbers, hyphens, and underscores.` + ); + } + + // Build frontmatter + const frontmatterLines = [ + '---', + `id: ${data.name}`, + data.title ? `title: "${data.title}"` : null, + data.description ? `description: "${data.description}"` : null, + data.argumentHint ? `argument-hint: ${data.argumentHint}` : null, + '---', + ].filter(Boolean); + + const fileContent = `${frontmatterLines.join('\n')}\n\n${data.content}\n`; + + let filePath: string; + + if (scope === 'shared') { + // Create in commands directory based on execution context + // Matches discovery logic in discoverCommandPrompts() + const { + getExecutionContext, + findDextoSourceRoot, + findDextoProjectRoot, + getDextoGlobalPath, + } = await import('@dexto/agent-management'); + + const context = getExecutionContext(); + let commandsDir: string; + + if (context === 'dexto-source') { + const isDevMode = process.env.DEXTO_DEV_MODE === 'true'; + if (isDevMode) { + const sourceRoot = findDextoSourceRoot(); + commandsDir = sourceRoot + ? join(sourceRoot, 'commands') + : getDextoGlobalPath('commands'); + } else { + commandsDir = getDextoGlobalPath('commands'); + } + } else if (context === 'dexto-project') { + const projectRoot = findDextoProjectRoot(); + commandsDir = projectRoot + ? join(projectRoot, 'commands') + : getDextoGlobalPath('commands'); + } else { + // global-cli + commandsDir = getDextoGlobalPath('commands'); + } + + filePath = join(commandsDir, `${data.name}.md`); + await mkdir(commandsDir, { recursive: true }); + await writeFile(filePath, fileContent, 'utf-8'); + + // Re-discover commands and refresh with enriched prompts + const { reloadAgentConfigFromFile, enrichAgentConfig } = await import( + '@dexto/agent-management' + ); + const newConfig = await reloadAgentConfigFromFile(agent.getAgentFilePath()); + const enrichedConfig = enrichAgentConfig( + newConfig, + agent.getAgentFilePath() + ); + await agent.refreshPrompts(enrichedConfig.prompts); + } else { + // Create in agent's prompts directory + const agentDir = dirname(agent.getAgentFilePath()); + const promptsDir = join(agentDir, 'prompts'); + filePath = join(promptsDir, `${data.name}.md`); + await mkdir(promptsDir, { recursive: true }); + await writeFile(filePath, fileContent, 'utf-8'); + + // Add file reference to agent config using surgical helper + const { + addPromptToAgentConfig, + reloadAgentConfigFromFile, + enrichAgentConfig, + } = await import('@dexto/agent-management'); + await addPromptToAgentConfig(agent.getAgentFilePath(), { + type: 'file', + file: `\${{dexto.agent_dir}}/prompts/${data.name}.md`, + }); + + // Reload config from disk, enrich to include discovered commands, then refresh + const newConfig = await reloadAgentConfigFromFile(agent.getAgentFilePath()); + const enrichedConfig = enrichAgentConfig( + newConfig, + agent.getAgentFilePath() + ); + await agent.refreshPrompts(enrichedConfig.prompts); + } + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Created prompt "${data.name}"\n📄 File: ${filePath}\n\nUse /${data.name} to run it.`, + timestamp: new Date(), + }, + ]); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Failed to create prompt: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + }, + [ui.promptAddWizard?.scope, setUi, setInput, setMessages, buffer, agent] + ); + + // Handle prompt delete + const handlePromptDelete = useCallback( + async (deletable: DeletablePrompt) => { + const displayName = deletable.prompt.displayName || deletable.prompt.name; + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `🗑️ Deleting prompt "${displayName}"...`, + timestamp: new Date(), + }, + ]); + + try { + const { deletePromptByMetadata, reloadAgentConfigFromFile, enrichAgentConfig } = + await import('@dexto/agent-management'); + + // Use the higher-level delete function that handles file + config + // Pass full metadata including originalId for inline prompt deletion + const promptMetadata = deletable.prompt.metadata as + | { filePath?: string; originalId?: string } + | undefined; + const result = await deletePromptByMetadata( + agent.getAgentFilePath(), + { + name: deletable.prompt.name, + metadata: { + filePath: deletable.filePath, + originalId: promptMetadata?.originalId, + }, + }, + { deleteFile: true } + ); + + if (!result.success) { + throw new Error(result.error || 'Failed to delete prompt'); + } + + // Reload config from disk, enrich to include discovered commands, then refresh + const newConfig = await reloadAgentConfigFromFile(agent.getAgentFilePath()); + const enrichedConfig = enrichAgentConfig(newConfig, agent.getAgentFilePath()); + await agent.refreshPrompts(enrichedConfig.prompts); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Deleted prompt "${displayName}"`, + timestamp: new Date(), + }, + ]); + + // Return to prompt list and refresh + setUi((prev) => ({ + ...prev, + activeOverlay: 'prompt-list', + })); + promptListRef.current?.refresh(); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Failed to delete prompt: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + + // Return to prompt list even on error + setUi((prev) => ({ + ...prev, + activeOverlay: 'prompt-list', + })); + } + }, + [setUi, setMessages, agent] + ); + + // Handle prompt add wizard close + const handlePromptAddWizardClose = useCallback(() => { + setUi((prev) => ({ + ...prev, + activeOverlay: 'prompt-add-choice', + promptAddWizard: null, + })); + }, [setUi]); + + // Handle prompt delete selector close + const handlePromptDeleteClose = useCallback(() => { + setUi((prev) => ({ + ...prev, + activeOverlay: 'prompt-list', + })); + // Refresh prompt list to show updated list + promptListRef.current?.refresh(); + }, [setUi]); + + // Handle prompt add choice close + const handlePromptAddChoiceClose = useCallback(() => { + setUi((prev) => ({ + ...prev, + activeOverlay: 'prompt-list', + })); + }, [setUi]); + + // State for current session title (for rename overlay) + const [currentSessionTitle, setCurrentSessionTitle] = useState<string | undefined>( + undefined + ); + + // Fetch current session title when rename overlay opens + React.useEffect(() => { + if (ui.activeOverlay === 'session-rename' && session.id) { + void agent.getSessionTitle(session.id).then(setCurrentSessionTitle); + } + }, [ui.activeOverlay, session.id, agent]); + + // Handle session rename + const handleSessionRename = useCallback( + async (newTitle: string) => { + if (!session.id) return; + + setUi((prev) => ({ ...prev, activeOverlay: 'none' })); + buffer.setText(''); + setInput((prev) => ({ ...prev, historyIndex: -1 })); + + try { + await agent.setSessionTitle(session.id, newTitle); + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: `✅ Session renamed to: ${newTitle}`, + timestamp: new Date(), + }, + ]); + } catch (error) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: `❌ Failed to rename session: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + }, + [session.id, setUi, setInput, setMessages, agent, buffer] + ); + + // Handle session rename close + const handleSessionRenameClose = useCallback(() => { + setUi((prev) => ({ ...prev, activeOverlay: 'none' })); + }, [setUi]); + + return ( + <> + {/* Approval prompt */} + {approval && ( + <ApprovalPrompt + ref={approvalRef} + approval={approval} + onApprove={handleApprove} + onDeny={handleDeny} + onCancel={handleCancelApproval} + /> + )} + + {/* Slash command autocomplete */} + {ui.activeOverlay === 'slash-autocomplete' && ( + <Box marginTop={1}> + <SlashCommandAutocomplete + ref={slashAutocompleteRef} + isVisible={true} + searchQuery={input.value} + onSelectPrompt={handlePromptSelect} + onSelectSystemCommand={handleSystemCommandSelect} + onLoadIntoInput={handleLoadIntoInput} + onSubmitRaw={onSubmitPromptCommand} + onClose={handleClose} + agent={agent} + /> + </Box> + )} + + {/* Resource autocomplete */} + {ui.activeOverlay === 'resource-autocomplete' && ( + <Box marginTop={1}> + <ResourceAutocomplete + ref={resourceAutocompleteRef} + isVisible={true} + searchQuery={input.value} + onSelectResource={handleResourceSelect} + onLoadIntoInput={handleLoadIntoInput} + onClose={handleClose} + agent={agent} + /> + </Box> + )} + + {/* Model selector */} + {ui.activeOverlay === 'model-selector' && ( + <Box marginTop={1}> + <ModelSelectorRefactored + ref={modelSelectorRef} + isVisible={true} + onSelectModel={handleModelSelect} + onClose={handleClose} + onAddCustomModel={handleAddCustomModel} + onEditCustomModel={handleEditCustomModel} + agent={agent} + /> + </Box> + )} + + {/* Session selector */} + {ui.activeOverlay === 'session-selector' && ( + <Box marginTop={1}> + <SessionSelectorRefactored + ref={sessionSelectorRef} + isVisible={true} + onSelectSession={handleSessionSelect} + onClose={handleClose} + agent={agent} + currentSessionId={session.id || undefined} + /> + </Box> + )} + + {/* Log level selector */} + {ui.activeOverlay === 'log-level-selector' && ( + <Box marginTop={1}> + <LogLevelSelector + ref={logLevelSelectorRef} + isVisible={true} + onSelect={handleLogLevelSelect} + onClose={handleClose} + agent={agent} + /> + </Box> + )} + + {/* Stream mode selector */} + {ui.activeOverlay === 'stream-selector' && ( + <Box marginTop={1}> + <StreamSelector + ref={streamSelectorRef} + isVisible={true} + onSelect={handleStreamSelect} + onClose={handleClose} + /> + </Box> + )} + + {/* Tool browser */} + {ui.activeOverlay === 'tool-browser' && ( + <Box marginTop={1}> + <ToolBrowser + ref={toolBrowserRef} + isVisible={true} + onClose={handleClose} + agent={agent} + /> + </Box> + )} + + {/* MCP server list (first screen) */} + {ui.activeOverlay === 'mcp-server-list' && ( + <Box marginTop={1}> + <McpServerList + ref={mcpServerListRef} + isVisible={true} + onAction={handleMcpServerListAction} + onClose={handleClose} + agent={agent} + /> + </Box> + )} + + {/* MCP server actions (enable/disable/delete) */} + {ui.activeOverlay === 'mcp-server-actions' && ui.selectedMcpServer && ( + <Box marginTop={1}> + <McpServerActions + ref={mcpServerActionsRef} + isVisible={true} + server={ui.selectedMcpServer} + onAction={handleMcpServerAction} + onClose={handleClose} + /> + </Box> + )} + + {/* MCP add choice (registry vs custom) */} + {ui.activeOverlay === 'mcp-add-choice' && ( + <Box marginTop={1}> + <McpAddChoice + ref={mcpAddChoiceRef} + isVisible={true} + onSelect={handleMcpAddChoice} + onClose={handleClose} + /> + </Box> + )} + + {/* MCP add selector (registry presets) */} + {ui.activeOverlay === 'mcp-add-selector' && ( + <Box marginTop={1}> + <McpAddSelector + ref={mcpAddSelectorRef} + isVisible={true} + onSelect={handleMcpAddSelect} + onClose={handleClose} + /> + </Box> + )} + + {/* MCP custom type selector */} + {ui.activeOverlay === 'mcp-custom-type-selector' && ( + <Box marginTop={1}> + <McpCustomTypeSelector + ref={mcpCustomTypeSelectorRef} + isVisible={true} + onSelect={handleMcpCustomTypeSelect} + onClose={handleClose} + /> + </Box> + )} + + {/* MCP custom wizard */} + {ui.activeOverlay === 'mcp-custom-wizard' && ui.mcpWizardServerType && ( + <McpCustomWizard + ref={mcpCustomWizardRef} + isVisible={true} + serverType={ui.mcpWizardServerType} + onComplete={handleMcpCustomWizardComplete} + onClose={handleClose} + /> + )} + + {/* Custom model wizard */} + {ui.activeOverlay === 'custom-model-wizard' && ( + <CustomModelWizard + ref={customModelWizardRef} + isVisible={true} + onComplete={handleCustomModelComplete} + onClose={() => { + setEditingModel(null); + handleClose(); + }} + initialModel={editingModel} + /> + )} + + {/* Plugin manager */} + {ui.activeOverlay === 'plugin-manager' && ( + <Box marginTop={1}> + <PluginManager + ref={pluginManagerRef} + isVisible={true} + onAction={handlePluginManagerAction} + onClose={handleClose} + /> + </Box> + )} + + {/* Plugin list */} + {ui.activeOverlay === 'plugin-list' && ( + <Box marginTop={1}> + <PluginList + ref={pluginListRef} + isVisible={true} + onPluginSelect={handlePluginSelect} + onClose={handleClose} + /> + </Box> + )} + + {/* Plugin actions */} + {ui.activeOverlay === 'plugin-actions' && ( + <Box marginTop={1}> + <PluginActions + ref={pluginActionsRef} + isVisible={true} + plugin={selectedPlugin} + onAction={handlePluginAction} + onClose={() => { + setSelectedPlugin(null); + setUi((prev) => ({ + ...prev, + activeOverlay: 'plugin-list', + })); + }} + /> + </Box> + )} + + {/* Marketplace browser */} + {ui.activeOverlay === 'marketplace-browser' && ( + <Box marginTop={1}> + <MarketplaceBrowser + ref={marketplaceBrowserRef} + isVisible={true} + onAction={handleMarketplaceBrowserAction} + onClose={handleClose} + /> + </Box> + )} + + {/* Marketplace add prompt */} + {ui.activeOverlay === 'marketplace-add' && ( + <Box marginTop={1}> + <MarketplaceAddPrompt + ref={marketplaceAddPromptRef} + isVisible={true} + onComplete={handleMarketplaceAddComplete} + onClose={() => + setUi((prev) => ({ ...prev, activeOverlay: 'marketplace-browser' })) + } + /> + </Box> + )} + + {/* Session subcommand selector */} + {ui.activeOverlay === 'session-subcommand-selector' && ( + <Box marginTop={1}> + <SessionSubcommandSelector + ref={sessionSubcommandSelectorRef} + isVisible={true} + onSelect={handleSessionSubcommandSelect} + onClose={handleClose} + /> + </Box> + )} + + {/* API key input */} + {ui.activeOverlay === 'api-key-input' && ui.pendingModelSwitch && ( + <ApiKeyInput + ref={apiKeyInputRef} + isVisible={true} + provider={ui.pendingModelSwitch.provider as LLMProvider} + onSaved={handleApiKeySaved} + onClose={handleApiKeyClose} + /> + )} + + {/* Search overlay */} + {ui.activeOverlay === 'search' && ( + <SearchOverlay + ref={searchOverlayRef} + isVisible={true} + agent={agent} + onClose={handleClose} + onSelectResult={handleSearchResultSelect} + /> + )} + + {/* Prompt list */} + {ui.activeOverlay === 'prompt-list' && ( + <Box marginTop={1}> + <PromptList + ref={promptListRef} + isVisible={true} + onAction={handlePromptListAction} + onLoadIntoInput={handlePromptLoadIntoInput} + onClose={handleClose} + agent={agent} + /> + </Box> + )} + + {/* Prompt add choice */} + {ui.activeOverlay === 'prompt-add-choice' && ( + <Box marginTop={1}> + <PromptAddChoice + ref={promptAddChoiceRef} + isVisible={true} + onSelect={handlePromptAddChoice} + onClose={handlePromptAddChoiceClose} + /> + </Box> + )} + + {/* Prompt add wizard */} + {ui.activeOverlay === 'prompt-add-wizard' && ui.promptAddWizard && ( + <PromptAddWizard + ref={promptAddWizardRef} + isVisible={true} + scope={ui.promptAddWizard.scope} + onComplete={handlePromptAddComplete} + onClose={handlePromptAddWizardClose} + /> + )} + + {/* Prompt delete selector */} + {ui.activeOverlay === 'prompt-delete-selector' && ( + <Box marginTop={1}> + <PromptDeleteSelector + ref={promptDeleteSelectorRef} + isVisible={true} + onDelete={handlePromptDelete} + onClose={handlePromptDeleteClose} + agent={agent} + /> + </Box> + )} + + {/* Session rename overlay */} + {ui.activeOverlay === 'session-rename' && ( + <SessionRenameOverlay + ref={sessionRenameRef} + isVisible={true} + currentTitle={currentSessionTitle} + onRename={handleSessionRename} + onClose={handleSessionRenameClose} + /> + )} + + {/* Context stats overlay */} + {ui.activeOverlay === 'context-stats' && session.id && ( + <Box marginTop={1}> + <ContextStatsOverlay + ref={contextStatsRef} + isVisible={true} + onClose={handleClose} + agent={agent} + sessionId={session.id} + /> + </Box> + )} + + {/* Export wizard overlay */} + {ui.activeOverlay === 'export-wizard' && ( + <ExportWizard + ref={exportWizardRef} + isVisible={true} + agent={agent} + sessionId={session.id} + onClose={handleClose} + /> + )} + </> + ); + } +); diff --git a/dexto/packages/cli/src/cli/ink-cli/containers/index.ts b/dexto/packages/cli/src/cli/ink-cli/containers/index.ts new file mode 100644 index 00000000..e697f2e3 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/containers/index.ts @@ -0,0 +1,6 @@ +/** + * Containers module exports + */ + +export { InputContainer, type InputContainerHandle } from './InputContainer.js'; +export { OverlayContainer } from './OverlayContainer.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/contexts/KeypressContext.tsx b/dexto/packages/cli/src/cli/ink-cli/contexts/KeypressContext.tsx new file mode 100644 index 00000000..b431537a --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/contexts/KeypressContext.tsx @@ -0,0 +1,521 @@ +/** + * KeypressContext + * + * Provides keyboard input handling by reading stdin directly. + * Filters out mouse events before broadcasting to subscribers. + * + * This replaces Ink's useInput hook with a custom implementation that: + * 1. Reads stdin directly for full control over escape sequence parsing + * 2. Filters out mouse events so they don't interfere with keyboard handlers + * 3. Handles paste buffering and backslash+enter detection + */ + +import { useStdin } from 'ink'; +import type React from 'react'; +import { createContext, useCallback, useContext, useEffect, useRef } from 'react'; +import { ESC, FOCUS_IN, FOCUS_OUT } from '../utils/input.js'; +import { parseMouseEvent } from '../utils/mouse.js'; + +export const BACKSLASH_ENTER_TIMEOUT = 5; +export const ESC_TIMEOUT = 50; +export const PASTE_TIMEOUT = 30_000; + +// Key name mapping for escape sequences +const KEY_INFO_MAP: Record<string, { name: string; shift?: boolean; ctrl?: boolean }> = { + '[200~': { name: 'paste-start' }, + '[201~': { name: 'paste-end' }, + '[[A': { name: 'f1' }, + '[[B': { name: 'f2' }, + '[[C': { name: 'f3' }, + '[[D': { name: 'f4' }, + '[[E': { name: 'f5' }, + '[1~': { name: 'home' }, + '[2~': { name: 'insert' }, + '[3~': { name: 'delete' }, + '[4~': { name: 'end' }, + '[5~': { name: 'pageup' }, + '[6~': { name: 'pagedown' }, + '[7~': { name: 'home' }, + '[8~': { name: 'end' }, + '[11~': { name: 'f1' }, + '[12~': { name: 'f2' }, + '[13~': { name: 'f3' }, + '[14~': { name: 'f4' }, + '[15~': { name: 'f5' }, + '[17~': { name: 'f6' }, + '[18~': { name: 'f7' }, + '[19~': { name: 'f8' }, + '[20~': { name: 'f9' }, + '[21~': { name: 'f10' }, + '[23~': { name: 'f11' }, + '[24~': { name: 'f12' }, + '[A': { name: 'up' }, + '[B': { name: 'down' }, + '[C': { name: 'right' }, + '[D': { name: 'left' }, + '[E': { name: 'clear' }, + '[F': { name: 'end' }, + '[H': { name: 'home' }, + '[P': { name: 'f1' }, + '[Q': { name: 'f2' }, + '[R': { name: 'f3' }, + '[S': { name: 'f4' }, + OA: { name: 'up' }, + OB: { name: 'down' }, + OC: { name: 'right' }, + OD: { name: 'left' }, + OE: { name: 'clear' }, + OF: { name: 'end' }, + OH: { name: 'home' }, + OP: { name: 'f1' }, + OQ: { name: 'f2' }, + OR: { name: 'f3' }, + OS: { name: 'f4' }, + '[[5~': { name: 'pageup' }, + '[[6~': { name: 'pagedown' }, + '[9u': { name: 'tab' }, + '[13u': { name: 'return' }, + '[27u': { name: 'escape' }, + '[127u': { name: 'backspace' }, + '[57414u': { name: 'return' }, // Numpad Enter + '[a': { name: 'up', shift: true }, + '[b': { name: 'down', shift: true }, + '[c': { name: 'right', shift: true }, + '[d': { name: 'left', shift: true }, + '[e': { name: 'clear', shift: true }, + '[2$': { name: 'insert', shift: true }, + '[3$': { name: 'delete', shift: true }, + '[5$': { name: 'pageup', shift: true }, + '[6$': { name: 'pagedown', shift: true }, + '[7$': { name: 'home', shift: true }, + '[8$': { name: 'end', shift: true }, + '[Z': { name: 'tab', shift: true }, + Oa: { name: 'up', ctrl: true }, + Ob: { name: 'down', ctrl: true }, + Oc: { name: 'right', ctrl: true }, + Od: { name: 'left', ctrl: true }, + Oe: { name: 'clear', ctrl: true }, + '[2^': { name: 'insert', ctrl: true }, + '[3^': { name: 'delete', ctrl: true }, + '[5^': { name: 'pageup', ctrl: true }, + '[6^': { name: 'pagedown', ctrl: true }, + '[7^': { name: 'home', ctrl: true }, + '[8^': { name: 'end', ctrl: true }, +}; + +// Mac Alt key character mapping +const MAC_ALT_KEY_CHARACTER_MAP: Record<string, string> = { + '\u222B': 'b', // "∫" back one word + '\u0192': 'f', // "ƒ" forward one word + '\u00B5': 'm', // "µ" toggle markup view +}; + +const kUTF16SurrogateThreshold = 0x10000; +function charLengthAt(str: string, i: number): number { + if (str.length <= i) return 1; + const code = str.codePointAt(i); + return code !== undefined && code >= kUTF16SurrogateThreshold ? 2 : 1; +} + +export interface Key { + name: string; + ctrl: boolean; + meta: boolean; + shift: boolean; + paste: boolean; + insertable: boolean; + sequence: string; +} + +export type KeypressHandler = (key: Key) => void; + +interface KeypressContextValue { + subscribe: (handler: KeypressHandler) => void; + unsubscribe: (handler: KeypressHandler) => void; +} + +const KeypressContext = createContext<KeypressContextValue | undefined>(undefined); + +export function useKeypressContext() { + const context = useContext(KeypressContext); + if (!context) { + throw new Error('useKeypressContext must be used within a KeypressProvider'); + } + return context; +} + +/** + * Filter out non-keyboard events (mouse, focus) before passing to handler + */ +function nonKeyboardEventFilter(keypressHandler: KeypressHandler): KeypressHandler { + return (key: Key) => { + if ( + !parseMouseEvent(key.sequence) && + key.sequence !== FOCUS_IN && + key.sequence !== FOCUS_OUT + ) { + keypressHandler(key); + } + }; +} + +/** + * Buffer backslash followed by enter as shift+enter (newline) + */ +function bufferBackslashEnter(keypressHandler: KeypressHandler): (key: Key | null) => void { + const bufferer = (function* (): Generator<void, void, Key | null> { + while (true) { + const key = yield; + + if (key == null) { + continue; + } else if (key.sequence !== '\\') { + keypressHandler(key); + continue; + } + + const timeoutId = setTimeout(() => bufferer.next(null), BACKSLASH_ENTER_TIMEOUT); + const nextKey = yield; + clearTimeout(timeoutId); + + if (nextKey === null) { + keypressHandler(key); + } else if (nextKey.name === 'return') { + keypressHandler({ + ...nextKey, + shift: true, + sequence: '\r', + }); + } else { + keypressHandler(key); + keypressHandler(nextKey); + } + } + })(); + + bufferer.next(); // Prime the generator + return (key: Key | null) => bufferer.next(key); +} + +/** + * Buffer paste events between paste-start and paste-end sequences + */ +function bufferPaste(keypressHandler: KeypressHandler): (key: Key | null) => void { + const bufferer = (function* (): Generator<void, void, Key | null> { + while (true) { + let key = yield; + + if (key === null) { + continue; + } else if (key.name !== 'paste-start') { + keypressHandler(key); + continue; + } + + let buffer = ''; + while (true) { + const timeoutId = setTimeout(() => bufferer.next(null), PASTE_TIMEOUT); + key = yield; + clearTimeout(timeoutId); + + if (key === null) { + // Paste timeout - emit what we have + break; + } + + if (key.name === 'paste-end') { + break; + } + buffer += key.sequence; + } + + if (buffer.length > 0) { + keypressHandler({ + name: '', + ctrl: false, + meta: false, + shift: false, + paste: true, + insertable: true, + sequence: buffer, + }); + } + } + })(); + bufferer.next(); // Prime the generator + return (key: Key | null) => bufferer.next(key); +} + +/** + * Create a data listener that parses raw stdin into keypress events + */ +function createDataListener(keypressHandler: KeypressHandler) { + const parser = emitKeys(keypressHandler); + parser.next(); // Prime the generator + + let timeoutId: NodeJS.Timeout; + return (data: string) => { + clearTimeout(timeoutId); + for (const char of data) { + parser.next(char); + } + if (data.length !== 0) { + timeoutId = setTimeout(() => parser.next(''), ESC_TIMEOUT); + } + }; +} + +/** + * Generator that translates raw keypress characters into key events. + * Buffers escape sequences until complete or timeout. + */ +function* emitKeys(keypressHandler: KeypressHandler): Generator<void, void, string> { + while (true) { + let ch = yield; + let sequence = ch; + let escaped = false; + + let name = undefined; + let ctrl = false; + let meta = false; + let shift = false; + let code = undefined; + let insertable = false; + + if (ch === ESC) { + escaped = true; + ch = yield; + sequence += ch; + + if (ch === ESC) { + ch = yield; + sequence += ch; + } + } + + if (escaped && (ch === 'O' || ch === '[')) { + // ANSI escape sequence + code = ch; + let modifier = 0; + + if (ch === 'O') { + // ESC O letter or ESC O modifier letter + ch = yield; + sequence += ch; + + if (ch >= '0' && ch <= '9') { + modifier = parseInt(ch, 10) - 1; + ch = yield; + sequence += ch; + } + + code += ch; + } else if (ch === '[') { + // ESC [ sequences + ch = yield; + sequence += ch; + + if (ch === '[') { + code += ch; + ch = yield; + sequence += ch; + } + + const cmdStart = sequence.length - 1; + + // Collect digits + while (ch >= '0' && ch <= '9') { + ch = yield; + sequence += ch; + } + + // Handle modifiers + if (ch === ';') { + while (ch === ';') { + ch = yield; + sequence += ch; + + while (ch >= '0' && ch <= '9') { + ch = yield; + sequence += ch; + } + } + } else if (ch === '<') { + // SGR mouse mode - consume until m/M + ch = yield; + sequence += ch; + while (ch === '' || ch === ';' || (ch >= '0' && ch <= '9')) { + ch = yield; + sequence += ch; + } + } else if (ch === 'M') { + // X11 mouse mode - 3 bytes after M + ch = yield; + sequence += ch; + ch = yield; + sequence += ch; + ch = yield; + sequence += ch; + } + + const cmd = sequence.slice(cmdStart); + let match; + + if ((match = /^(\d+)(?:;(\d+))?(?:;(\d+))?([~^$u])$/.exec(cmd))) { + if (match[1] === '27' && match[3] && match[4] === '~') { + // modifyOtherKeys format + code += match[3] + 'u'; + modifier = parseInt(match[2] ?? '1', 10) - 1; + } else { + code += (match[1] ?? '') + (match[4] ?? ''); + modifier = parseInt(match[2] ?? '1', 10) - 1; + } + } else if ((match = /^(\d+)?(?:;(\d+))?([A-Za-z])$/.exec(cmd))) { + code += match[3] ?? ''; + modifier = parseInt(match[2] ?? match[1] ?? '1', 10) - 1; + } else { + code += cmd; + } + } + + // Parse modifiers (xterm encoding: shift=1, meta/alt=2, ctrl=4) + ctrl = !!(modifier & 4); + meta = !!(modifier & 2); + shift = !!(modifier & 1); + + const keyInfo = KEY_INFO_MAP[code]; + if (keyInfo) { + name = keyInfo.name; + if (keyInfo.shift) shift = true; + if (keyInfo.ctrl) ctrl = true; + } else { + name = 'undefined'; + if ((ctrl || meta) && (code.endsWith('u') || code.endsWith('~'))) { + const codeNumber = parseInt(code.slice(1, -1), 10); + if (codeNumber >= 'a'.charCodeAt(0) && codeNumber <= 'z'.charCodeAt(0)) { + name = String.fromCharCode(codeNumber); + } + } + } + } else if (ch === '\r') { + name = 'return'; + meta = escaped; + } else if (ch === '\n') { + name = 'enter'; + meta = escaped; + } else if (ch === '\t') { + name = 'tab'; + meta = escaped; + } else if (ch === '\b' || ch === '\x7f') { + name = 'backspace'; + meta = escaped; + } else if (ch === ESC) { + name = 'escape'; + meta = escaped; + } else if (ch === ' ') { + name = 'space'; + meta = escaped; + insertable = true; + } else if (!escaped && ch <= '\x1a') { + // ctrl+letter + name = String.fromCharCode(ch.charCodeAt(0) + 'a'.charCodeAt(0) - 1); + ctrl = true; + } else if (/^[0-9A-Za-z]$/.exec(ch) !== null) { + name = ch.toLowerCase(); + shift = /^[A-Z]$/.exec(ch) !== null; + meta = escaped; + insertable = true; + } else if (MAC_ALT_KEY_CHARACTER_MAP[ch] && process.platform === 'darwin') { + name = MAC_ALT_KEY_CHARACTER_MAP[ch]; + meta = true; + } else if (sequence === `${ESC}${ESC}`) { + name = 'escape'; + meta = true; + + // Emit first escape key + keypressHandler({ + name: 'escape', + ctrl, + meta, + shift, + paste: false, + insertable: false, + sequence: ESC, + }); + } else if (escaped) { + name = ch.length ? undefined : 'escape'; + meta = true; + } else { + // Any other character is printable + insertable = true; + } + + if ( + (sequence.length !== 0 && (name !== undefined || escaped)) || + charLengthAt(sequence, 0) === sequence.length + ) { + keypressHandler({ + name: name || '', + ctrl, + meta, + shift, + paste: false, + insertable, + sequence, + }); + } + } +} + +export interface KeypressProviderProps { + children: React.ReactNode; +} + +export function KeypressProvider({ children }: KeypressProviderProps) { + const { stdin, setRawMode } = useStdin(); + + const subscribers = useRef<Set<KeypressHandler>>(new Set()).current; + const subscribe = useCallback( + (handler: KeypressHandler) => subscribers.add(handler), + [subscribers] + ); + const unsubscribe = useCallback( + (handler: KeypressHandler) => subscribers.delete(handler), + [subscribers] + ); + const broadcast = useCallback( + (key: Key) => subscribers.forEach((handler) => handler(key)), + [subscribers] + ); + + useEffect(() => { + const wasRaw = stdin.isRaw; + if (wasRaw === false) { + setRawMode(true); + } + + process.stdin.setEncoding('utf8'); + + // Build the processing pipeline: + // dataListener -> pasteBufferer -> backslashBufferer -> mouseFilterer -> broadcast + const mouseFilterer = nonKeyboardEventFilter(broadcast); + const backslashBufferer = bufferBackslashEnter(mouseFilterer); + const pasteBufferer = bufferPaste(backslashBufferer); + const dataListener = createDataListener(pasteBufferer); + + stdin.on('data', dataListener); + + return () => { + stdin.removeListener('data', dataListener); + if (wasRaw === false) { + setRawMode(false); + } + }; + }, [stdin, setRawMode, broadcast]); + + return ( + <KeypressContext.Provider value={{ subscribe, unsubscribe }}> + {children} + </KeypressContext.Provider> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/contexts/MouseContext.tsx b/dexto/packages/cli/src/cli/ink-cli/contexts/MouseContext.tsx new file mode 100644 index 00000000..37df7bae --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/contexts/MouseContext.tsx @@ -0,0 +1,154 @@ +/** + * MouseContext + * + * Provides mouse event handling by reading stdin directly. + * Runs in parallel with KeypressProvider - each reads stdin independently. + */ + +import { useStdin } from 'ink'; +import type React from 'react'; +import { createContext, useCallback, useContext, useEffect, useRef } from 'react'; +import { ESC } from '../utils/input.js'; +import { + parseMouseEvent, + isIncompleteMouseSequence, + enableMouseEvents, + disableMouseEvents, + type MouseEvent, + type MouseEventName, + type MouseHandler, +} from '../utils/mouse.js'; + +export type { MouseEvent, MouseEventName, MouseHandler }; + +const MAX_MOUSE_BUFFER_SIZE = 4096; + +interface MouseContextValue { + subscribe: (handler: MouseHandler) => void; + unsubscribe: (handler: MouseHandler) => void; +} + +const MouseContext = createContext<MouseContextValue | undefined>(undefined); + +export function useMouseContext() { + const context = useContext(MouseContext); + if (!context) { + throw new Error('useMouseContext must be used within a MouseProvider'); + } + return context; +} + +/** + * Hook to subscribe to mouse events + */ +export function useMouse(handler: MouseHandler, { isActive = true } = {}) { + const { subscribe, unsubscribe } = useMouseContext(); + + useEffect(() => { + if (!isActive) { + return; + } + + subscribe(handler); + return () => unsubscribe(handler); + }, [isActive, handler, subscribe, unsubscribe]); +} + +export interface MouseProviderProps { + children: React.ReactNode; + /** Whether mouse events are enabled (sends terminal escape codes) */ + mouseEventsEnabled?: boolean; +} + +export function MouseProvider({ children, mouseEventsEnabled = true }: MouseProviderProps) { + const { stdin } = useStdin(); + const subscribers = useRef<Set<MouseHandler>>(new Set()).current; + + const subscribe = useCallback( + (handler: MouseHandler) => { + subscribers.add(handler); + }, + [subscribers] + ); + + const unsubscribe = useCallback( + (handler: MouseHandler) => { + subscribers.delete(handler); + }, + [subscribers] + ); + + // Enable/disable mouse events in terminal + useEffect(() => { + if (!mouseEventsEnabled) { + return; + } + + enableMouseEvents(); + return () => { + disableMouseEvents(); + }; + }, [mouseEventsEnabled]); + + // Listen to stdin for mouse events + useEffect(() => { + if (!mouseEventsEnabled) { + return; + } + + let mouseBuffer = ''; + + const broadcast = (event: MouseEvent) => { + for (const handler of subscribers) { + // Handler can return true to indicate it handled the event + if (handler(event) === true) { + break; + } + } + }; + + const handleData = (data: Buffer | string) => { + mouseBuffer += typeof data === 'string' ? data : data.toString('utf-8'); + + // Safety cap to prevent infinite buffer growth + if (mouseBuffer.length > MAX_MOUSE_BUFFER_SIZE) { + mouseBuffer = mouseBuffer.slice(-MAX_MOUSE_BUFFER_SIZE); + } + + while (mouseBuffer.length > 0) { + const parsed = parseMouseEvent(mouseBuffer); + + if (parsed) { + broadcast(parsed.event); + mouseBuffer = mouseBuffer.slice(parsed.length); + continue; + } + + if (isIncompleteMouseSequence(mouseBuffer)) { + break; // Wait for more data + } + + // Not a valid sequence at start, and not waiting for more data. + // Discard garbage until next possible sequence start. + const nextEsc = mouseBuffer.indexOf(ESC, 1); + if (nextEsc !== -1) { + mouseBuffer = mouseBuffer.slice(nextEsc); + // Loop continues to try parsing at new location + } else { + mouseBuffer = ''; + break; + } + } + }; + + stdin.on('data', handleData); + + return () => { + stdin.removeListener('data', handleData); + }; + }, [stdin, mouseEventsEnabled, subscribers]); + + return ( + <MouseContext.Provider value={{ subscribe, unsubscribe }}>{children}</MouseContext.Provider> + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/contexts/ScrollProvider.tsx b/dexto/packages/cli/src/cli/ink-cli/contexts/ScrollProvider.tsx new file mode 100644 index 00000000..3b2e0a0e --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/contexts/ScrollProvider.tsx @@ -0,0 +1,257 @@ +/** + * ScrollProvider + * + * Manages scrollable components and handles mouse scroll events. + * Components register themselves as scrollable, and the provider + * routes scroll events to the appropriate component. + */ + +import type React from 'react'; +import { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { getBoundingBox, type DOMElement } from 'ink'; +import { useMouse, type MouseEvent } from './MouseContext.js'; + +export interface ScrollState { + scrollTop: number; + scrollHeight: number; + innerHeight: number; +} + +export interface ScrollableEntry { + id: string; + ref: React.RefObject<DOMElement | null>; + getScrollState: () => ScrollState; + scrollBy: (delta: number) => void; + scrollTo?: (scrollTop: number, duration?: number) => void; + hasFocus: () => boolean; +} + +interface ScrollContextType { + register: (entry: ScrollableEntry) => void; + unregister: (id: string) => void; +} + +const ScrollContext = createContext<ScrollContextType | null>(null); + +/** + * Find scrollable components under the mouse cursor + */ +const findScrollableCandidates = ( + mouseEvent: MouseEvent, + scrollables: Map<string, ScrollableEntry> +) => { + const candidates: Array<ScrollableEntry & { area: number }> = []; + + for (const entry of scrollables.values()) { + if (!entry.ref.current || !entry.hasFocus()) { + continue; + } + + const boundingBox = getBoundingBox(entry.ref.current); + if (!boundingBox) continue; + + const { x, y, width, height } = boundingBox; + + // Check if mouse is inside this component + // Add 1 to width to include scrollbar column + const isInside = + mouseEvent.col >= x && + mouseEvent.col < x + width + 1 && + mouseEvent.row >= y && + mouseEvent.row < y + height; + + if (isInside) { + candidates.push({ ...entry, area: width * height }); + } + } + + // Sort by smallest area first (innermost component) + candidates.sort((a, b) => a.area - b.area); + return candidates; +}; + +export interface ScrollProviderProps { + children: React.ReactNode; + /** Callback when user attempts to select text (drag without Option key) */ + onSelectionAttempt?: () => void; +} + +/** + * Hook to detect text selection attempts (click + drag) + * Returns a handler to process mouse events for drag detection + */ +function useDragDetection(onSelectionAttempt?: () => void) { + const isLeftButtonDownRef = useRef(false); + const selectionHintShownRef = useRef(false); + + const handleDragDetection = useCallback( + (event: MouseEvent) => { + if (event.name === 'left-press') { + isLeftButtonDownRef.current = true; + selectionHintShownRef.current = false; + } else if (event.name === 'left-release') { + isLeftButtonDownRef.current = false; + } else if (event.name === 'move' && isLeftButtonDownRef.current) { + // User is dragging (trying to select text) + // Only show hint once per drag operation + if (!selectionHintShownRef.current && onSelectionAttempt) { + selectionHintShownRef.current = true; + onSelectionAttempt(); + } + } + }, + [onSelectionAttempt] + ); + + return handleDragDetection; +} + +export const ScrollProvider: React.FC<ScrollProviderProps> = ({ children, onSelectionAttempt }) => { + const [scrollables, setScrollables] = useState(new Map<string, ScrollableEntry>()); + + // Drag detection for selection hint + const handleDragDetection = useDragDetection(onSelectionAttempt); + + const register = useCallback((entry: ScrollableEntry) => { + setScrollables((prev) => new Map(prev).set(entry.id, entry)); + }, []); + + const unregister = useCallback((id: string) => { + setScrollables((prev) => { + const next = new Map(prev); + next.delete(id); + return next; + }); + }, []); + + const scrollablesRef = useRef(scrollables); + useEffect(() => { + scrollablesRef.current = scrollables; + }, [scrollables]); + + // Batch scroll events to prevent jitter + const pendingScrollsRef = useRef(new Map<string, number>()); + const flushScheduledRef = useRef(false); + + const scheduleFlush = useCallback(() => { + if (!flushScheduledRef.current) { + flushScheduledRef.current = true; + setTimeout(() => { + flushScheduledRef.current = false; + for (const [id, delta] of pendingScrollsRef.current.entries()) { + const entry = scrollablesRef.current.get(id); + if (entry) { + entry.scrollBy(delta); + } + } + pendingScrollsRef.current.clear(); + }, 0); + } + }, []); + + const handleScroll = useCallback( + (direction: 'up' | 'down', mouseEvent: MouseEvent): boolean => { + const delta = direction === 'up' ? -1 : 1; + const candidates = findScrollableCandidates(mouseEvent, scrollablesRef.current); + + for (const candidate of candidates) { + const { scrollTop, scrollHeight, innerHeight } = candidate.getScrollState(); + const pendingDelta = pendingScrollsRef.current.get(candidate.id) || 0; + const effectiveScrollTop = scrollTop + pendingDelta; + + // Epsilon for floating point comparison + const canScrollUp = effectiveScrollTop > 0.001; + const canScrollDown = effectiveScrollTop < scrollHeight - innerHeight - 0.001; + + if (direction === 'up' && canScrollUp) { + pendingScrollsRef.current.set(candidate.id, pendingDelta + delta); + scheduleFlush(); + return true; + } + + if (direction === 'down' && canScrollDown) { + pendingScrollsRef.current.set(candidate.id, pendingDelta + delta); + scheduleFlush(); + return true; + } + } + return false; + }, + [scheduleFlush] + ); + + // Subscribe to mouse events + useMouse( + useCallback( + (event: MouseEvent) => { + // Handle scroll events + if (event.name === 'scroll-up') { + return handleScroll('up', event); + } else if (event.name === 'scroll-down') { + return handleScroll('down', event); + } + + // Detect selection attempts (click + drag) + handleDragDetection(event); + + return false; + }, + [handleScroll, handleDragDetection] + ), + { isActive: true } + ); + + const contextValue = useMemo(() => ({ register, unregister }), [register, unregister]); + + return <ScrollContext.Provider value={contextValue}>{children}</ScrollContext.Provider>; +}; + +let nextId = 0; + +/** + * Hook to register a component as scrollable + */ +export const useScrollable = (entry: Omit<ScrollableEntry, 'id'>, isActive: boolean) => { + const context = useContext(ScrollContext); + if (!context) { + throw new Error('useScrollable must be used within a ScrollProvider'); + } + + const [id] = useState(() => `scrollable-${nextId++}`); + + // Store entry in ref to avoid re-registration on every render + // when callers don't memoize the entry object + const entryRef = useRef(entry); + entryRef.current = entry; + + useEffect(() => { + if (isActive) { + // Build entry object, conditionally including scrollTo to satisfy exactOptionalPropertyTypes + const registrationEntry: ScrollableEntry = { + id, + ref: entryRef.current.ref, + getScrollState: () => entryRef.current.getScrollState(), + scrollBy: (delta) => entryRef.current.scrollBy(delta), + hasFocus: () => entryRef.current.hasFocus(), + }; + if (entryRef.current.scrollTo) { + registrationEntry.scrollTo = (scrollTop: number, duration?: number) => { + entryRef.current.scrollTo?.(scrollTop, duration); + }; + } + context.register(registrationEntry); + return () => { + context.unregister(id); + }; + } + return; + }, [context, id, isActive]); +}; diff --git a/dexto/packages/cli/src/cli/ink-cli/contexts/SoundContext.tsx b/dexto/packages/cli/src/cli/ink-cli/contexts/SoundContext.tsx new file mode 100644 index 00000000..249a8bd7 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/contexts/SoundContext.tsx @@ -0,0 +1,31 @@ +/** + * SoundContext - Provides sound notification service to components + * + * Initialized at CLI startup based on user preferences. + * Components can use useSoundService() to access the service. + */ + +import React, { createContext, useContext, type ReactNode } from 'react'; +import type { SoundNotificationService } from '../utils/soundNotification.js'; + +const SoundContext = createContext<SoundNotificationService | null>(null); + +interface SoundProviderProps { + soundService: SoundNotificationService | null; + children: ReactNode; +} + +/** + * Provider component for sound notification service + */ +export function SoundProvider({ soundService, children }: SoundProviderProps) { + return <SoundContext.Provider value={soundService}>{children}</SoundContext.Provider>; +} + +/** + * Hook to access the sound notification service + * Returns null if sounds are not configured + */ +export function useSoundService(): SoundNotificationService | null { + return useContext(SoundContext); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/contexts/index.ts b/dexto/packages/cli/src/cli/ink-cli/contexts/index.ts new file mode 100644 index 00000000..dcf662a8 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/contexts/index.ts @@ -0,0 +1,30 @@ +/** + * Context providers for ink-cli + */ + +export { + KeypressProvider, + useKeypressContext, + type Key, + type KeypressHandler, + type KeypressProviderProps, +} from './KeypressContext.js'; + +export { + MouseProvider, + useMouseContext, + useMouse, + type MouseEvent, + type MouseEventName, + type MouseHandler, + type MouseProviderProps, +} from './MouseContext.js'; + +export { + ScrollProvider, + useScrollable, + type ScrollState, + type ScrollableEntry, +} from './ScrollProvider.js'; + +export { SoundProvider, useSoundService } from './SoundContext.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/index.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/index.ts new file mode 100644 index 00000000..d9e4d0eb --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/index.ts @@ -0,0 +1,35 @@ +/** + * Custom hooks module exports + */ + +export { useAgentEvents } from './useAgentEvents.js'; +export { useKeyboardShortcuts } from './useKeyboardShortcuts.js'; +export { + useInputOrchestrator, + createApprovalInputHandler, + createSelectorInputHandler, + createAutocompleteInputHandler, + createMainInputHandler, + type InputHandler, + type InputHandlers, + type UseInputOrchestratorProps, + type ApprovalHandlerProps, + type SelectorHandlerProps, + type AutocompleteHandlerProps, + type MainInputHandlerProps, +} from './useInputOrchestrator.js'; +export { useKeypress } from './useKeypress.js'; +export { useTerminalSize, type TerminalSize } from './useTerminalSize.js'; +export { + useCLIState, + type UseCLIStateProps, + type CLIStateReturn, + type UIState, + type InputState, + type SessionState, +} from './useCLIState.js'; +export type { Key } from './useInputOrchestrator.js'; +// useMouseScroll removed - mouse events now handled by MouseProvider/ScrollProvider +// useInputHistory removed - history navigation now handled by TextBufferInput +// useOverlayManager removed - overlay management now done via dispatch directly +// useSessionSync removed - sessionId now managed in state directly diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useAgentEvents.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useAgentEvents.ts new file mode 100644 index 00000000..cedfe31e --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useAgentEvents.ts @@ -0,0 +1,455 @@ +/** + * Hook for managing agent event bus subscriptions for NON-STREAMING events. + * + * Streaming events (llm:thinking, llm:chunk, llm:response, llm:tool-call, llm:tool-result, + * llm:error, run:complete, message:dequeued) are handled via agent.stream() iterator in processStream. + * + * This hook handles: + * - approval:request - Tool/command confirmation requests + * - llm:switched - Model change notifications + * - session:reset - Conversation reset + * - session:created - New session creation (e.g., from /clear) + * - context:compacting - Context compaction starting (from /compact or auto) + * - context:compacted - Context compaction finished + * - message:queued - New message added to queue + * - message:removed - Message removed from queue (e.g., up arrow edit) + * - run:invoke - External trigger (scheduler, A2A, API) starting a run + * - run:complete (for external triggers) - External run finished + * + * Uses AbortController pattern for cleanup. + */ + +import type React from 'react'; +import { useEffect, useRef } from 'react'; +import { setMaxListeners } from 'events'; +import { + getModelDisplayName, + type DextoAgent, + type QueuedMessage, + type ContentPart, +} from '@dexto/core'; +import type { Message, UIState, SessionState, InputState } from '../state/types.js'; +import type { ApprovalRequest } from '../components/ApprovalPrompt.js'; +import { generateMessageId } from '../utils/idGenerator.js'; +import type { TextBuffer } from '../components/shared/text-buffer.js'; + +interface UseAgentEventsProps { + agent: DextoAgent; + setMessages: React.Dispatch<React.SetStateAction<Message[]>>; + setPendingMessages: React.Dispatch<React.SetStateAction<Message[]>>; + setUi: React.Dispatch<React.SetStateAction<UIState>>; + setSession: React.Dispatch<React.SetStateAction<SessionState>>; + setInput: React.Dispatch<React.SetStateAction<InputState>>; + setApproval: React.Dispatch<React.SetStateAction<ApprovalRequest | null>>; + setApprovalQueue: React.Dispatch<React.SetStateAction<ApprovalRequest[]>>; + setQueuedMessages: React.Dispatch<React.SetStateAction<QueuedMessage[]>>; + /** Current session ID for filtering events */ + currentSessionId: string | null; + /** Text buffer for input (source of truth) - needed to clear on session reset */ + buffer: TextBuffer; +} + +/** + * Extract text content from ContentPart array + */ +function extractTextContent(content: ContentPart[]): string { + return content + .filter((part): part is { type: 'text'; text: string } => part.type === 'text') + .map((part) => part.text) + .join('\n'); +} + +/** + * Subscribes to agent event bus for non-streaming events only. + * Streaming events are handled via agent.stream() iterator. + * + * Also handles external triggers (run:invoke) from scheduler, A2A, etc. + */ +export function useAgentEvents({ + agent, + setMessages, + setPendingMessages, + setUi, + setSession, + setInput, + setApproval, + setApprovalQueue, + setQueuedMessages, + currentSessionId, + buffer, +}: UseAgentEventsProps): void { + // Track if an external trigger is active (scheduler, A2A, etc.) + const externalTriggerRef = useRef<{ + active: boolean; + sessionId: string | null; + messageId: string | null; + }>({ active: false, sessionId: null, messageId: null }); + + useEffect(() => { + const bus = agent.agentEventBus; + const controller = new AbortController(); + const { signal } = controller; + + // Increase listener limit for safety (added more for external trigger events) + setMaxListeners(25, signal); + + // NOTE: approval:request is now handled in processStream (via iterator) for proper + // event ordering. Direct bus subscription here caused a race condition where + // approval UI showed before text messages were added. + + // Handle model switch + bus.on( + 'llm:switched', + (payload) => { + if (payload.newConfig?.model) { + setSession((prev) => ({ + ...prev, + modelName: getModelDisplayName( + payload.newConfig.model, + payload.newConfig.provider + ), + })); + } + }, + { signal } + ); + + // Handle conversation reset + bus.on( + 'session:reset', + () => { + setMessages([]); + setApproval(null); + setApprovalQueue([]); + setQueuedMessages([]); + setUi((prev) => ({ ...prev, activeOverlay: 'none' })); + }, + { signal } + ); + + // Handle session creation (e.g., from /new command) + bus.on( + 'session:created', + (payload) => { + if (payload.switchTo) { + // Clear the terminal screen for a fresh start (only in TTY environments) + // \x1B[2J clears visible screen, \x1B[3J clears scrollback, \x1B[H moves cursor to top + if (process.stdout.isTTY) { + process.stdout.write('\x1B[2J\x1B[3J\x1B[H'); + } + + // Clear all message state + setMessages([]); + setPendingMessages([]); + setApproval(null); + setApprovalQueue([]); + setQueuedMessages([]); + + // Reset input state including history (up/down arrow) and Ctrl+R search state + // Clear TextBuffer first (source of truth), then sync React state + buffer.setText(''); + setInput((prev) => ({ + ...prev, + value: '', + history: [], + historyIndex: -1, + draftBeforeHistory: '', + images: [], + pastedBlocks: [], + pasteCounter: 0, + })); + + if (payload.sessionId === null) { + setSession((prev) => ({ ...prev, id: null, hasActiveSession: false })); + } else { + setSession((prev) => ({ + ...prev, + id: payload.sessionId, + hasActiveSession: true, + })); + } + + // Reset UI state including history search + setUi((prev) => ({ + ...prev, + activeOverlay: 'none', + historySearch: { + isActive: false, + query: '', + matchIndex: 0, + originalInput: '', + lastMatch: '', + }, + })); + } + }, + { signal } + ); + + // Handle context cleared (from /clear command) + // Keep messages visible for user reference - only context sent to LLM is cleared + // Just clean up any pending approvals/overlays/queued messages + bus.on( + 'context:cleared', + () => { + setApproval(null); + setApprovalQueue([]); + setQueuedMessages([]); + setUi((prev) => ({ ...prev, activeOverlay: 'none' })); + }, + { signal } + ); + + // Handle context compacting (from /compact command or auto-compaction) + // Single source of truth - handles both manual /compact and auto-compaction during streaming + bus.on( + 'context:compacting', + (payload) => { + if (payload.sessionId !== currentSessionId) return; + setUi((prev) => ({ ...prev, isCompacting: true })); + }, + { signal } + ); + + // Handle context compacted + // Single source of truth - shows notification for all compaction (manual and auto) + bus.on( + 'context:compacted', + (payload) => { + if (payload.sessionId !== currentSessionId) return; + setUi((prev) => ({ ...prev, isCompacting: false })); + + const reductionPercent = + payload.originalTokens > 0 + ? Math.round( + ((payload.originalTokens - payload.compactedTokens) / + payload.originalTokens) * + 100 + ) + : 0; + + const compactionContent = + `📦 Context compacted\n` + + ` ${payload.originalMessages} messages → ${payload.compactedMessages} messages\n` + + ` ~${payload.originalTokens.toLocaleString()} → ~${payload.compactedTokens.toLocaleString()} tokens (${reductionPercent}% reduction)`; + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system' as const, + content: compactionContent, + timestamp: new Date(), + }, + ]); + }, + { signal } + ); + + // Handle message queued - fetch full queue state from agent + bus.on( + 'message:queued', + (payload) => { + if (!payload.sessionId) return; + // Fetch fresh queue state from agent to ensure consistency + agent + .getQueuedMessages(payload.sessionId) + .then((messages) => { + setQueuedMessages(messages); + }) + .catch(() => { + // Silently ignore - queue state will sync on next event + }); + }, + { signal } + ); + + // Handle message removed from queue + bus.on( + 'message:removed', + (payload) => { + setQueuedMessages((prev) => prev.filter((m) => m.id !== payload.id)); + }, + { signal } + ); + + // Note: message:dequeued is handled in processStream (via iterator) for proper synchronization + // with streaming events. Don't handle it here via event bus. + + // ============================================================================ + // EXTERNAL TRIGGER HANDLING (scheduler, A2A, API) + // When an external source invokes the agent, we receive events here instead of + // through processStream (which only handles user-initiated streams). + // ============================================================================ + + // Handle external trigger invocation (scheduler, A2A, API) + bus.on( + 'run:invoke', + (payload) => { + // Only handle if this is for the current session + if (payload.sessionId !== currentSessionId) { + return; + } + + // Mark external trigger as active + const messageId = generateMessageId('assistant'); + externalTriggerRef.current = { + active: true, + sessionId: payload.sessionId, + messageId, + }; + + // Extract prompt text from content parts + const promptText = extractTextContent(payload.content); + + // Add the scheduled prompt as a "user" message with a source indicator + const sourceLabel = + payload.source === 'scheduler' + ? '⏰ Scheduled Task' + : payload.source === 'a2a' + ? '🤖 A2A Request' + : payload.source === 'api' + ? '🔌 API Request' + : '📥 External Request'; + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system' as const, + content: `${sourceLabel}`, + timestamp: new Date(), + }, + { + id: generateMessageId('user'), + role: 'user' as const, + content: promptText, + timestamp: new Date(), + }, + ]); + + // Set processing state + setUi((prev) => ({ + ...prev, + isProcessing: true, + isThinking: true, + })); + + // Add assistant pending message for streaming + setPendingMessages([ + { + id: messageId, + role: 'assistant' as const, + content: '', + timestamp: new Date(), + isStreaming: true, + }, + ]); + }, + { signal } + ); + + // Handle streaming chunks for external triggers + bus.on( + 'llm:chunk', + (payload) => { + // Only handle if this is for an active external trigger + if ( + !externalTriggerRef.current.active || + payload.sessionId !== externalTriggerRef.current.sessionId + ) { + return; + } + + // Only handle text chunks (not reasoning) + if (payload.chunkType !== 'text') { + return; + } + + // Update pending message with new content + setPendingMessages((prev) => + prev.map((msg) => + msg.id === externalTriggerRef.current.messageId + ? { ...msg, content: msg.content + payload.content } + : msg + ) + ); + + // Clear thinking state once we start receiving chunks + setUi((prev) => (prev.isThinking ? { ...prev, isThinking: false } : prev)); + }, + { signal } + ); + + // Handle LLM thinking for external triggers + bus.on( + 'llm:thinking', + (payload) => { + if ( + !externalTriggerRef.current.active || + payload.sessionId !== externalTriggerRef.current.sessionId + ) { + return; + } + + setUi((prev) => ({ ...prev, isThinking: true })); + }, + { signal } + ); + + // Handle run completion for external triggers + bus.on( + 'run:complete', + (payload) => { + // Only handle if this is for an active external trigger + if ( + !externalTriggerRef.current.active || + payload.sessionId !== externalTriggerRef.current.sessionId + ) { + return; + } + + // Finalize the pending message + setPendingMessages((prev) => { + const pendingMsg = prev.find( + (m) => m.id === externalTriggerRef.current.messageId + ); + if (pendingMsg) { + // Move to finalized messages + setMessages((msgs) => [...msgs, { ...pendingMsg, isStreaming: false }]); + } + // Clear pending + return prev.filter((m) => m.id !== externalTriggerRef.current.messageId); + }); + + // Clear processing state + setUi((prev) => ({ + ...prev, + isProcessing: false, + isThinking: false, + })); + + // Reset external trigger tracking + externalTriggerRef.current = { active: false, sessionId: null, messageId: null }; + }, + { signal } + ); + + // Cleanup: abort controller removes all listeners at once + return () => { + controller.abort(); + }; + }, [ + agent, + setMessages, + setPendingMessages, + setUi, + setSession, + setInput, + setApproval, + setApprovalQueue, + setQueuedMessages, + currentSessionId, + buffer, + ]); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useBatchedScroll.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useBatchedScroll.ts new file mode 100644 index 00000000..0a624bd9 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useBatchedScroll.ts @@ -0,0 +1,34 @@ +/** + * useBatchedScroll Hook + * Prevents scroll jitter by batching multiple scroll updates in the same frame. + */ + +import { useRef, useEffect, useCallback } from 'react'; + +/** + * A hook to manage batched scroll state updates. + * It allows multiple scroll operations within the same tick to accumulate + * by keeping track of a 'pending' state that resets after render. + */ +export function useBatchedScroll(currentScrollTop: number) { + const pendingScrollTopRef = useRef<number | null>(null); + // We use a ref for currentScrollTop to allow getScrollTop to be stable + // and not depend on the currentScrollTop value directly in its dependency array. + const currentScrollTopRef = useRef(currentScrollTop); + + useEffect(() => { + currentScrollTopRef.current = currentScrollTop; + pendingScrollTopRef.current = null; + }); + + const getScrollTop = useCallback( + () => pendingScrollTopRef.current ?? currentScrollTopRef.current, + [] + ); + + const setPendingScrollTop = useCallback((newScrollTop: number) => { + pendingScrollTopRef.current = newScrollTop; + }, []); + + return { getScrollTop, setPendingScrollTop }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useCLIState.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useCLIState.ts new file mode 100644 index 00000000..830f2b85 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useCLIState.ts @@ -0,0 +1,324 @@ +/** + * Shared CLI State Hook + * + * Contains all common state and logic shared between rendering modes. + * Both AlternateBufferCLI and StaticCLI use this hook. + */ + +import React, { useState, useMemo, useEffect, useRef, useCallback } from 'react'; +import { useStdout } from 'ink'; +import { + getModelDisplayName, + isUserMessage, + type DextoAgent, + type QueuedMessage, +} from '@dexto/core'; +import type { + Message, + StartupInfo, + UIState, + InputState, + SessionState, + TodoItem, +} from '../state/types.js'; +import type { ApprovalRequest } from '../components/ApprovalPrompt.js'; +import { useAgentEvents } from './useAgentEvents.js'; +import { useInputOrchestrator, type Key } from './useInputOrchestrator.js'; +import { InputService, MessageService } from '../services/index.js'; +import { convertHistoryToUIMessages } from '../utils/messageFormatting.js'; +import type { OverlayContainerHandle } from '../containers/OverlayContainer.js'; +import { useTextBuffer, type TextBuffer } from '../components/shared/text-buffer.js'; + +// Re-export types for backwards compatibility +export type { UIState, InputState, SessionState, TodoItem } from '../state/types.js'; + +export interface UseCLIStateProps { + agent: DextoAgent; + initialSessionId: string | null; + startupInfo: StartupInfo; + /** Optional keyboard scroll handler for alternate buffer mode */ + onKeyboardScroll?: (direction: 'up' | 'down') => void; +} + +export interface CLIStateReturn { + // State - finalized messages (rendered in <Static>) + messages: Message[]; + setMessages: React.Dispatch<React.SetStateAction<Message[]>>; + // Pending messages (streaming/in-progress, rendered dynamically) + pendingMessages: Message[]; + setPendingMessages: React.Dispatch<React.SetStateAction<Message[]>>; + // Dequeued buffer - user messages waiting to render after pending + // (ensures correct visual order regardless of React batching) + dequeuedBuffer: Message[]; + setDequeuedBuffer: React.Dispatch<React.SetStateAction<Message[]>>; + // Queued messages (messages waiting to be processed) + queuedMessages: QueuedMessage[]; + setQueuedMessages: React.Dispatch<React.SetStateAction<QueuedMessage[]>>; + // Todo items for workflow tracking + todos: TodoItem[]; + setTodos: React.Dispatch<React.SetStateAction<TodoItem[]>>; + ui: UIState; + setUi: React.Dispatch<React.SetStateAction<UIState>>; + input: InputState; + setInput: React.Dispatch<React.SetStateAction<InputState>>; + session: SessionState; + setSession: React.Dispatch<React.SetStateAction<SessionState>>; + approval: ApprovalRequest | null; + setApproval: React.Dispatch<React.SetStateAction<ApprovalRequest | null>>; + approvalQueue: ApprovalRequest[]; + setApprovalQueue: React.Dispatch<React.SetStateAction<ApprovalRequest[]>>; + + // Text buffer (source of truth for input) + buffer: TextBuffer; + + // Services + inputService: InputService; + messageService: MessageService; + + // Ref for overlay container + overlayContainerRef: React.RefObject<OverlayContainerHandle | null>; + + // Computed data + visibleMessages: Message[]; + + // Agent reference + agent: DextoAgent; + startupInfo: StartupInfo; +} + +export function useCLIState({ + agent, + initialSessionId, + startupInfo, + onKeyboardScroll: _onKeyboardScroll, +}: UseCLIStateProps): CLIStateReturn { + // Messages state - finalized messages (rendered in <Static>) + const [messages, setMessages] = useState<Message[]>([]); + // Pending messages - streaming/in-progress (rendered dynamically outside <Static>) + const [pendingMessages, setPendingMessages] = useState<Message[]>([]); + // Dequeued buffer - user messages rendered after pending (guarantees visual order) + const [dequeuedBuffer, setDequeuedBuffer] = useState<Message[]>([]); + // Queued messages - messages waiting to be processed (uses core type) + const [queuedMessages, setQueuedMessages] = useState<QueuedMessage[]>([]); + // Todo items for workflow tracking (populated via service:event from todo tools) + const [todos, setTodos] = useState<TodoItem[]>([]); + + // UI state + const [ui, setUi] = useState<UIState>({ + isProcessing: false, + isCancelling: false, + isThinking: false, + isCompacting: false, + activeOverlay: 'none', + exitWarningShown: false, + exitWarningTimestamp: null, + mcpWizardServerType: null, + copyModeEnabled: false, + pendingModelSwitch: null, + selectedMcpServer: null, + historySearch: { + isActive: false, + query: '', + matchIndex: 0, + originalInput: '', + lastMatch: '', + }, + promptAddWizard: null, + autoApproveEdits: false, + todoExpanded: true, // Default to expanded to show full todo list + planModeActive: false, + planModeInitialized: false, + }); + + // Input state + const [input, setInput] = useState<InputState>({ + value: '', + history: [], + historyIndex: -1, + draftBeforeHistory: '', + images: [], + pastedBlocks: [], + pasteCounter: 0, + }); + + // Session state - use display name for built-in models at startup + const initialConfig = agent.getCurrentLLMConfig(); + const [session, setSession] = useState<SessionState>({ + id: initialSessionId, + hasActiveSession: initialSessionId !== null, + modelName: getModelDisplayName(initialConfig.model, initialConfig.provider), + }); + + // Approval state + const [approval, setApproval] = useState<ApprovalRequest | null>(null); + const [approvalQueue, setApprovalQueue] = useState<ApprovalRequest[]>([]); + + // Initialize services (memoized) + const inputService = useMemo(() => new InputService(), []); + const messageService = useMemo(() => new MessageService(), []); + + // Get terminal dimensions for buffer + const { stdout } = useStdout(); + const terminalWidth = stdout?.columns || 80; + const inputWidth = Math.max(20, terminalWidth - 4); + + // Memoize onChange to prevent infinite loops (useTextBuffer has onChange in deps) + const handleBufferChange = useCallback((text: string) => { + setInput((prev) => ({ ...prev, value: text })); + }, []); + + // Create text buffer (source of truth for input) + const buffer = useTextBuffer({ + initialText: '', + viewport: { width: inputWidth, height: 10 }, + onChange: handleBufferChange, + }); + + // Update viewport on terminal resize + useEffect(() => { + buffer.setViewport(inputWidth, 10); + }, [inputWidth, buffer.setViewport]); + + // Ref for overlay container (input no longer needs ref) + const overlayContainerRef = useRef<OverlayContainerHandle>(null); + + // Setup event bus subscriptions for non-streaming events + // (streaming events are handled directly via agent.stream() iterator in InputContainer) + // Also handles external triggers (run:invoke) from scheduler, A2A, API + useAgentEvents({ + agent, + setMessages, + setPendingMessages, + setUi, + setSession, + setInput, + setApproval, + setApprovalQueue, + setQueuedMessages, + currentSessionId: session.id, + buffer, + }); + + // Create input handlers for the orchestrator + // Note: Main input is NOT routed through orchestrator - TextBufferInput handles it directly + const approvalHandler = useCallback((inputStr: string, key: Key): boolean => { + return overlayContainerRef.current?.handleInput(inputStr, key) ?? false; + }, []); + + const overlayHandler = useCallback((inputStr: string, key: Key): boolean => { + return overlayContainerRef.current?.handleInput(inputStr, key) ?? false; + }, []); + + // Setup unified input orchestrator (handles global shortcuts, approval, overlay only) + useInputOrchestrator({ + ui, + approval, + input, + session, + queuedMessages, + buffer, + setUi, + setInput, + setMessages, + setPendingMessages, + setQueuedMessages, + agent, + handlers: { + approval: approvalHandler, + overlay: overlayHandler, + }, + }); + + // Clear todos when session changes (todos are per-session) + useEffect(() => { + setTodos([]); + }, [session.id]); + + // Hydrate conversation history when resuming a session + useEffect(() => { + if (!initialSessionId || !session.hasActiveSession || messages.length > 0) { + return; + } + + let cancelled = false; + + (async () => { + try { + const history = await agent.getSessionHistory(initialSessionId); + if (!history?.length || cancelled) return; + const historyMessages = convertHistoryToUIMessages(history, initialSessionId); + setMessages(historyMessages); + + // Extract user messages for input history (arrow up navigation) + const userInputHistory = history + .filter(isUserMessage) + .map((msg) => + msg.content + .filter( + (part): part is { type: 'text'; text: string } => + part.type === 'text' + ) + .map((part) => part.text) + .join('\n') + ) + .filter((text) => text.trim().length > 0); + + setInput((prev) => ({ + ...prev, + history: userInputHistory, + historyIndex: -1, + })); + } catch (error) { + if (cancelled) return; + setMessages((prev) => [ + ...prev, + { + id: `error-${Date.now()}`, + role: 'system', + content: `Error: ${error instanceof Error ? error.message : String(error)}`, + timestamp: new Date(), + }, + ]); + } + })(); + + return () => { + cancelled = true; + }; + }, [agent, initialSessionId, messages.length, session.hasActiveSession]); + + // Get visible messages - no limit needed + // Static mode: items are permanent in terminal scrollback, Ink only renders NEW keys + // AlternateBuffer mode: VirtualizedList handles its own virtualization + const visibleMessages = messages; + + return { + messages, + setMessages, + pendingMessages, + setPendingMessages, + dequeuedBuffer, + setDequeuedBuffer, + queuedMessages, + setQueuedMessages, + todos, + setTodos, + ui, + setUi, + input, + setInput, + session, + setSession, + approval, + setApproval, + approvalQueue, + setApprovalQueue, + buffer, + inputService, + messageService, + overlayContainerRef, + visibleMessages, + agent, + startupInfo, + }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useElapsedTime.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useElapsedTime.ts new file mode 100644 index 00000000..07e7266a --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useElapsedTime.ts @@ -0,0 +1,96 @@ +/** + * useElapsedTime Hook + * Tracks elapsed time during processing with live updates + */ + +import { useState, useEffect, useRef } from 'react'; + +export interface ElapsedTimeOptions { + /** Whether timing is active (should only run during processing) */ + isActive: boolean; + /** Update interval in milliseconds (default: 100ms) */ + intervalMs?: number; +} + +export interface ElapsedTimeResult { + /** Elapsed time in milliseconds */ + elapsedMs: number; + /** Formatted elapsed time string (e.g., "1.2s", "1m 23s") */ + formatted: string; +} + +/** + * Format milliseconds into a human-readable string + * - Under 1 minute: "1.2s" + * - 1+ minutes: "1m 23s" + * - 1+ hours: "1h 2m 3s" + */ +function formatElapsedTime(ms: number): string { + const seconds = Math.floor(ms / 1000); + const tenths = Math.floor((ms % 1000) / 100); + + if (seconds < 60) { + return `${seconds}.${tenths}s`; + } + + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + + if (minutes < 60) { + return `${minutes}m ${remainingSeconds}s`; + } + + const hours = Math.floor(minutes / 60); + const remainingMinutes = minutes % 60; + return `${hours}h ${remainingMinutes}m ${remainingSeconds}s`; +} + +/** + * Hook that tracks elapsed time during processing + * + * @param options - Configuration options + * @returns Elapsed time in ms and formatted string + */ +export function useElapsedTime({ + isActive, + intervalMs = 100, +}: ElapsedTimeOptions): ElapsedTimeResult { + const [elapsedMs, setElapsedMs] = useState(0); + const startTimeRef = useRef<number | null>(null); + const intervalRef = useRef<NodeJS.Timeout | null>(null); + + useEffect(() => { + if (isActive) { + // Start timing + startTimeRef.current = Date.now(); + setElapsedMs(0); + + // Update elapsed time at regular intervals + intervalRef.current = setInterval(() => { + if (startTimeRef.current !== null) { + setElapsedMs(Date.now() - startTimeRef.current); + } + }, intervalMs); + } else { + // Stop timing + if (intervalRef.current) { + clearInterval(intervalRef.current); + intervalRef.current = null; + } + startTimeRef.current = null; + setElapsedMs(0); + } + + return () => { + if (intervalRef.current) { + clearInterval(intervalRef.current); + intervalRef.current = null; + } + }; + }, [isActive, intervalMs]); + + return { + elapsedMs, + formatted: formatElapsedTime(elapsedMs), + }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useHistorySearch.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useHistorySearch.ts new file mode 100644 index 00000000..04341eee --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useHistorySearch.ts @@ -0,0 +1,326 @@ +/** + * useHistorySearch - Hook for Ctrl+R reverse history search + * + * Manages search state and provides handlers for search operations. + */ + +import { useCallback, useMemo, type Dispatch, type SetStateAction } from 'react'; +import type { InputState, UIState } from '../state/types.js'; +import type { TextBuffer } from '../components/shared/text-buffer.js'; + +interface UseHistorySearchProps { + ui: UIState; + input: InputState; + buffer: TextBuffer; + setUi: Dispatch<SetStateAction<UIState>>; + setInput: Dispatch<SetStateAction<InputState>>; +} + +interface UseHistorySearchReturn { + /** Whether search mode is active */ + isActive: boolean; + /** Current search query */ + query: string; + /** Current match (or last valid match if no current results) */ + currentMatch: string | null; + /** Whether there's a match for the current query */ + hasMatch: boolean; + /** Original input to potentially restore */ + originalInput: string; + /** Enter search mode */ + enter: () => void; + /** Exit search mode (keep current input) */ + exit: (showRestoreHint?: boolean) => void; + /** Cancel search mode (restore original - used by Escape) */ + cancel: () => void; + /** Update search query */ + updateQuery: (query: string) => void; + /** Add character to query */ + appendToQuery: (char: string) => void; + /** Remove last character from query */ + backspace: () => void; + /** Cycle to next (older) match - Ctrl+R */ + cycleNext: () => void; + /** Cycle to previous (newer) match - Ctrl+Shift+R */ + cyclePrev: () => void; + /** Accept current match (for Enter - also exits search mode) */ + accept: () => void; + /** Handle a keypress - returns true if consumed */ + handleKey: ( + inputStr: string, + key: { + ctrl: boolean; + shift: boolean; + return: boolean; + backspace: boolean; + delete: boolean; + escape: boolean; + meta: boolean; + } + ) => boolean; +} + +/** + * Find matches in history for a query (reversed so newest is first) + */ +function findMatches(history: string[], query: string): string[] { + if (!query) return []; + const lowerQuery = query.toLowerCase(); + return history.filter((item) => item.toLowerCase().includes(lowerQuery)).reverse(); +} + +/** + * Hook for managing reverse history search + */ +export function useHistorySearch({ + ui, + input, + buffer, + setUi, + setInput, +}: UseHistorySearchProps): UseHistorySearchReturn { + const { historySearch } = ui; + const { history } = input; + + // Compute current matches and whether we have a match + const matches = useMemo( + () => findMatches(history, historySearch.query), + [history, historySearch.query] + ); + + const hasMatch = matches.length > 0; + const currentMatch = hasMatch + ? matches[Math.min(historySearch.matchIndex, matches.length - 1)] || null + : historySearch.lastMatch || null; + + // Enter search mode + const enter = useCallback(() => { + const currentText = buffer.text; + setUi((prev) => ({ + ...prev, + historySearch: { + isActive: true, + query: '', + matchIndex: 0, + originalInput: currentText, + lastMatch: '', + }, + })); + }, [buffer, setUi]); + + // Exit search mode (keep current input) + const exit = useCallback(() => { + setUi((prev) => ({ + ...prev, + historySearch: { + isActive: false, + query: '', + matchIndex: 0, + originalInput: '', + lastMatch: '', + }, + })); + }, [setUi]); + + // Cancel search mode (restore original) - kept for potential future use + const cancel = useCallback(() => { + const originalInput = historySearch.originalInput; + buffer.setText(originalInput); + setInput((prev) => ({ ...prev, value: originalInput })); + exit(); + }, [historySearch.originalInput, buffer, setInput, exit]); + + // Apply a match to the input buffer + const applyMatch = useCallback( + (match: string | null) => { + if (match) { + buffer.setText(match); + setInput((prev) => ({ ...prev, value: match })); + } + }, + [buffer, setInput] + ); + + // Update query and apply resulting match + const updateQuery = useCallback( + (newQuery: string) => { + const newMatches = findMatches(history, newQuery); + const newHasMatch = newMatches.length > 0; + const newMatch = newHasMatch ? newMatches[0] : null; + + setUi((prev) => ({ + ...prev, + historySearch: { + ...prev.historySearch, + query: newQuery, + matchIndex: 0, + // Update lastMatch only if we have a new match + lastMatch: newMatch || prev.historySearch.lastMatch, + }, + })); + + // Apply match (or keep last match if no new match) + if (newMatch) { + applyMatch(newMatch); + } + // If no match, keep whatever is currently in input (the last match) + }, + [history, setUi, applyMatch] + ); + + // Append character to query + const appendToQuery = useCallback( + (char: string) => { + updateQuery(historySearch.query + char); + }, + [historySearch.query, updateQuery] + ); + + // Backspace - remove last character + const backspace = useCallback(() => { + if (historySearch.query.length > 0) { + updateQuery(historySearch.query.slice(0, -1)); + } + }, [historySearch.query, updateQuery]); + + // Cycle to next (older) match - Ctrl+R + const cycleNext = useCallback(() => { + if (matches.length === 0) return; + + const newIndex = Math.min(historySearch.matchIndex + 1, matches.length - 1); + const newMatch = matches[newIndex]; + + setUi((prev) => ({ + ...prev, + historySearch: { + ...prev.historySearch, + matchIndex: newIndex, + lastMatch: newMatch || prev.historySearch.lastMatch, + }, + })); + + if (newMatch) { + applyMatch(newMatch); + } + }, [matches, historySearch.matchIndex, setUi, applyMatch]); + + // Cycle to previous (newer) match - Ctrl+Shift+R + const cyclePrev = useCallback(() => { + if (matches.length === 0) return; + + const newIndex = Math.max(0, historySearch.matchIndex - 1); + const newMatch = matches[newIndex]; + + setUi((prev) => ({ + ...prev, + historySearch: { + ...prev.historySearch, + matchIndex: newIndex, + lastMatch: newMatch || prev.historySearch.lastMatch, + }, + })); + + if (newMatch) { + applyMatch(newMatch); + } + }, [matches, historySearch.matchIndex, setUi, applyMatch]); + + // Accept current match and exit + const accept = useCallback(() => { + // Input already has the match, just exit + exit(); + }, [exit]); + + // Handle keypress - returns true if consumed + const handleKey = useCallback( + ( + inputStr: string, + key: { + ctrl: boolean; + shift: boolean; + return: boolean; + backspace: boolean; + delete: boolean; + escape: boolean; + meta: boolean; + } + ): boolean => { + if (!historySearch.isActive) { + // Not in search mode - check if Ctrl+R to enter + if (key.ctrl && inputStr === 'r') { + enter(); + return true; + } + return false; + } + + // In search mode - handle all keys + + // Ctrl+E: cycle to previous (newer) match + if (key.ctrl && inputStr === 'e') { + cyclePrev(); + return true; + } + + // Ctrl+R: cycle to next (older) match + if (key.ctrl && inputStr === 'r') { + cycleNext(); + return true; + } + + // Enter: accept match and exit (keep matched text, don't submit) + if (key.return) { + accept(); + return true; // Consume - don't submit + } + + // Escape: restore original and exit + if (key.escape) { + cancel(); + return true; + } + + // Backspace: remove character from query + if (key.backspace || key.delete) { + backspace(); + return true; + } + + // Regular typing: add to query + if (inputStr && !key.ctrl && !key.meta) { + appendToQuery(inputStr); + return true; + } + + return true; // Consume other keys while in search mode + }, + [ + historySearch.isActive, + enter, + cycleNext, + cyclePrev, + accept, + cancel, + backspace, + appendToQuery, + ] + ); + + return { + isActive: historySearch.isActive, + query: historySearch.query, + currentMatch, + hasMatch, + originalInput: historySearch.originalInput, + enter, + exit, + cancel, + updateQuery, + appendToQuery, + backspace, + cycleNext, + cyclePrev, + accept, + handleKey, + }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useInputHistory.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useInputHistory.ts new file mode 100644 index 00000000..d364017e --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useInputHistory.ts @@ -0,0 +1,39 @@ +/** + * Hook for managing input history navigation + * Handles keyboard shortcuts for history traversal + */ + +import type React from 'react'; +import { useInput } from 'ink'; +import type { CLIAction } from '../state/actions.js'; +import type { InputState } from '../state/types.js'; + +interface UseInputHistoryProps { + inputState: InputState; + dispatch: React.Dispatch<CLIAction>; + isActive: boolean; +} + +/** + * Manages input history navigation with arrow keys + */ +export function useInputHistory({ inputState, dispatch, isActive }: UseInputHistoryProps): void { + useInput( + (inputChar, key) => { + if (!isActive || inputState.history.length === 0) return; + + if (key.upArrow) { + dispatch({ + type: 'INPUT_HISTORY_NAVIGATE', + direction: 'up', + }); + } else if (key.downArrow) { + dispatch({ + type: 'INPUT_HISTORY_NAVIGATE', + direction: 'down', + }); + } + }, + { isActive } + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useInputOrchestrator.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useInputOrchestrator.ts new file mode 100644 index 00000000..3dee180c --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useInputOrchestrator.ts @@ -0,0 +1,952 @@ +/** + * Unified Input Orchestrator + * + * Single point of keyboard input handling for the entire ink-cli. + * Routes keystrokes based on focus state to prevent conflicts. + * + * Focus Priority (highest to lowest): + * 1. Approval prompt (when visible) + * 2. Active overlay (selector/autocomplete) + * 3. Global shortcuts (Ctrl+C, Escape - handled specially) + * 4. Main text input (default) + * + * Mouse scroll events are handled separately by ScrollProvider. + */ + +import type React from 'react'; +import { useEffect, useRef, useCallback } from 'react'; +import { useApp } from 'ink'; +import type { UIState, InputState, SessionState, OverlayType, Message } from '../state/types.js'; +import type { ApprovalRequest } from '../components/ApprovalPrompt.js'; +import type { DextoAgent, QueuedMessage } from '@dexto/core'; +import { useKeypress, type Key as RawKey } from './useKeypress.js'; +import { enableMouseEvents, disableMouseEvents } from '../utils/mouse.js'; +import type { TextBuffer } from '../components/shared/text-buffer.js'; +import { generateMessageId } from '../utils/idGenerator.js'; + +/** Time window for double Ctrl+C to exit (in milliseconds) */ +const EXIT_WARNING_TIMEOUT = 3000; + +/** + * Ink-compatible Key interface + * Converted from our custom KeypressContext Key + */ +export interface Key { + upArrow: boolean; + downArrow: boolean; + leftArrow: boolean; + rightArrow: boolean; + pageUp: boolean; + pageDown: boolean; + return: boolean; + escape: boolean; + ctrl: boolean; + shift: boolean; + meta: boolean; + tab: boolean; + backspace: boolean; + delete: boolean; + /** True if this input came from a paste operation (bracketed paste) */ + paste: boolean; +} + +/** + * Convert our KeypressContext Key to Ink-compatible Key + */ +function convertKey(rawKey: RawKey): { input: string; key: Key } { + const key: Key = { + upArrow: rawKey.name === 'up', + downArrow: rawKey.name === 'down', + leftArrow: rawKey.name === 'left', + rightArrow: rawKey.name === 'right', + pageUp: rawKey.name === 'pageup', + pageDown: rawKey.name === 'pagedown', + return: rawKey.name === 'return' || rawKey.name === 'enter', + escape: rawKey.name === 'escape', + ctrl: rawKey.ctrl, + shift: rawKey.shift, + meta: rawKey.meta, + tab: rawKey.name === 'tab', + backspace: rawKey.name === 'backspace', + delete: rawKey.name === 'delete', + paste: rawKey.paste, + }; + + // For insertable characters, use the sequence + // For named keys like 'a', 'b', use the name + let input = rawKey.sequence; + + // For Ctrl+letter combinations, use the name (e.g., 'c' for Ctrl+C) + if (rawKey.ctrl && rawKey.name && rawKey.name.length === 1) { + input = rawKey.name; + } + + // For paste events, use the full sequence + if (rawKey.paste) { + input = rawKey.sequence; + } + + return { input, key }; +} + +/** + * Input handler function signature + * Returns true if the input was consumed, false to continue to next handler + */ +export type InputHandler = (input: string, key: Key) => boolean | void; + +/** + * Handler configuration for the orchestrator + * + * Note: Main text input is NOT routed through the orchestrator. + * TextBufferInput handles its own input directly via useKeypress. + */ +export interface InputHandlers { + /** Handler for approval prompt (highest priority) */ + approval?: InputHandler; + /** Handler for active overlay (selector/autocomplete) */ + overlay?: InputHandler; +} + +export interface UseInputOrchestratorProps { + ui: UIState; + approval: ApprovalRequest | null; + input: InputState; + session: SessionState; + /** Queued messages (for cancel handling) */ + queuedMessages: QueuedMessage[]; + /** Text buffer for clearing input on first Ctrl+C */ + buffer: TextBuffer; + setUi: React.Dispatch<React.SetStateAction<UIState>>; + setInput: React.Dispatch<React.SetStateAction<InputState>>; + setMessages: React.Dispatch<React.SetStateAction<Message[]>>; + setPendingMessages: React.Dispatch<React.SetStateAction<Message[]>>; + setQueuedMessages: React.Dispatch<React.SetStateAction<QueuedMessage[]>>; + agent: DextoAgent; + handlers: InputHandlers; +} + +/** + * Determines the current focus target based on state + */ +type FocusTarget = 'approval' | 'overlay' | 'input'; + +function getFocusTarget(approval: ApprovalRequest | null, activeOverlay: OverlayType): FocusTarget { + // Approval has highest priority + if (approval !== null) { + return 'approval'; + } + + // Active overlay has next priority + if (activeOverlay !== 'none' && activeOverlay !== 'approval') { + return 'overlay'; + } + + // Default to main input + return 'input'; +} + +/** + * Unified input orchestrator hook + * + * This is the ONLY keyboard input hook in the entire ink-cli. + * All keyboard handling is routed through this single point. + * Mouse events are handled separately by MouseProvider/ScrollProvider. + */ +export function useInputOrchestrator({ + ui, + approval, + input, + session, + queuedMessages, + buffer, + setUi, + setInput, + setMessages, + setPendingMessages, + setQueuedMessages, + agent, + handlers, +}: UseInputOrchestratorProps): void { + const { exit } = useApp(); + + // Use refs to avoid stale closures in the callback + const uiRef = useRef(ui); + const approvalRef = useRef(approval); + const inputRef = useRef(input); + const sessionRef = useRef(session); + const queuedMessagesRef = useRef(queuedMessages); + const bufferRef = useRef(buffer); + const handlersRef = useRef(handlers); + + // Keep refs in sync + useEffect(() => { + uiRef.current = ui; + approvalRef.current = approval; + inputRef.current = input; + sessionRef.current = session; + queuedMessagesRef.current = queuedMessages; + bufferRef.current = buffer; + handlersRef.current = handlers; + }, [ui, approval, input, session, queuedMessages, buffer, handlers]); + + // Auto-clear exit warning after timeout + useEffect(() => { + if (!ui.exitWarningShown || !ui.exitWarningTimestamp) return; + + const elapsed = Date.now() - ui.exitWarningTimestamp; + const remaining = EXIT_WARNING_TIMEOUT - elapsed; + + if (remaining <= 0) { + setUi((prev) => ({ ...prev, exitWarningShown: false, exitWarningTimestamp: null })); + return; + } + + const timer = setTimeout(() => { + setUi((prev) => ({ ...prev, exitWarningShown: false, exitWarningTimestamp: null })); + }, remaining); + + return () => clearTimeout(timer); + }, [ui.exitWarningShown, ui.exitWarningTimestamp, setUi]); + + // Handle Ctrl+C (special case - handled globally regardless of focus) + // Priority: 1) Clear input if has text, 2) Exit warning/exit + // Note: Ctrl+C does NOT cancel processing - use Escape for that + const handleCtrlC = useCallback(() => { + const currentUi = uiRef.current; + const currentBuffer = bufferRef.current; + + if (currentBuffer.text.length > 0) { + // Has input text - clear it AND show exit warning + // This way: first Ctrl+C clears, second Ctrl+C exits + currentBuffer.setText(''); + setUi((prev) => ({ + ...prev, + exitWarningShown: true, + exitWarningTimestamp: Date.now(), + })); + } else { + // No text - handle exit with double-press safety + if (currentUi.exitWarningShown) { + // Second Ctrl+C within timeout - actually exit + exit(); + } else { + // First Ctrl+C - show warning + setUi((prev) => ({ + ...prev, + exitWarningShown: true, + exitWarningTimestamp: Date.now(), + })); + } + } + }, [setUi, exit]); + + // Handle Escape (context-aware) + const handleEscape = useCallback((): boolean => { + const currentUi = uiRef.current; + const currentSession = sessionRef.current; + const currentQueuedMessages = queuedMessagesRef.current; + const currentBuffer = bufferRef.current; + + // Exit history search mode if active - restore original input + if (currentUi.historySearch.isActive) { + const originalInput = currentUi.historySearch.originalInput; + currentBuffer.setText(originalInput); + setInput((prev) => ({ ...prev, value: originalInput })); + setUi((prev) => ({ + ...prev, + historySearch: { + isActive: false, + query: '', + matchIndex: 0, + originalInput: '', + lastMatch: '', + }, + })); + return true; + } + + // Clear exit warning if shown + if (currentUi.exitWarningShown) { + setUi((prev) => ({ ...prev, exitWarningShown: false, exitWarningTimestamp: null })); + return true; + } + + // Cancel processing if active + if (currentUi.isProcessing) { + if (currentSession.id) { + // Cancel current run + void agent.cancel(currentSession.id).catch(() => {}); + // Clear the queue on server (we'll bring messages to input for editing) + void agent.clearMessageQueue(currentSession.id).catch(() => {}); + } + + // Finalize any pending messages first (move to messages) + // Mark running tools as cancelled with error state + setPendingMessages((pending) => { + if (pending.length > 0) { + const updated = pending.map((msg) => { + // Mark running tools as cancelled + if (msg.role === 'tool' && msg.toolStatus === 'running') { + return { + ...msg, + toolStatus: 'finished' as const, + toolResult: 'Cancelled', + isError: true, + }; + } + return msg; + }); + setMessages((prev) => [...prev, ...updated]); + } + return []; + }); + + setUi((prev) => ({ + ...prev, + isCancelling: true, + isProcessing: false, + isThinking: false, + })); + + // Add interrupted message + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('system'), + role: 'system', + content: 'Interrupted - what should Dexto do next?', + timestamp: new Date(), + }, + ]); + + // If there were queued messages, bring them back to input for editing + if (currentQueuedMessages.length > 0) { + // Extract and coalesce text content from all queued messages + const coalescedText = currentQueuedMessages + .map((msg) => + msg.content + .filter( + (part): part is { type: 'text'; text: string } => + part.type === 'text' + ) + .map((part) => part.text) + .join('\n') + ) + .filter((text) => text.length > 0) + .join('\n\n'); + + if (coalescedText) { + currentBuffer.setText(coalescedText); + setInput((prev) => ({ ...prev, value: coalescedText })); + } + + // Clear the queue state immediately (don't wait for server events) + setQueuedMessages([]); + } + + return true; + } + + // Close overlay if active (let the overlay handler deal with specifics) + if (currentUi.activeOverlay !== 'none') { + // Don't consume - let overlay handler close it with proper cleanup + return false; + } + + return false; + }, [agent, setUi, setMessages, setPendingMessages, setInput, setQueuedMessages]); + + // The keypress handler for the entire application + const handleKeypress = useCallback( + (rawKey: RawKey) => { + const currentUi = uiRef.current; + const currentApproval = approvalRef.current; + const currentHandlers = handlersRef.current; + + // Convert to Ink-compatible format + const { input: inputStr, key } = convertKey(rawKey); + + // === COPY MODE HANDLING === + // When in copy mode, any key exits copy mode (mouse events re-enabled) + if (currentUi.copyModeEnabled) { + setUi((prev) => ({ ...prev, copyModeEnabled: false })); + enableMouseEvents(); // Re-enable mouse events + return; // Don't process any other keys while exiting copy mode + } + + // === GLOBAL SHORTCUTS (always handled first) === + + // === HISTORY SEARCH MODE HANDLING === + if (currentUi.historySearch.isActive) { + const currentInput = inputRef.current; + const currentBuffer = bufferRef.current; + + // Helper to find matches (reversed so newest is first) + const findMatches = (query: string): string[] => { + if (!query) return []; + const lowerQuery = query.toLowerCase(); + return currentInput.history + .filter((item) => item.toLowerCase().includes(lowerQuery)) + .reverse(); + }; + + // Helper to apply a match to the input buffer and track lastMatch + const applyMatchAndUpdateState = (query: string, matchIdx: number) => { + if (!query) { + // No query - restore original + const orig = currentUi.historySearch.originalInput; + currentBuffer.setText(orig); + setInput((prev) => ({ ...prev, value: orig })); + return; + } + + const matches = findMatches(query); + if (matches.length > 0) { + const idx = Math.min(matchIdx, matches.length - 1); + const match = matches[idx]; + if (match) { + currentBuffer.setText(match); + setInput((prev) => ({ ...prev, value: match })); + // Update lastMatch in state + setUi((prev) => ({ + ...prev, + historySearch: { ...prev.historySearch, lastMatch: match }, + })); + } + } + // If no match, keep current buffer content (which has last valid match) + }; + + // Ctrl+E in search mode: cycle to previous (newer) match + if (key.ctrl && inputStr === 'e') { + const matches = findMatches(currentUi.historySearch.query); + if (matches.length > 0) { + const newIdx = Math.max(0, currentUi.historySearch.matchIndex - 1); + setUi((prev) => ({ + ...prev, + historySearch: { ...prev.historySearch, matchIndex: newIdx }, + })); + applyMatchAndUpdateState(currentUi.historySearch.query, newIdx); + } + return; + } + + // Ctrl+R in search mode: cycle to next (older) match + if (key.ctrl && inputStr === 'r') { + const matches = findMatches(currentUi.historySearch.query); + if (matches.length > 0) { + const newIdx = Math.min( + currentUi.historySearch.matchIndex + 1, + matches.length - 1 + ); + setUi((prev) => ({ + ...prev, + historySearch: { ...prev.historySearch, matchIndex: newIdx }, + })); + applyMatchAndUpdateState(currentUi.historySearch.query, newIdx); + } + return; + } + + // Enter: Accept current match and exit search mode (input already has match) + if (key.return) { + setUi((prev) => ({ + ...prev, + historySearch: { + isActive: false, + query: '', + matchIndex: 0, + originalInput: '', + lastMatch: '', + }, + })); + return; + } + + // Backspace: Remove last character from search query + if (key.backspace || key.delete) { + const newQuery = currentUi.historySearch.query.slice(0, -1); + setUi((prev) => ({ + ...prev, + historySearch: { ...prev.historySearch, query: newQuery, matchIndex: 0 }, + })); + applyMatchAndUpdateState(newQuery, 0); + return; + } + + // Regular typing: Add to search query + if (inputStr && !key.ctrl && !key.meta && !key.escape) { + const newQuery = currentUi.historySearch.query + inputStr; + setUi((prev) => ({ + ...prev, + historySearch: { ...prev.historySearch, query: newQuery, matchIndex: 0 }, + })); + applyMatchAndUpdateState(newQuery, 0); + return; + } + + // Escape is handled by handleEscape, so fall through + } + + // Ctrl+R: Enter history search mode - save current input + if (key.ctrl && inputStr === 'r') { + const currentBuffer = bufferRef.current; + const currentText = currentBuffer.text; + setUi((prev) => ({ + ...prev, + historySearch: { + isActive: true, + query: '', + matchIndex: 0, + originalInput: currentText, + lastMatch: '', + }, + })); + return; + } + + // Ctrl+S: Toggle copy mode (for text selection in alternate buffer) + if (key.ctrl && inputStr === 's') { + setUi((prev) => ({ ...prev, copyModeEnabled: true })); + disableMouseEvents(); // Disable mouse events so terminal can handle selection + return; + } + + // Ctrl+T: Toggle todo list expansion (collapsed shows only current task) + if (key.ctrl && inputStr === 't') { + setUi((prev) => ({ ...prev, todoExpanded: !prev.todoExpanded })); + return; + } + + // Shift+Tab: Cycle through modes (when not in approval modal) + // Modes: Normal → Plan Mode → Accept All Edits → Normal + // Note: When in approval modal for edit/write tools, ApprovalPrompt handles Shift+Tab differently + if (key.shift && key.tab && currentApproval === null) { + setUi((prev) => { + // Determine current mode and cycle to next + if (!prev.planModeActive && !prev.autoApproveEdits) { + // Normal → Plan Mode + return { + ...prev, + planModeActive: true, + planModeInitialized: false, + }; + } else if (prev.planModeActive) { + // Plan Mode → Accept All Edits + return { + ...prev, + planModeActive: false, + planModeInitialized: false, + autoApproveEdits: true, + }; + } else { + // Accept All Edits → Normal + return { + ...prev, + autoApproveEdits: false, + }; + } + }); + return; + } + + // Ctrl+C: Always handle globally for cancellation/exit + if (key.ctrl && inputStr === 'c') { + handleCtrlC(); + return; + } + + // Escape: Try global handling first + if (key.escape) { + if (handleEscape()) { + return; // Consumed by global handler + } + // Fall through to focused component + } + + // === ROUTE TO FOCUSED COMPONENT === + // Only approval and overlay handlers are routed through the orchestrator. + // Main text input handles its own keypress directly (via TextBufferInput). + + const focusTarget = getFocusTarget(currentApproval, currentUi.activeOverlay); + + switch (focusTarget) { + case 'approval': + if (currentHandlers.approval) { + currentHandlers.approval(inputStr, key); + } + // Approval always consumes - main input won't see it + break; + + case 'overlay': + if (currentHandlers.overlay) { + currentHandlers.overlay(inputStr, key); + } + // Overlay may or may not consume - main input handles independently + break; + + case 'input': + // No routing needed - TextBufferInput handles its own input + // Clear exit warning on any typing (user changed their mind) + if ( + currentUi.exitWarningShown && + !key.ctrl && + !key.meta && + !key.escape && + inputStr.length > 0 + ) { + setUi((prev) => ({ + ...prev, + exitWarningShown: false, + exitWarningTimestamp: null, + })); + } + break; + } + }, + [handleCtrlC, handleEscape, setUi] + ); + + // Subscribe to keypress events + useKeypress(handleKeypress, { isActive: true }); +} + +/** + * Create an input handler for the approval prompt + */ +export interface ApprovalHandlerProps { + onApprove: (rememberChoice: boolean) => void; + onDeny: () => void; + onCancel: () => void; + selectedOption: 'yes' | 'yes-session' | 'no'; + setSelectedOption: (option: 'yes' | 'yes-session' | 'no') => void; + isCommandConfirmation: boolean; +} + +export function createApprovalInputHandler({ + onApprove, + onDeny, + onCancel, + selectedOption, + setSelectedOption, + isCommandConfirmation, +}: ApprovalHandlerProps): InputHandler { + return (_input: string, key: Key) => { + if (key.upArrow) { + // Move up (skip yes-session for command confirmations) + if (selectedOption === 'yes') { + setSelectedOption('no'); + } else if (selectedOption === 'yes-session') { + setSelectedOption('yes'); + } else { + // no -> yes-session (or yes for command confirmations) + setSelectedOption(isCommandConfirmation ? 'yes' : 'yes-session'); + } + return true; + } + + if (key.downArrow) { + // Move down (skip yes-session for command confirmations) + if (selectedOption === 'yes') { + setSelectedOption(isCommandConfirmation ? 'no' : 'yes-session'); + } else if (selectedOption === 'yes-session') { + setSelectedOption('no'); + } else { + setSelectedOption('yes'); // no -> yes (wrap) + } + return true; + } + + if (key.return) { + // Enter key - confirm selection + if (selectedOption === 'yes') { + onApprove(false); + } else if (selectedOption === 'yes-session') { + onApprove(true); + } else { + onDeny(); + } + return true; + } + + if (key.escape) { + onCancel(); + return true; + } + + return false; + }; +} + +/** + * Create an input handler for selector components (BaseSelector pattern) + */ +export interface SelectorHandlerProps { + itemsLength: number; + selectedIndexRef: React.RefObject<number>; + onSelectIndex: (index: number) => void; + onSelect: () => void; + onClose: () => void; +} + +export function createSelectorInputHandler({ + itemsLength, + selectedIndexRef, + onSelectIndex, + onSelect, + onClose, +}: SelectorHandlerProps): InputHandler { + return (_input: string, key: Key) => { + if (itemsLength === 0) return false; + + if (key.upArrow) { + const currentIndex = selectedIndexRef.current ?? 0; + const nextIndex = (currentIndex - 1 + itemsLength) % itemsLength; + onSelectIndex(nextIndex); + return true; + } + + if (key.downArrow) { + const currentIndex = selectedIndexRef.current ?? 0; + const nextIndex = (currentIndex + 1) % itemsLength; + onSelectIndex(nextIndex); + return true; + } + + if (key.escape) { + onClose(); + return true; + } + + if (key.return) { + onSelect(); + return true; + } + + return false; + }; +} + +/** + * Create an input handler for autocomplete components + */ +export interface AutocompleteHandlerProps extends SelectorHandlerProps { + onTab?: () => void; +} + +export function createAutocompleteInputHandler({ + itemsLength, + selectedIndexRef, + onSelectIndex, + onSelect, + onClose, + onTab, +}: AutocompleteHandlerProps): InputHandler { + const baseHandler = createSelectorInputHandler({ + itemsLength, + selectedIndexRef, + onSelectIndex, + onSelect, + onClose, + }); + + return (input: string, key: Key) => { + // Handle Tab for "load into input" functionality + if (key.tab && onTab) { + onTab(); + return true; + } + + return baseHandler(input, key); + }; +} + +/** + * Create an input handler for the main text input + */ +export interface MainInputHandlerProps { + value: string; + cursorPos: number; + onChange: (value: string) => void; + setCursorPos: (pos: number) => void; + onSubmit: (value: string) => void; + isDisabled: boolean; + history: string[]; + historyIndex: number; + onHistoryNavigate?: (direction: 'up' | 'down') => void; + getLineInfo: (pos: number) => { + lines: string[]; + lineIndex: number; + colIndex: number; + charCount: number; + }; + getLineStart: (lineIndex: number) => number; +} + +export function createMainInputHandler({ + value, + cursorPos, + onChange, + setCursorPos, + onSubmit, + isDisabled, + history, + historyIndex, + onHistoryNavigate, + getLineInfo, + getLineStart, +}: MainInputHandlerProps): InputHandler { + return (input: string, key: Key) => { + if (isDisabled) return false; + + const lines = value.split('\n'); + const isMultiLine = lines.length > 1; + const { lineIndex, colIndex } = getLineInfo(cursorPos); + + // Newline detection based on actual terminal behavior + const isCtrlJ = input === '\n'; + const isShiftEnter = + input === '\\\r' || + (key.return && key.shift) || + input === '\x1b[13;2u' || + input === '\x1bOM'; + const wantsNewline = isCtrlJ || isShiftEnter || (key.return && key.meta); + + if (wantsNewline) { + const newValue = value.slice(0, cursorPos) + '\n' + value.slice(cursorPos); + onChange(newValue); + setCursorPos(cursorPos + 1); + return true; + } + + // Enter = submit + if (key.return) { + if (value.trim()) { + onSubmit(value); + } + return true; + } + + // Backspace - delete character before cursor + const isBackspace = key.backspace || input === '\x7f' || input === '\x08'; + if (isBackspace) { + if (cursorPos > 0) { + const newValue = value.slice(0, cursorPos - 1) + value.slice(cursorPos); + onChange(newValue); + setCursorPos(cursorPos - 1); + } + return true; + } + + // Delete - delete character at cursor (forward delete) + if (key.delete) { + if (cursorPos < value.length) { + const newValue = value.slice(0, cursorPos) + value.slice(cursorPos + 1); + onChange(newValue); + } + return true; + } + + // Left arrow + if (key.leftArrow) { + setCursorPos(Math.max(0, cursorPos - 1)); + return true; + } + + // Right arrow + if (key.rightArrow) { + setCursorPos(Math.min(value.length, cursorPos + 1)); + return true; + } + + // Up arrow + if (key.upArrow) { + if (isMultiLine && lineIndex > 0) { + const prevLineStart = getLineStart(lineIndex - 1); + const prevLineLength = lines[lineIndex - 1]!.length; + const newCol = Math.min(colIndex, prevLineLength); + setCursorPos(prevLineStart + newCol); + } else if (onHistoryNavigate && history.length > 0) { + onHistoryNavigate('up'); + } + return true; + } + + // Down arrow + if (key.downArrow) { + if (isMultiLine && lineIndex < lines.length - 1) { + const nextLineStart = getLineStart(lineIndex + 1); + const nextLineLength = lines[lineIndex + 1]!.length; + const newCol = Math.min(colIndex, nextLineLength); + setCursorPos(nextLineStart + newCol); + } else if (onHistoryNavigate && historyIndex >= 0) { + onHistoryNavigate('down'); + } + return true; + } + + // Ctrl+A - start of line + if (key.ctrl && input === 'a') { + setCursorPos(getLineStart(lineIndex)); + return true; + } + + // Ctrl+E - end of line + if (key.ctrl && input === 'e') { + const lineStart = getLineStart(lineIndex); + setCursorPos(lineStart + lines[lineIndex]!.length); + return true; + } + + // Ctrl+K - delete to end of line + if (key.ctrl && input === 'k') { + const lineStart = getLineStart(lineIndex); + const lineEnd = lineStart + lines[lineIndex]!.length; + if (cursorPos < lineEnd) { + onChange(value.slice(0, cursorPos) + value.slice(lineEnd)); + } else if (cursorPos < value.length) { + onChange(value.slice(0, cursorPos) + value.slice(cursorPos + 1)); + } + return true; + } + + // Ctrl+U - delete to start of line + if (key.ctrl && input === 'u') { + const lineStart = getLineStart(lineIndex); + if (cursorPos > lineStart) { + onChange(value.slice(0, lineStart) + value.slice(cursorPos)); + setCursorPos(lineStart); + } + return true; + } + + // Ctrl+W - delete word + if (key.ctrl && input === 'w') { + if (cursorPos > 0) { + let wordStart = cursorPos - 1; + while (wordStart > 0 && value[wordStart] === ' ') wordStart--; + while ( + wordStart > 0 && + value[wordStart - 1] !== ' ' && + value[wordStart - 1] !== '\n' + ) { + wordStart--; + } + onChange(value.slice(0, wordStart) + value.slice(cursorPos)); + setCursorPos(wordStart); + } + return true; + } + + // Regular character input + if (input && !key.ctrl && !key.meta) { + const newValue = value.slice(0, cursorPos) + input + value.slice(cursorPos); + onChange(newValue); + setCursorPos(cursorPos + input.length); + return true; + } + + return false; + }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useKeyboardShortcuts.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useKeyboardShortcuts.ts new file mode 100644 index 00000000..33817c06 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useKeyboardShortcuts.ts @@ -0,0 +1,103 @@ +/** + * Hook for global keyboard shortcuts + * Handles shortcuts like Ctrl+C (with double-press to exit), Escape, etc. + */ + +import type React from 'react'; +import { useEffect, useRef } from 'react'; +import { useInput, useApp } from 'ink'; +import type { DextoAgent } from '@dexto/core'; +import type { CLIAction } from '../state/actions.js'; +import type { CLIState } from '../state/types.js'; + +interface UseKeyboardShortcutsProps { + state: CLIState; + dispatch: React.Dispatch<CLIAction>; + agent: DextoAgent; +} + +/** Time window for double Ctrl+C to exit (in milliseconds) */ +const EXIT_WARNING_TIMEOUT = 3000; + +/** + * Manages global keyboard shortcuts + * - Ctrl+C: Cancel processing (if running) or show exit warning (press again to exit) + * - Escape: Cancel processing or close overlays + */ +export function useKeyboardShortcuts({ state, dispatch, agent }: UseKeyboardShortcutsProps): void { + const { exit } = useApp(); + + // Use ref for session.id to avoid stale closures in async operations + const sessionIdRef = useRef(state.session.id); + useEffect(() => { + sessionIdRef.current = state.session.id; + }, [state.session.id]); + + // Auto-clear exit warning after timeout + useEffect(() => { + if (!state.ui.exitWarningShown || !state.ui.exitWarningTimestamp) return; + + const elapsed = Date.now() - state.ui.exitWarningTimestamp; + const remaining = EXIT_WARNING_TIMEOUT - elapsed; + + if (remaining <= 0) { + dispatch({ type: 'EXIT_WARNING_CLEAR' }); + return; + } + + const timer = setTimeout(() => { + dispatch({ type: 'EXIT_WARNING_CLEAR' }); + }, remaining); + + return () => clearTimeout(timer); + }, [state.ui.exitWarningShown, state.ui.exitWarningTimestamp, dispatch]); + + useInput( + (inputChar, key) => { + // Don't intercept if approval prompt is active (it handles its own keys) + if (state.approval) { + return; + } + + // Don't intercept if autocomplete/selector is active (they handle their own keys) + if (state.ui.activeOverlay !== 'none' && state.ui.activeOverlay !== 'approval') { + return; + } + + // Ctrl+C: Exit only (with double-press safety) + // Use Escape to cancel processing + if (key.ctrl && inputChar === 'c') { + if (state.ui.exitWarningShown) { + // Second Ctrl+C within timeout - actually exit + exit(); + } else { + // First Ctrl+C - show warning + dispatch({ type: 'EXIT_WARNING_SHOW' }); + } + return; + } + + // Escape: Cancel processing or close overlay + if (key.escape) { + // Clear exit warning if shown + if (state.ui.exitWarningShown) { + dispatch({ type: 'EXIT_WARNING_CLEAR' }); + return; + } + + if (state.ui.isProcessing) { + const currentSessionId = sessionIdRef.current; + if (!currentSessionId) { + return; + } + void agent.cancel(currentSessionId).catch(() => {}); + dispatch({ type: 'CANCEL_START' }); + } else if (state.ui.activeOverlay !== 'none') { + dispatch({ type: 'CLOSE_OVERLAY' }); + } + } + }, + // Always active - we handle guards internally for more reliable behavior + { isActive: true } + ); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useKeypress.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useKeypress.ts new file mode 100644 index 00000000..5cf3aa31 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useKeypress.ts @@ -0,0 +1,33 @@ +/** + * useKeypress Hook + * + * Subscribe to keyboard events from the KeypressProvider. + * This replaces Ink's useInput hook. + */ + +import { useEffect } from 'react'; +import type { KeypressHandler, Key } from '../contexts/KeypressContext.js'; +import { useKeypressContext } from '../contexts/KeypressContext.js'; + +export type { Key }; + +/** + * Hook to subscribe to keypress events. + * + * @param onKeypress - Callback for each keypress + * @param options.isActive - Whether to listen for input + */ +export function useKeypress(onKeypress: KeypressHandler, { isActive }: { isActive: boolean }) { + const { subscribe, unsubscribe } = useKeypressContext(); + + useEffect(() => { + if (!isActive) { + return; + } + + subscribe(onKeypress); + return () => { + unsubscribe(onKeypress); + }; + }, [isActive, onKeypress, subscribe, unsubscribe]); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/usePhraseCycler.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/usePhraseCycler.ts new file mode 100644 index 00000000..383c7d1b --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/usePhraseCycler.ts @@ -0,0 +1,96 @@ +/** + * usePhraseCycler Hook + * Cycles through processing phrases and tips at regular intervals + * + * Behavior: + * - 1/3 chance to show tip, 2/3 chance for witty phrase + * - Phrases cycle every 8 seconds while isActive is true + */ + +import { useState, useEffect, useCallback, useRef } from 'react'; +import { getRandomPhrase } from '../constants/processingPhrases.js'; +import { getRandomTip } from '../constants/tips.js'; + +export interface PhraseCyclerOptions { + /** Whether the cycler is active (should only run during processing) */ + isActive: boolean; + /** Interval in milliseconds between phrase changes (default: 8000ms = 8 seconds) */ + intervalMs?: number; + /** Disable tips (only show witty phrases) */ + disableTips?: boolean; +} + +export interface PhraseCyclerResult { + /** Current phrase to display */ + phrase: string; + /** Manually trigger a new phrase */ + nextPhrase: () => void; +} + +/** + * Get a random phrase or tip based on probability + * 1/3 chance tip, 2/3 chance phrase + */ +function getRandomPhraseOrTip(disableTips: boolean = false): string { + if (disableTips) { + return getRandomPhrase(); + } + + // 1/3 chance to show a tip (roughly 33%) + const showTip = Math.random() < 1 / 3; + return showTip ? getRandomTip() : getRandomPhrase(); +} + +/** + * Hook that cycles through witty processing phrases and informative tips + * + * @param options - Configuration options + * @returns Current phrase and control functions + */ +export function usePhraseCycler({ + isActive, + intervalMs = 8000, + disableTips = false, +}: PhraseCyclerOptions): PhraseCyclerResult { + const [phrase, setPhrase] = useState(() => getRandomPhraseOrTip(disableTips)); + const intervalRef = useRef<NodeJS.Timeout | null>(null); + + const nextPhrase = useCallback(() => { + // Get a new phrase that's different from current + let newPhrase = getRandomPhraseOrTip(disableTips); + // Avoid showing the same phrase twice in a row + let attempts = 0; + while (newPhrase === phrase && attempts < 3) { + newPhrase = getRandomPhraseOrTip(disableTips); + attempts++; + } + setPhrase(newPhrase); + }, [phrase, disableTips]); + + useEffect(() => { + if (isActive) { + // Set initial phrase when becoming active + setPhrase(getRandomPhraseOrTip(disableTips)); + + // Start cycling + intervalRef.current = setInterval(() => { + setPhrase(getRandomPhraseOrTip(disableTips)); + }, intervalMs); + } else { + // Clear interval when inactive + if (intervalRef.current) { + clearInterval(intervalRef.current); + intervalRef.current = null; + } + } + + return () => { + if (intervalRef.current) { + clearInterval(intervalRef.current); + intervalRef.current = null; + } + }; + }, [isActive, intervalMs, disableTips]); + + return { phrase, nextPhrase }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useStreaming.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useStreaming.ts new file mode 100644 index 00000000..3503a27c --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useStreaming.ts @@ -0,0 +1,45 @@ +/** + * React hook for streaming state + * + * Subscribes to the streaming state manager and returns current value. + * Components using this hook will re-render when streaming is toggled. + */ + +import { useState, useEffect } from 'react'; +import { + isStreamingEnabled, + subscribeToStreaming, + setStreamingEnabled, + toggleStreaming, +} from '../state/streaming-state.js'; + +export interface UseStreamingResult { + /** Current streaming state */ + streaming: boolean; + /** Set streaming state */ + setStreaming: (enabled: boolean) => void; + /** Toggle streaming state */ + toggleStreaming: () => boolean; +} + +/** + * Hook to access and modify streaming state + */ +export function useStreaming(): UseStreamingResult { + const [streaming, setStreamingState] = useState(isStreamingEnabled); + + useEffect(() => { + // Subscribe to changes from command or other sources + const unsubscribe = subscribeToStreaming((enabled) => { + setStreamingState(enabled); + }); + + return unsubscribe; + }, []); + + return { + streaming, + setStreaming: setStreamingEnabled, + toggleStreaming, + }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useTerminalSize.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useTerminalSize.ts new file mode 100644 index 00000000..826bb834 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useTerminalSize.ts @@ -0,0 +1,43 @@ +/** + * useTerminalSize Hook + * + * Listens to terminal resize events and provides current dimensions. + */ + +import { useState, useEffect } from 'react'; + +export interface TerminalSize { + columns: number; + rows: number; +} + +/** + * Hook that returns current terminal size and updates on resize + */ +export function useTerminalSize(): TerminalSize { + const [size, setSize] = useState<TerminalSize>({ + columns: process.stdout.columns || 80, + rows: process.stdout.rows || 24, + }); + + useEffect(() => { + function updateSize() { + setSize({ + columns: process.stdout.columns || 80, + rows: process.stdout.rows || 24, + }); + } + + // Listen for resize events + process.stdout.on('resize', updateSize); + + // Initial update in case size changed between render and effect + updateSize(); + + return () => { + process.stdout.off('resize', updateSize); + }; + }, []); + + return size; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/hooks/useTokenCounter.ts b/dexto/packages/cli/src/cli/ink-cli/hooks/useTokenCounter.ts new file mode 100644 index 00000000..141a2191 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/hooks/useTokenCounter.ts @@ -0,0 +1,160 @@ +/** + * useTokenCounter Hook + * Tracks token usage during streaming with live estimates + * + * Accumulation strategy for multi-step turns: + * - lastInputTokens: Input tokens from the most recent LLM call (not summed to avoid double-counting) + * - cumulativeOutputTokens: Sum of output tokens across all LLM calls in the turn + * - currentSegmentEstimate: Estimated tokens for current streaming segment + * - Display = lastInput + cumulativeOutput + currentEstimate + * + * This approach avoids double-counting shared context (system prompt, history) across + * multiple LLM calls in a single turn while accurately capturing output generation. + * + * For queued messages: Tokens continue accumulating (same turn) + * Reset only when isActive transitions false→true (new turn) + */ + +import { useState, useEffect, useRef } from 'react'; +import type { DextoAgent } from '@dexto/core'; + +export interface TokenCounterOptions { + /** DextoAgent instance for event bus access */ + agent: DextoAgent; + /** Whether counting is active (should only run during processing) */ + isActive: boolean; +} + +export interface TokenCounterResult { + /** Total actual tokens (lastInput + cumulativeOutput) */ + totalActualTokens: number; + /** Estimated tokens for current streaming segment */ + currentSegmentEstimate: number; + /** Combined display count (actual + current estimate) */ + displayCount: number; + /** Whether the display includes an estimate component */ + includesEstimate: boolean; + /** Formatted display string (e.g., "~125 tokens" or "125 tokens") */ + formatted: string; +} + +/** + * Estimate tokens from character count + * Uses ~4 characters per token as a rough approximation + * This matches common tokenizer behavior for English text + */ +function estimateTokens(charCount: number): number { + return Math.ceil(charCount / 4); +} + +/** + * Format token count for display + * Only shows count when >= 1000, using x.xK format + */ +function formatTokenCount(count: number, includesEstimate: boolean): string { + if (count < 1000) return ''; + const prefix = includesEstimate ? '~' : ''; + const kValue = (count / 1000).toFixed(1); + return `${prefix}${kValue}K tokens`; +} + +/** + * Hook that tracks token usage during LLM streaming + * + * Tracks tokens across multi-step turns (text → tool → text → tool) + * using: lastInputTokens + cumulativeOutputTokens to avoid double-counting. + * + * @param options - Configuration options + * @returns Token counts (actual + current segment estimate) + */ +export function useTokenCounter({ agent, isActive }: TokenCounterOptions): TokenCounterResult { + // Input tokens from the most recent LLM response (replaced, not summed) + const [lastInputTokens, setLastInputTokens] = useState(0); + // Cumulative output tokens across all LLM responses in this turn + const [cumulativeOutputTokens, setCumulativeOutputTokens] = useState(0); + // Estimated tokens for current streaming segment (resets after each response) + const [currentSegmentEstimate, setCurrentSegmentEstimate] = useState(0); + // Character count for current segment (ref to avoid re-renders on each chunk) + const currentCharCountRef = useRef(0); + + useEffect(() => { + if (!isActive) { + // Reset when turn ends (isActive becomes false) + setLastInputTokens(0); + setCumulativeOutputTokens(0); + setCurrentSegmentEstimate(0); + currentCharCountRef.current = 0; + return; + } + + const bus = agent.agentEventBus; + const controller = new AbortController(); + const { signal } = controller; + + // Reset on new turn (isActive just became true) + currentCharCountRef.current = 0; + setLastInputTokens(0); + setCumulativeOutputTokens(0); + setCurrentSegmentEstimate(0); + + // Track streaming chunks - accumulate estimate for current segment + bus.on( + 'llm:chunk', + (payload) => { + if (payload.chunkType === 'text') { + currentCharCountRef.current += payload.content.length; + setCurrentSegmentEstimate(estimateTokens(currentCharCountRef.current)); + } + }, + { signal } + ); + + // On response: update input (replace), accumulate output, reset estimate + bus.on( + 'llm:response', + (payload) => { + const usage = payload.tokenUsage; + if (usage) { + // Replace input tokens (most recent call's context) + // Subtract cacheWriteTokens to exclude system prompt on first call + const rawInputTokens = usage.inputTokens ?? 0; + const cacheWriteTokens = usage.cacheWriteTokens ?? 0; + const inputTokens = Math.max(0, rawInputTokens - cacheWriteTokens); + if (inputTokens > 0) { + setLastInputTokens(inputTokens); + } + // Accumulate output tokens (additive across calls) + const outputTokens = usage.outputTokens ?? 0; + if (outputTokens > 0) { + setCumulativeOutputTokens((prev) => prev + outputTokens); + } + } + // Reset current segment for next streaming segment + currentCharCountRef.current = 0; + setCurrentSegmentEstimate(0); + }, + { signal } + ); + + // Note: No reset on llm:thinking - queued messages continue the same turn + // Reset only happens when isActive transitions (new user-initiated turn) + + return () => { + controller.abort(); + }; + }, [agent, isActive]); + + // Total = lastInput + cumulativeOutput (avoids double-counting shared context) + const totalActualTokens = lastInputTokens + cumulativeOutputTokens; + // Display = actual + current streaming estimate + const displayCount = totalActualTokens + currentSegmentEstimate; + const includesEstimate = currentSegmentEstimate > 0; + + return { + totalActualTokens, + currentSegmentEstimate, + displayCount, + includesEstimate, + formatted: formatTokenCount(displayCount, includesEstimate), + }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/services/CommandService.ts b/dexto/packages/cli/src/cli/ink-cli/services/CommandService.ts new file mode 100644 index 00000000..a631bb66 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/services/CommandService.ts @@ -0,0 +1,114 @@ +/** + * Command execution service + * Handles command parsing and execution + */ + +import type { DextoAgent } from '@dexto/core'; +import { parseInput } from '../utils/inputParsing.js'; +import { executeCommand } from '../../commands/interactive-commands/commands.js'; +import type { CommandResult } from '../../commands/interactive-commands/command-parser.js'; +import type { StyledMessageType, StyledData } from '../state/types.js'; + +/** + * Styled output for command execution + */ +export interface StyledOutput { + styledType: StyledMessageType; + styledData: StyledData; + fallbackText: string; // Plain text fallback for logging/history +} + +/** + * Result of command execution + */ +export interface CommandExecutionResult { + type: 'handled' | 'output' | 'styled' | 'sendMessage'; + output?: string; + styled?: StyledOutput; + /** Message text to send through normal streaming flow (for prompt commands) */ + messageToSend?: string; +} + +/** + * Check if a result is a styled output + */ +export function isStyledOutput(result: unknown): result is StyledOutput { + return ( + typeof result === 'object' && + result !== null && + 'styledType' in result && + 'styledData' in result && + 'fallbackText' in result + ); +} + +/** + * Marker object for commands that want to send a message through the normal stream flow + */ +export interface SendMessageMarker { + __sendMessage: true; + text: string; +} + +/** + * Create a send message marker (used by prompt commands) + */ +export function createSendMessageMarker(text: string): SendMessageMarker { + return { __sendMessage: true, text }; +} + +/** + * Check if a result is a send message marker + */ +export function isSendMessageMarker(result: unknown): result is SendMessageMarker { + return ( + typeof result === 'object' && + result !== null && + '__sendMessage' in result && + (result as SendMessageMarker).__sendMessage === true && + 'text' in result && + typeof (result as SendMessageMarker).text === 'string' + ); +} + +/** + * Service for executing commands + */ +export class CommandService { + /** + * Parses input and determines if it's a command or prompt + */ + parseInput(input: string): CommandResult { + return parseInput(input); + } + + /** + * Executes a command and returns the result + */ + async executeCommand( + command: string, + args: string[], + agent: DextoAgent, + sessionId?: string + ): Promise<CommandExecutionResult> { + const result = await executeCommand(command, args, agent, sessionId); + + // If result is a send message marker, return the text to send through normal flow + if (isSendMessageMarker(result)) { + return { type: 'sendMessage' as const, messageToSend: result.text }; + } + + // If result is a string, it's output for display + if (typeof result === 'string') { + return { type: 'output', output: result }; + } + + // If result is a styled output object + if (isStyledOutput(result)) { + return { type: 'styled', styled: result }; + } + + // If result is boolean, command was handled + return { type: 'handled' }; + } +} diff --git a/dexto/packages/cli/src/cli/ink-cli/services/InputService.ts b/dexto/packages/cli/src/cli/ink-cli/services/InputService.ts new file mode 100644 index 00000000..bb80df5f --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/services/InputService.ts @@ -0,0 +1,104 @@ +/** + * Input management service + * Handles input detection and manipulation + */ + +import { + detectAutocompleteType, + extractSlashQuery, + extractResourceQuery, + parseInput, +} from '../utils/inputParsing.js'; +import type { AutocompleteType } from '../utils/inputParsing.js'; +import type { CommandResult } from '../../commands/interactive-commands/command-parser.js'; + +/** + * Service for managing input + */ +export class InputService { + /** + * Detects what type of autocomplete should be shown + */ + detectAutocompleteType(input: string): AutocompleteType { + return detectAutocompleteType(input); + } + + /** + * Extracts slash command query + */ + extractSlashQuery(input: string): string { + return extractSlashQuery(input); + } + + /** + * Extracts resource mention query + */ + extractResourceQuery(input: string): string { + return extractResourceQuery(input); + } + + /** + * Parses input + */ + parseInput(input: string): CommandResult { + return parseInput(input); + } + + /** + * Deletes word backward from cursor position + */ + deleteWordBackward(text: string, cursorPos: number = text.length): string { + if (cursorPos === 0) return text; + + let pos = cursorPos - 1; + + // Skip whitespace + while (pos >= 0) { + const char = text[pos]; + if (char && !/\s/.test(char)) break; + pos--; + } + + // Skip word characters + while (pos >= 0) { + const char = text[pos]; + if (char && /\s/.test(char)) break; + pos--; + } + + const deleteStart = pos + 1; + return text.slice(0, deleteStart) + text.slice(cursorPos); + } + + /** + * Deletes word forward from cursor position + */ + deleteWordForward(text: string, cursorPos: number = text.length): string { + if (cursorPos >= text.length) return text; + + let pos = cursorPos; + + // Skip whitespace + while (pos < text.length) { + const char = text[pos]; + if (char && !/\s/.test(char)) break; + pos++; + } + + // Skip word characters + while (pos < text.length) { + const char = text[pos]; + if (char && /\s/.test(char)) break; + pos++; + } + + return text.slice(0, cursorPos) + text.slice(pos); + } + + /** + * Deletes entire line (for single-line input) + */ + deleteLine(_text: string): string { + return ''; + } +} diff --git a/dexto/packages/cli/src/cli/ink-cli/services/MessageService.ts b/dexto/packages/cli/src/cli/ink-cli/services/MessageService.ts new file mode 100644 index 00000000..19d27cba --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/services/MessageService.ts @@ -0,0 +1,63 @@ +/** + * Message management service + * Handles message creation and formatting + */ + +import type { Message } from '../state/types.js'; +import { + createUserMessage, + createSystemMessage, + createErrorMessage, + createToolMessage, + createStreamingMessage, +} from '../utils/messageFormatting.js'; + +/** + * Service for managing messages + */ +export class MessageService { + /** + * Creates a user message + */ + createUserMessage(content: string): Message { + return createUserMessage(content); + } + + /** + * Creates a system message + */ + createSystemMessage(content: string): Message { + return createSystemMessage(content); + } + + /** + * Creates an error message + */ + createErrorMessage(error: Error | string): Message { + return createErrorMessage(error); + } + + /** + * Creates a tool call message + */ + createToolMessage(toolName: string): Message { + return createToolMessage(toolName); + } + + /** + * Creates a streaming placeholder message + */ + createStreamingMessage(): Message { + return createStreamingMessage(); + } + + /** + * Gets visible messages (for performance - limit to recent messages) + */ + getVisibleMessages(messages: Message[], limit: number = 50): Message[] { + if (limit <= 0) { + return []; + } + return messages.slice(-limit); + } +} diff --git a/dexto/packages/cli/src/cli/ink-cli/services/index.ts b/dexto/packages/cli/src/cli/ink-cli/services/index.ts new file mode 100644 index 00000000..c23fb7e2 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/services/index.ts @@ -0,0 +1,18 @@ +/** + * Services module exports + */ + +export { + CommandService, + createSendMessageMarker, + type CommandExecutionResult, + type SendMessageMarker, + type StyledOutput, +} from './CommandService.js'; +export { MessageService } from './MessageService.js'; +export { InputService } from './InputService.js'; +export { + processStream, + type ProcessStreamSetters, + type ProcessStreamOptions, +} from './processStream.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/services/processStream.ts b/dexto/packages/cli/src/cli/ink-cli/services/processStream.ts new file mode 100644 index 00000000..ead10531 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/services/processStream.ts @@ -0,0 +1,1094 @@ +/** + * Process Stream Service + * + * Processes the async iterator from agent.stream() and updates UI state. + * This replaces the event bus subscriptions for streaming events, + * providing direct, synchronous control over the streaming lifecycle. + * + * Architecture: + * - Messages being streamed are tracked in `pendingMessages` (rendered dynamically) + * - Only finalized messages are added to `messages` (rendered in <Static>) + * - Progressive finalization: large streaming content is split at safe markdown + * boundaries, moving completed paragraphs to Static to reduce flickering + * - This prevents duplicate output in static terminal mode + * + * IMPORTANT: React batching fix (see commit history for race condition details) + * - We use a local `localPending` array that mirrors React state synchronously + * - This allows us to flatten nested setState calls (which caused ordering bugs) + * - Nested setState: inner setMessages inside setPendingMessages callback gets + * queued and runs AFTER other setMessages calls in the same batch + * - Flattened: setMessages and setPendingMessages are sibling calls, processed in order + */ + +import type React from 'react'; +import type { StreamingEvent, SanitizedToolResult } from '@dexto/core'; +import { createDebugLogger } from '../utils/debugLog.js'; +import { ApprovalType as ApprovalTypeEnum, ApprovalStatus } from '@dexto/core'; +import type { Message, UIState, ToolStatus } from '../state/types.js'; +import type { ApprovalRequest } from '../components/ApprovalPrompt.js'; +import { generateMessageId } from '../utils/idGenerator.js'; +import { checkForSplit } from '../utils/streamSplitter.js'; +import { formatToolHeader } from '../utils/messageFormatting.js'; +import { isAutoApprovableInEditMode } from '../utils/toolUtils.js'; +import { capture } from '../../../analytics/index.js'; +import chalk from 'chalk'; + +/** + * Build error message with recovery guidance if available + */ +function buildErrorContent(error: unknown, prefix: string): string { + const errorMessage = error instanceof Error ? error.message : String(error); + let errorContent = `${prefix}${errorMessage}`; + + // Add recovery guidance if available (for DextoRuntimeError) + if (error instanceof Error && 'recovery' in error && error.recovery) { + const recoveryMessages = Array.isArray(error.recovery) ? error.recovery : [error.recovery]; + errorContent += '\n\n' + recoveryMessages.map((msg) => `💡 ${msg}`).join('\n'); + } + + return errorContent; +} + +/** + * State setters needed by processStream + */ +export interface ProcessStreamSetters { + /** Setter for finalized messages (rendered in <Static>) */ + setMessages: React.Dispatch<React.SetStateAction<Message[]>>; + /** Setter for pending/streaming messages (rendered dynamically outside <Static>) */ + setPendingMessages: React.Dispatch<React.SetStateAction<Message[]>>; + /** Setter for dequeued buffer (user messages waiting to render after pending) */ + setDequeuedBuffer: React.Dispatch<React.SetStateAction<Message[]>>; + setUi: React.Dispatch<React.SetStateAction<UIState>>; + /** Setter for session state (for session switch on compaction) */ + setSession: React.Dispatch<React.SetStateAction<import('../state/types.js').SessionState>>; + /** Setter for queued messages (cleared when dequeued) */ + setQueuedMessages: React.Dispatch<React.SetStateAction<import('@dexto/core').QueuedMessage[]>>; + /** Setter for current approval request (for approval UI) */ + setApproval: React.Dispatch<React.SetStateAction<ApprovalRequest | null>>; + /** Setter for approval queue (for queued approvals) */ + setApprovalQueue: React.Dispatch<React.SetStateAction<ApprovalRequest[]>>; +} + +/** + * Options for processStream + */ +export interface ProcessStreamOptions { + /** Whether to stream chunks (true) or wait for complete response (false). Default: true */ + useStreaming?: boolean; + /** Ref to check if "accept all edits" mode is enabled (reads .current for latest value) */ + autoApproveEditsRef: { current: boolean }; + /** Event bus for emitting auto-approval responses */ + eventBus: import('@dexto/core').AgentEventBus; + /** Sound notification service for playing sounds on events */ + soundService?: import('../utils/soundNotification.js').SoundNotificationService; + /** Optional setter for todos (from service:event todo updates) */ + setTodos?: React.Dispatch<React.SetStateAction<import('../state/types.js').TodoItem[]>>; +} + +/** + * Internal state for tracking the current streaming message + */ +interface StreamState { + messageId: string | null; + content: string; + /** Input tokens from most recent LLM response (replaced, not summed) */ + lastInputTokens: number; + /** Cumulative output tokens across all LLM responses in this turn */ + cumulativeOutputTokens: number; + /** Content that has been finalized (moved to Static) */ + finalizedContent: string; + /** Counter for generating unique IDs for split messages */ + splitCounter: number; + /** Flag to track if text was finalized early (before tools) to avoid duplication */ + textFinalizedBeforeTool: boolean; + /** + * Accumulated text in non-streaming mode. + * In non-streaming mode, we don't update UI on each chunk, but we need to track + * the text so we can add it BEFORE tool calls for correct message ordering. + */ + nonStreamingAccumulatedText: string; +} + +/** + * Processes the async iterator from agent.stream() and updates UI state. + * + * For static mode compatibility: + * - Streaming content goes to `pendingMessages` (rendered dynamically) + * - Finalized content is moved to `messages` (rendered in <Static>) + * + * @param iterator - The async iterator from agent.stream() + * @param setters - State setters for updating UI + * @param options - Configuration options + */ +export async function processStream( + iterator: AsyncIterableIterator<StreamingEvent>, + setters: ProcessStreamSetters, + options: ProcessStreamOptions +): Promise<void> { + const { + setMessages, + setPendingMessages, + setDequeuedBuffer, + setUi, + setSession: _setSession, + setQueuedMessages, + setApproval, + setApprovalQueue, + } = setters; + const useStreaming = options?.useStreaming ?? true; + + // Track streaming state (synchronous, not React state) + const state: StreamState = { + messageId: null, + content: '', + lastInputTokens: 0, + cumulativeOutputTokens: 0, + finalizedContent: '', + splitCounter: 0, + textFinalizedBeforeTool: false, + nonStreamingAccumulatedText: '', + }; + + // LOCAL PENDING TRACKING - mirrors React state synchronously + // This allows us to flatten nested setState calls (which caused ordering bugs). + // See: https://github.com/facebook/react/issues/8132 - nested setState not supported + let localPending: Message[] = []; + + /** + * Extract text content from ContentPart array + */ + const extractTextContent = (content: import('@dexto/core').ContentPart[]): string => { + return content + .filter((part): part is { type: 'text'; text: string } => part.type === 'text') + .map((part) => part.text) + .join('\n'); + }; + + /** + * Move a message from pending to finalized. + * FLATTENED: Uses localPending to avoid nested setState (which breaks ordering). + */ + const finalizeMessage = (messageId: string, updates: Partial<Message> = {}) => { + const msg = localPending.find((m) => m.id === messageId); + if (msg) { + // Add to messages FIRST (sibling call, not nested) + setMessages((prev) => [...prev, { ...msg, ...updates }]); + } + // Update local tracking + localPending = localPending.filter((m) => m.id !== messageId); + // Then update React state (sibling call) + setPendingMessages(localPending); + }; + + /** + * Move all pending messages to finalized (used at run:complete and message:dequeued). + * FLATTENED: Uses localPending to avoid nested setState. + */ + const finalizeAllPending = () => { + if (localPending.length > 0) { + // Add to messages FIRST (sibling call, not nested) + const toFinalize = [...localPending]; + setMessages((prev) => [...prev, ...toFinalize]); + } + // Update local tracking + localPending = []; + // Then update React state (sibling call) + setPendingMessages([]); + }; + + /** + * Move dequeued buffer to messages (called at start of new run) + * This ensures user messages appear in correct order after previous response + * NOTE: This still uses nested setState but dequeuedBuffer is separate from + * the main message flow and only flushed at llm:thinking (start of run) + */ + const flushDequeuedBuffer = () => { + setDequeuedBuffer((buffer) => { + if (buffer.length > 0) { + setMessages((prev) => [...prev, ...buffer]); + } + return []; + }); + }; + + /** + * Add message to pending (updates both local tracking and React state) + */ + const addToPending = (msg: Message) => { + localPending = [...localPending, msg]; + setPendingMessages(localPending); + }; + + /** + * Update a message in pending (updates both local tracking and React state) + */ + const updatePending = (messageId: string, updates: Partial<Message>) => { + localPending = localPending.map((m) => (m.id === messageId ? { ...m, ...updates } : m)); + setPendingMessages(localPending); + }; + + /** + * Remove a message from pending without finalizing (updates both local and React state) + */ + const removeFromPending = (messageId: string) => { + localPending = localPending.filter((m) => m.id !== messageId); + setPendingMessages(localPending); + }; + + /** + * Clear all pending (updates both local tracking and React state) + */ + const clearPending = () => { + localPending = []; + setPendingMessages([]); + }; + + /** + * Update toolStatus for a pending message by ID + * Used for tool status transitions: pending → pending_approval → running → finished + */ + const updatePendingStatus = (messageId: string, status: ToolStatus) => { + localPending = localPending.map((msg) => + msg.id === messageId ? { ...msg, toolStatus: status } : msg + ); + setPendingMessages(localPending); + }; + + /** + * Progressive finalization: split large streaming content at safe markdown + * boundaries and move completed portions to Static to reduce flickering. + * + * Safe to use with message queueing because dequeued user messages are + * rendered in a separate buffer AFTER pendingMessages, guaranteeing + * correct visual order regardless of React batching timing. + * + * RACE CONDITION FIX: We clear the pending message content BEFORE adding + * the split, then restore with afterContent. This ensures any intermediate + * render sees empty pending (not stale full content), avoiding duplication. + */ + const progressiveFinalize = (content: string): string => { + const splitResult = checkForSplit(content); + + if (splitResult.shouldSplit && splitResult.before && splitResult.after !== undefined) { + // Add the completed portion directly to finalized messages + state.splitCounter++; + const splitId = `${state.messageId}-split-${state.splitCounter}`; + const beforeContent = splitResult.before; + const afterContent = splitResult.after; + const isFirstSplit = state.splitCounter === 1; + + // STEP 1: Clear pending message content to avoid showing stale content + // during React's batched render cycle + if (state.messageId) { + localPending = localPending.map((m) => + m.id === state.messageId ? { ...m, content: '', isContinuation: true } : m + ); + setPendingMessages(localPending); + } + + // STEP 2: Add split message to finalized + setMessages((prev) => [ + ...prev, + { + id: splitId, + role: 'assistant' as const, + content: beforeContent, + timestamp: new Date(), + isStreaming: false, + // First split shows the indicator, subsequent splits are continuations + isContinuation: !isFirstSplit, + }, + ]); + + // STEP 3: Restore pending with afterContent + if (state.messageId) { + localPending = localPending.map((m) => + m.id === state.messageId ? { ...m, content: afterContent } : m + ); + setPendingMessages(localPending); + } + + // Track total finalized content for final message assembly + state.finalizedContent += beforeContent; + + // Return only the remaining content for pending + return afterContent; + } + + return content; + }; + + // Debug logging: enable via DEXTO_DEBUG_STREAM=true + const debug = createDebugLogger('stream'); + debug.reset(); + debug.log('CONFIG', { useStreaming }); + + try { + for await (const event of iterator) { + debug.log(`EVENT: ${event.name}`, { + ...(event.name === 'llm:chunk' && + 'chunkType' in event && { + chunkType: event.chunkType, + contentLen: event.content?.length, + }), + ...(event.name === 'llm:tool-call' && + 'toolName' in event && { + toolName: event.toolName, + }), + }); + + switch (event.name) { + case 'llm:thinking': { + debug.log('THINKING: resetting state', { + prevMessageId: state.messageId, + prevContentLen: state.content.length, + }); + // Flush dequeued buffer to messages at start of new run + // This ensures user messages appear after the previous response + flushDequeuedBuffer(); + + // Start thinking state, reset streaming state + setUi((prev) => ({ ...prev, isThinking: true })); + state.messageId = null; + state.content = ''; + state.lastInputTokens = 0; + state.cumulativeOutputTokens = 0; + state.finalizedContent = ''; + state.splitCounter = 0; + state.textFinalizedBeforeTool = false; + state.nonStreamingAccumulatedText = ''; + break; + } + + case 'llm:chunk': { + // In non-streaming mode, accumulate text but don't update UI + // We need to track text so we can add it BEFORE tool calls (ordering fix) + if (!useStreaming) { + if (event.chunkType === 'text') { + state.nonStreamingAccumulatedText += event.content; + debug.log('CHUNK (non-stream): accumulated', { + chunkLen: event.content?.length, + totalLen: state.nonStreamingAccumulatedText.length, + preview: state.nonStreamingAccumulatedText.slice(0, 50), + }); + } + break; + } + + // End thinking state when first chunk arrives + setUi((prev) => ({ ...prev, isThinking: false })); + + if (event.chunkType === 'text') { + debug.log('CHUNK (stream): text', { + hasMessageId: !!state.messageId, + chunkLen: event.content?.length, + currentContentLen: state.content.length, + preview: event.content?.slice(0, 30), + }); + // Create streaming message on first text chunk + if (!state.messageId) { + const newId = generateMessageId('assistant'); + state.messageId = newId; + state.content = event.content; + state.finalizedContent = ''; + state.splitCounter = 0; + + // Add to PENDING (not messages) - renders dynamically + addToPending({ + id: newId, + role: 'assistant', + content: event.content, + timestamp: new Date(), + isStreaming: true, + }); + } else { + // Accumulate content + state.content += event.content; + + // Check for progressive finalization (move completed paragraphs to Static) + // progressiveFinalize updates pending message internally when split occurs + const pendingContent = progressiveFinalize(state.content); + const splitOccurred = pendingContent !== state.content; + + // Update state with remaining content + state.content = pendingContent; + + // Only update pending if no split occurred (split already handled by progressiveFinalize) + if (!splitOccurred) { + const messageId = state.messageId; + // Mark as continuation if we've had any splits + const isContinuation = state.splitCounter > 0; + updatePending(messageId, { + content: pendingContent, + isContinuation, + }); + } + } + } + break; + } + + case 'llm:response': { + // In non-streaming mode, end thinking state when response arrives + // (In streaming mode, thinking ends when first chunk arrives) + if (!useStreaming) { + setUi((prev) => ({ ...prev, isThinking: false })); + } + + // Track token usage: replace input (last context), accumulate output + // Subtract cacheWriteTokens to exclude system prompt on first call + if (event.tokenUsage) { + const rawInputTokens = event.tokenUsage.inputTokens ?? 0; + const cacheWriteTokens = event.tokenUsage.cacheWriteTokens ?? 0; + const inputTokens = Math.max(0, rawInputTokens - cacheWriteTokens); + if (inputTokens > 0) { + state.lastInputTokens = inputTokens; + } + if (event.tokenUsage.outputTokens) { + state.cumulativeOutputTokens += event.tokenUsage.outputTokens; + } + } + + // Track token usage analytics + if ( + event.tokenUsage && + (event.tokenUsage.inputTokens || event.tokenUsage.outputTokens) + ) { + // Calculate estimate accuracy if both estimate and actual are available + let estimateAccuracyPercent: number | undefined; + if ( + event.estimatedInputTokens !== undefined && + event.tokenUsage.inputTokens + ) { + const diff = event.estimatedInputTokens - event.tokenUsage.inputTokens; + estimateAccuracyPercent = Math.round( + (diff / event.tokenUsage.inputTokens) * 100 + ); + } + + capture('dexto_llm_tokens_consumed', { + source: 'cli', + sessionId: event.sessionId, + provider: event.provider, + model: event.model, + inputTokens: event.tokenUsage.inputTokens, + outputTokens: event.tokenUsage.outputTokens, + reasoningTokens: event.tokenUsage.reasoningTokens, + totalTokens: event.tokenUsage.totalTokens, + cacheReadTokens: event.tokenUsage.cacheReadTokens, + cacheWriteTokens: event.tokenUsage.cacheWriteTokens, + estimatedInputTokens: event.estimatedInputTokens, + estimateAccuracyPercent, + }); + } + + const finalContent = event.content || ''; + + if (state.messageId) { + // Finalize existing streaming message (streaming mode) + const messageId = state.messageId; + const content = state.content || finalContent; + + // Move from pending to finalized + finalizeMessage(messageId, { content, isStreaming: false }); + + // Reset for potential next response (multi-step) + state.messageId = null; + state.content = ''; + } else if (finalContent && !state.textFinalizedBeforeTool) { + // No streaming message exists - add directly to finalized + // This handles: non-streaming mode, or multi-step turns after tool calls + // Skip if text was already finalized before tools (avoid duplication) + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('assistant'), + role: 'assistant', + content: finalContent, + timestamp: new Date(), + isStreaming: false, + }, + ]); + } + // Reset the flag for this response (new text after tools will create new message) + state.textFinalizedBeforeTool = false; + break; + } + + case 'llm:tool-call': { + debug.log('TOOL-CALL: state check', { + toolName: event.toolName, + hasMessageId: !!state.messageId, + contentLen: state.content.length, + nonStreamAccumLen: state.nonStreamingAccumulatedText.length, + contentPreview: state.content.slice(0, 50), + nonStreamPreview: state.nonStreamingAccumulatedText.slice(0, 50), + useStreaming, + }); + // ORDERING FIX: Add any accumulated text BEFORE adding tool + // This ensures text appears before tools in the message list. + + // Streaming mode: handle pending assistant message before tool + if (state.messageId) { + if (state.content) { + // Finalize pending message with content + const messageId = state.messageId; + const content = state.content; + const isContinuation = state.splitCounter > 0; + debug.log('TOOL-CALL: finalizing pending message', { + messageId, + contentLen: content.length, + }); + finalizeMessage(messageId, { + content, + isStreaming: false, + isContinuation, + }); + // Mark that we finalized text early - prevents duplicate in llm:response + state.textFinalizedBeforeTool = true; + } else { + // Empty pending message (first chunk had no content) - remove it + // This prevents empty bullets when LLM/SDK sends empty initial chunk + debug.log('TOOL-CALL: removing empty pending message', { + messageId: state.messageId, + }); + removeFromPending(state.messageId); + } + state.messageId = null; + state.content = ''; + } else { + debug.log('TOOL-CALL: no pending message to finalize'); + } + + // Non-streaming mode: add accumulated text as finalized message + if (!useStreaming && state.nonStreamingAccumulatedText) { + debug.log('TOOL-CALL: adding non-stream accumulated text', { + len: state.nonStreamingAccumulatedText.length, + }); + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('assistant'), + role: 'assistant', + content: state.nonStreamingAccumulatedText, + timestamp: new Date(), + isStreaming: false, + }, + ]); + state.nonStreamingAccumulatedText = ''; + // Mark that we finalized text early - prevents duplicate in llm:response + state.textFinalizedBeforeTool = true; + } + + const toolMessageId = event.callId + ? `tool-${event.callId}` + : generateMessageId('tool'); + + // Format tool header using shared utility + const { header: toolContent } = formatToolHeader( + event.toolName, + (event.args as Record<string, unknown>) || {} + ); + + // Add description if present (dim styling, on new line) + let finalToolContent = toolContent; + const description = event.args?.description; + if (description && typeof description === 'string') { + finalToolContent += `\n${chalk.dim(description)}`; + } + + // Tool calls start in 'pending' state (don't know if approval needed yet) + // Status transitions: pending → pending_approval (if approval needed) → running → finished + // Or for pre-approved: pending → running → finished + addToPending({ + id: toolMessageId, + role: 'tool', + content: finalToolContent, + timestamp: new Date(), + toolStatus: 'pending', + }); + + // Track tool called analytics + capture('dexto_tool_called', { + source: 'cli', + sessionId: event.sessionId, + toolName: event.toolName, + }); + break; + } + + case 'llm:tool-result': { + // Extract structured display data and content from sanitized result + const sanitized = event.sanitized as SanitizedToolResult | undefined; + const toolDisplayData = sanitized?.meta?.display; + const toolContent = sanitized?.content; + + // Generate text preview for fallback display + let resultPreview = ''; + try { + const result = event.sanitized || event.rawResult; + if (result) { + let resultStr = ''; + if (typeof result === 'string') { + resultStr = result; + } else if (result && typeof result === 'object') { + const resultObj = result as { + content?: unknown[]; + text?: string; + }; + if (Array.isArray(resultObj.content)) { + resultStr = resultObj.content + .filter( + (item): item is { type: string; text?: string } => + typeof item === 'object' && + item !== null && + 'type' in item && + item.type === 'text' + ) + .map((item) => item.text || '') + .join('\n'); + } else if (resultObj.text) { + resultStr = resultObj.text; + } else { + resultStr = JSON.stringify(result, null, 2); + } + } + + const maxChars = 400; + if (resultStr.length > maxChars) { + resultPreview = resultStr.slice(0, maxChars) + '\n...'; + } else { + resultPreview = resultStr; + } + } + } catch { + resultPreview = ''; + } + + if (event.callId) { + const toolMessageId = `tool-${event.callId}`; + // Finalize tool message - move to messages with result and display data + finalizeMessage(toolMessageId, { + toolResult: resultPreview, + toolStatus: 'finished', + isError: !event.success, + ...(toolDisplayData && { toolDisplayData }), + ...(toolContent && { toolContent }), + }); + } + + // Handle plan_review tool results - update UI state when plan is approved + if (event.toolName === 'plan_review' && event.success !== false) { + try { + const planReviewResult = event.rawResult as { + approved?: boolean; + } | null; + if (planReviewResult?.approved) { + // User approved the plan - disable plan mode + setUi((prev) => ({ + ...prev, + planModeActive: false, + planModeInitialized: false, + })); + } + } catch { + // Silently ignore parsing errors - plan mode state remains unchanged + } + } + + // Track tool result analytics + capture('dexto_tool_result', { + source: 'cli', + sessionId: event.sessionId, + toolName: event.toolName || 'unknown', + success: event.success !== false, + }); + break; + } + + case 'llm:error': { + const errorContent = buildErrorContent(event.error, '❌ Error: '); + + // Add error message to finalized + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: errorContent, + timestamp: new Date(), + }, + ]); + + // Only stop processing for non-recoverable errors (fatal) + // Tool errors are recoverable - agent continues after them + if (event.recoverable !== true) { + // Cancel any streaming message in pending + if (state.messageId) { + removeFromPending(state.messageId); + state.messageId = null; + state.content = ''; + } + + // Clear any remaining pending messages + clearPending(); + + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } + break; + } + + case 'llm:unsupported-input': { + // Show warning for unsupported features (e.g., model doesn't support tool calling) + const warningContent = '⚠️ ' + event.errors.join('\n⚠️ '); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('warning'), + role: 'system', + content: warningContent, + timestamp: new Date(), + }, + ]); + break; + } + + case 'run:complete': { + const { durationMs } = event; + // Total = lastInput + cumulativeOutput (avoids double-counting shared context) + const totalTokens = state.lastInputTokens + state.cumulativeOutputTokens; + + // Ensure any remaining pending messages are finalized + finalizeAllPending(); + + // Add run summary message at the END (not inserted in middle) + // IMPORTANT: Ink's <Static> tracks rendered items by array position, not key. + // Inserting in the middle shifts existing items, causing them to re-render. + // Always append to avoid duplicate rendering. + if (durationMs > 0 || totalTokens > 0) { + const summaryMessage = { + id: generateMessageId('summary'), + role: 'system' as const, + content: '', // Content rendered via styledType + timestamp: new Date(), + styledType: 'run-summary' as const, + styledData: { + durationMs, + totalTokens, + }, + }; + + setMessages((prev) => [...prev, summaryMessage]); + } + + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + isCompacting: false, + })); + + // Play completion sound to notify user task is done + options.soundService?.playCompleteSound(); + break; + } + + case 'message:dequeued': { + // Queued message is being processed + // NOTE: llm:thinking only fires ONCE at the start of execute(), + // NOT when each queued message starts. So we must finalize here. + + // 1. Finalize any pending from previous response + // This ensures the previous assistant response is in messages + // before we add the next user message + finalizeAllPending(); + + // 2. Add user message directly to messages (not buffer) + // The buffer approach doesn't work because llm:thinking + // doesn't fire between queued message runs + const textContent = extractTextContent(event.content); + + if (textContent || event.content.length > 0) { + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('user'), + role: 'user' as const, + content: textContent || '[attachment]', + timestamp: new Date(), + }, + ]); + } + + // Clear queue state - message was consumed + setQueuedMessages([]); + + // Set processing state for the queued message run + setUi((prev) => ({ ...prev, isProcessing: true })); + break; + } + + case 'tool:running': { + // Tool execution actually started (after approval if needed) + // Update status from 'pending' or 'pending_approval' to 'running' + const runningToolId = `tool-${event.toolCallId}`; + updatePendingStatus(runningToolId, 'running'); + break; + } + + // Note: context:compacting and context:compacted are handled in useAgentEvents.ts + // as the single source of truth for both manual /compact and auto-compaction + + case 'approval:request': { + // Handle approval requests in processStream (NOT useAgentEvents) to ensure + // proper ordering - text messages must be added BEFORE approval UI shows. + // This fixes a race condition where direct event bus subscription in + // useAgentEvents fired before the iterator processed llm:tool-call. + + // Check for auto-approval of edit/write tools FIRST + // Read from ref to get latest value (may have changed mid-stream) + const autoApproveEdits = options.autoApproveEditsRef.current; + const { eventBus } = options; + + if (autoApproveEdits && event.type === ApprovalTypeEnum.TOOL_CONFIRMATION) { + // Type is narrowed - metadata is now ToolConfirmationMetadata + const { toolName } = event.metadata; + + if (isAutoApprovableInEditMode(toolName)) { + // Auto-approve immediately - emit response and let tool:running handle status + eventBus.emit('approval:response', { + approvalId: event.approvalId, + status: ApprovalStatus.APPROVED, + sessionId: event.sessionId, + data: {}, + }); + break; + } + } + + // Manual approval needed - update tool status to 'pending_approval' + // Extract toolCallId based on approval type + const toolCallId = + event.type === ApprovalTypeEnum.TOOL_CONFIRMATION + ? event.metadata.toolCallId + : undefined; + if (toolCallId) { + updatePendingStatus(`tool-${toolCallId}`, 'pending_approval'); + } + + // Show approval UI (moved from useAgentEvents for ordering) + if ( + event.type === ApprovalTypeEnum.TOOL_CONFIRMATION || + event.type === ApprovalTypeEnum.COMMAND_CONFIRMATION || + event.type === ApprovalTypeEnum.ELICITATION || + event.type === ApprovalTypeEnum.DIRECTORY_ACCESS + ) { + const newApproval: ApprovalRequest = { + approvalId: event.approvalId, + type: event.type, + timestamp: event.timestamp, + metadata: event.metadata, + }; + + if (event.sessionId !== undefined) { + newApproval.sessionId = event.sessionId; + } + if (event.timeout !== undefined) { + newApproval.timeout = event.timeout; + } + + // Queue if there's already an approval, otherwise show immediately + setApproval((current) => { + if (current !== null) { + setApprovalQueue((queue) => [...queue, newApproval]); + return current; + } + setUi((prev) => ({ ...prev, activeOverlay: 'approval' })); + return newApproval; + }); + + // Play approval sound to notify user + options.soundService?.playApprovalSound(); + } + break; + } + + case 'approval:response': { + // Handle approval responses - dismisses auto-approved parallel tool calls + // + // When user approves a tool with "remember", other pending parallel requests + // for the same tool are auto-approved. The handler emits approval:response + // for each, and we need to dismiss them from the UI. + // + // For user-initiated approvals, completeApproval() in OverlayContainer + // already clears the state before emitting, so these are no-ops. + + const { approvalId } = event; + + // Step 1: Remove from queue if present + setApprovalQueue((queue) => queue.filter((a) => a.approvalId !== approvalId)); + + // Step 2: If this is the current approval, dismiss and show next + // We use the same pattern as completeApproval in OverlayContainer: + // setApprovalQueue as coordinator, calling setApproval inside + setApproval((currentApproval) => { + if (currentApproval?.approvalId !== approvalId) { + return currentApproval; // Not current, nothing to do + } + + // Current approval was auto-approved - show next or close + // Note: queue was already filtered in Step 1, so we read updated queue + setApprovalQueue((queue) => { + if (queue.length > 0) { + const [next, ...rest] = queue; + setApproval(next!); + setUi((prev) => ({ ...prev, activeOverlay: 'approval' })); + return rest; + } else { + setUi((prev) => ({ ...prev, activeOverlay: 'none' })); + return []; + } + }); + + return null; // Clear current while setApprovalQueue handles next + }); + + break; + } + + case 'service:event': { + // Handle service events - extensible pattern for non-core services + debug.log('SERVICE-EVENT received', { + service: event.service, + eventType: event.event, + toolCallId: event.toolCallId, + sessionId: event.sessionId, + }); + + // Handle agent-spawner progress events + if (event.service === 'agent-spawner' && event.event === 'progress') { + const { toolCallId, data } = event; + // Guard against null/non-object data payloads + if (toolCallId && data && typeof data === 'object') { + // Update the tool message with sub-agent progress + const toolMessageId = `tool-${toolCallId}`; + const progressData = data as { + task: string; + agentId: string; + toolsCalled: number; + currentTool: string; + currentArgs?: Record<string, unknown>; + tokenUsage?: { + input: number; + output: number; + total: number; + }; + }; + debug.log('SERVICE-EVENT updating progress', { + toolMessageId, + toolsCalled: progressData.toolsCalled, + currentTool: progressData.currentTool, + tokenUsage: progressData.tokenUsage, + }); + updatePending(toolMessageId, { + subAgentProgress: { + task: progressData.task, + agentId: progressData.agentId, + toolsCalled: progressData.toolsCalled, + currentTool: progressData.currentTool, + ...(progressData.currentArgs && { + currentArgs: progressData.currentArgs, + }), + ...(progressData.tokenUsage && { + tokenUsage: progressData.tokenUsage, + }), + }, + }); + } + } + + // Handle todo update events + if (event.service === 'todo' && event.event === 'updated') { + const { data, sessionId } = event; + if (data && typeof data === 'object' && sessionId) { + const todoData = data as { + todos?: Array<{ + id: string; + sessionId: string; + content: string; + activeForm: string; + status: 'pending' | 'in_progress' | 'completed'; + position: number; + createdAt: Date | string; + updatedAt: Date | string; + }>; + stats?: { created: number; updated: number; deleted: number }; + }; + if (!Array.isArray(todoData.todos)) { + debug.log('SERVICE-EVENT todo updated: invalid payload', { + sessionId, + }); + break; + } + debug.log('SERVICE-EVENT todo updated', { + sessionId, + todoCount: todoData.todos.length, + stats: todoData.stats, + }); + // Update todos state via the setter passed in options + if (options.setTodos) { + options.setTodos(todoData.todos); + } + } + } + break; + } + + // Ignore other events + default: + break; + } + } + } catch (error) { + // Handle iterator errors (e.g., aborted) + if (error instanceof Error && error.name === 'AbortError') { + // Expected when cancelled, clean up UI state + clearPending(); + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } else { + // Unexpected error, show to user + clearPending(); + + const errorContent = buildErrorContent(error, '❌ Stream error: '); + + setMessages((prev) => [ + ...prev, + { + id: generateMessageId('error'), + role: 'system', + content: errorContent, + timestamp: new Date(), + }, + ]); + setUi((prev) => ({ + ...prev, + isProcessing: false, + isCancelling: false, + isThinking: false, + })); + } + } +} diff --git a/dexto/packages/cli/src/cli/ink-cli/state/actions.ts b/dexto/packages/cli/src/cli/ink-cli/state/actions.ts new file mode 100644 index 00000000..bb85248c --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/state/actions.ts @@ -0,0 +1,184 @@ +/** + * State actions for CLI state machine + * All state mutations go through these actions + * + * Note: Message/streaming state is handled separately via useState in InkCLIRefactored + * to simplify the reducer and match WebUI's direct event handling pattern. + */ + +import type { OverlayType, McpWizardServerType, PendingImage } from './types.js'; +import type { ApprovalRequest } from '../components/ApprovalPrompt.js'; + +/** + * Input actions + */ +export type InputChangeAction = { + type: 'INPUT_CHANGE'; + value: string; +}; + +export type InputClearAction = { + type: 'INPUT_CLEAR'; +}; + +export type InputHistoryNavigateAction = { + type: 'INPUT_HISTORY_NAVIGATE'; + direction: 'up' | 'down'; +}; + +export type InputHistoryResetAction = { + type: 'INPUT_HISTORY_RESET'; +}; + +export type InputHistoryAddAction = { + type: 'INPUT_HISTORY_ADD'; + value: string; +}; + +/** + * Image attachment actions + */ +export type ImageAddAction = { + type: 'IMAGE_ADD'; + image: PendingImage; +}; + +export type ImageRemoveAction = { + type: 'IMAGE_REMOVE'; + imageId: string; +}; + +export type ImagesClearAction = { + type: 'IMAGES_CLEAR'; +}; + +export type CancelStartAction = { + type: 'CANCEL_START'; +}; + +export type ThinkingStartAction = { + type: 'THINKING_START'; +}; + +export type ThinkingEndAction = { + type: 'THINKING_END'; +}; + +/** + * UI actions + */ +export type ProcessingStartAction = { + type: 'PROCESSING_START'; +}; + +export type ProcessingEndAction = { + type: 'PROCESSING_END'; +}; + +export type ShowOverlayAction = { + type: 'SHOW_OVERLAY'; + overlay: OverlayType; +}; + +export type CloseOverlayAction = { + type: 'CLOSE_OVERLAY'; +}; + +export type SetMcpWizardServerTypeAction = { + type: 'SET_MCP_WIZARD_SERVER_TYPE'; + serverType: McpWizardServerType; +}; + +/** + * Session actions + */ +export type SessionSetAction = { + type: 'SESSION_SET'; + sessionId: string; + hasActiveSession: boolean; +}; + +export type SessionClearAction = { + type: 'SESSION_CLEAR'; +}; + +export type ModelUpdateAction = { + type: 'MODEL_UPDATE'; + modelName: string; +}; + +export type ConversationResetAction = { + type: 'CONVERSATION_RESET'; +}; + +/** + * Approval actions + */ +export type ApprovalRequestAction = { + type: 'APPROVAL_REQUEST'; + approval: ApprovalRequest; +}; + +export type ApprovalCompleteAction = { + type: 'APPROVAL_COMPLETE'; +}; + +/** + * Exit warning actions (for double Ctrl+C to exit) + */ +export type ExitWarningShowAction = { + type: 'EXIT_WARNING_SHOW'; +}; + +export type ExitWarningClearAction = { + type: 'EXIT_WARNING_CLEAR'; +}; + +/** + * Copy mode actions (for text selection in alternate buffer) + */ +export type CopyModeEnableAction = { + type: 'COPY_MODE_ENABLE'; +}; + +export type CopyModeDisableAction = { + type: 'COPY_MODE_DISABLE'; +}; + +/** + * Combined action type + */ +export type CLIAction = + // Input actions + | InputChangeAction + | InputClearAction + | InputHistoryNavigateAction + | InputHistoryResetAction + | InputHistoryAddAction + // Image actions + | ImageAddAction + | ImageRemoveAction + | ImagesClearAction + // Processing/streaming state + | CancelStartAction + | ThinkingStartAction + | ThinkingEndAction + | ProcessingStartAction + | ProcessingEndAction + // UI actions + | ShowOverlayAction + | CloseOverlayAction + | SetMcpWizardServerTypeAction + // Session actions + | SessionSetAction + | SessionClearAction + | ModelUpdateAction + | ConversationResetAction + // Approval actions + | ApprovalRequestAction + | ApprovalCompleteAction + // Exit/copy mode actions + | ExitWarningShowAction + | ExitWarningClearAction + | CopyModeEnableAction + | CopyModeDisableAction; diff --git a/dexto/packages/cli/src/cli/ink-cli/state/index.ts b/dexto/packages/cli/src/cli/ink-cli/state/index.ts new file mode 100644 index 00000000..e5cf8d37 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/state/index.ts @@ -0,0 +1,54 @@ +/** + * State management module exports + * + * Note: State is now managed via useState hooks in InkCLIRefactored. + * This module exports types and remaining actions for UI/overlay state. + */ + +// Types +export type { + StartupInfo, + Message, + StreamingMessage, + InputState, + PendingImage, + OverlayType, + McpWizardServerType, + UIState, + SessionState, + CLIState, +} from './types.js'; + +// Actions (reduced set - UI/overlay/session only, no message/streaming actions) +export type { + InputChangeAction, + InputClearAction, + InputHistoryNavigateAction, + InputHistoryResetAction, + InputHistoryAddAction, + ImageAddAction, + ImageRemoveAction, + ImagesClearAction, + CancelStartAction, + ThinkingStartAction, + ThinkingEndAction, + ProcessingStartAction, + ProcessingEndAction, + ShowOverlayAction, + CloseOverlayAction, + SetMcpWizardServerTypeAction, + SessionSetAction, + SessionClearAction, + ModelUpdateAction, + ConversationResetAction, + ApprovalRequestAction, + ApprovalCompleteAction, + ExitWarningShowAction, + ExitWarningClearAction, + CopyModeEnableAction, + CopyModeDisableAction, + CLIAction, +} from './actions.js'; + +// Initial state (for types reference only) +export { createInitialState } from './initialState.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/state/initialState.ts b/dexto/packages/cli/src/cli/ink-cli/state/initialState.ts new file mode 100644 index 00000000..02078ce1 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/state/initialState.ts @@ -0,0 +1,57 @@ +/** + * Initial state for CLI state machine + * + * Note: Messages are handled separately via useState in InkCLIRefactored + */ + +import type { CLIState } from './types.js'; + +/** + * Creates the initial CLI state + * @param initialModelName - Initial model name + */ +export function createInitialState(initialModelName: string = ''): CLIState { + return { + input: { + value: '', + history: [], + historyIndex: -1, + draftBeforeHistory: '', + images: [], + pastedBlocks: [], + pasteCounter: 0, + }, + ui: { + isProcessing: false, + isCancelling: false, + isThinking: false, + isCompacting: false, + activeOverlay: 'none', + exitWarningShown: false, + exitWarningTimestamp: null, + mcpWizardServerType: null, + copyModeEnabled: false, + pendingModelSwitch: null, + selectedMcpServer: null, + historySearch: { + isActive: false, + query: '', + matchIndex: 0, + originalInput: '', + lastMatch: '', + }, + promptAddWizard: null, + autoApproveEdits: false, + todoExpanded: true, + planModeActive: false, + planModeInitialized: false, + }, + session: { + id: null, + hasActiveSession: false, + modelName: initialModelName, + }, + approval: null, + approvalQueue: [], + }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/state/reducer.ts b/dexto/packages/cli/src/cli/ink-cli/state/reducer.ts new file mode 100644 index 00000000..83bbdfc2 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/state/reducer.ts @@ -0,0 +1,370 @@ +/** + * State reducer for CLI state machine + * Pure function that handles all state transitions + * + * Note: Message/streaming state is handled separately via useState in InkCLIRefactored + * to simplify the reducer and match WebUI's direct event handling pattern. + */ + +import type { CLIState } from './types.js'; +import type { CLIAction } from './actions.js'; + +/** + * Main CLI state reducer + * Handles all state transitions in a predictable, testable way + */ +export function cliReducer(state: CLIState, action: CLIAction): CLIState { + switch (action.type) { + // Input actions + case 'INPUT_CHANGE': + return { + ...state, + input: { + ...state.input, + value: action.value, + }, + // Clear exit warning when user starts typing + ui: state.ui.exitWarningShown + ? { + ...state.ui, + exitWarningShown: false, + exitWarningTimestamp: null, + } + : state.ui, + }; + + case 'INPUT_CLEAR': + return { + ...state, + input: { + ...state.input, + value: '', + historyIndex: -1, + }, + }; + + case 'INPUT_HISTORY_NAVIGATE': { + const { history } = state.input; + if (history.length === 0) return state; + + let newIndex = state.input.historyIndex; + if (action.direction === 'up') { + // Navigate backward through history (older items) + // From -1 (current input) -> last history item -> ... -> first history item + if (newIndex < 0) { + newIndex = history.length - 1; // Start at most recent + } else if (newIndex > 0) { + newIndex = newIndex - 1; // Go to older + } + // If at 0, stay there (oldest item) + } else { + // Navigate forward through history (newer items) + // From first history item -> ... -> last history item -> current input (-1) + if (newIndex >= 0 && newIndex < history.length - 1) { + newIndex = newIndex + 1; // Go to newer + } else if (newIndex === history.length - 1) { + // At most recent history, go back to current input + return { + ...state, + input: { + ...state.input, + value: '', + historyIndex: -1, + }, + }; + } + // If at -1, stay there (no change needed) + if (newIndex < 0) return state; + } + + const historyItem = history[newIndex]; + return { + ...state, + input: { + ...state.input, + value: historyItem || '', + historyIndex: newIndex, + }, + }; + } + + case 'INPUT_HISTORY_RESET': + return { + ...state, + input: { + ...state.input, + historyIndex: -1, + }, + }; + + case 'INPUT_HISTORY_ADD': { + // Add to history if not duplicate of last entry + const { history } = state.input; + if ( + !action.value || + (history.length > 0 && history[history.length - 1] === action.value) + ) { + return state; + } + return { + ...state, + input: { + ...state.input, + history: [...history, action.value].slice(-100), // Keep last 100 + historyIndex: -1, + }, + }; + } + + // Image actions + case 'IMAGE_ADD': + return { + ...state, + input: { + ...state.input, + images: [...state.input.images, action.image], + }, + }; + + case 'IMAGE_REMOVE': + return { + ...state, + input: { + ...state.input, + images: state.input.images.filter((img) => img.id !== action.imageId), + }, + }; + + case 'IMAGES_CLEAR': + return { + ...state, + input: { + ...state.input, + images: [], + }, + }; + + case 'CANCEL_START': + return { + ...state, + ui: { + ...state.ui, + isProcessing: false, + isCancelling: true, + }, + }; + + case 'THINKING_START': + return { + ...state, + ui: { + ...state.ui, + isThinking: true, + }, + }; + + case 'THINKING_END': + return { + ...state, + ui: { + ...state.ui, + isThinking: false, + }, + }; + + // UI actions + case 'PROCESSING_START': + return { + ...state, + ui: { + ...state.ui, + isProcessing: true, + isCancelling: false, // Clear cancellation flag for new request + activeOverlay: 'none', + exitWarningShown: false, // Clear exit warning on new submission + exitWarningTimestamp: null, + }, + }; + + case 'PROCESSING_END': + return { + ...state, + ui: { + ...state.ui, + isProcessing: false, + isCancelling: false, + isThinking: false, // Clear thinking state when processing ends + }, + }; + + case 'SHOW_OVERLAY': + return { + ...state, + ui: { + ...state.ui, + activeOverlay: action.overlay, + }, + }; + + case 'CLOSE_OVERLAY': + return { + ...state, + ui: { + ...state.ui, + activeOverlay: 'none', + mcpWizardServerType: null, // Clear wizard state when closing overlay + }, + }; + + case 'SET_MCP_WIZARD_SERVER_TYPE': + return { + ...state, + ui: { + ...state.ui, + mcpWizardServerType: action.serverType, + }, + }; + + // Session actions + case 'SESSION_SET': + return { + ...state, + session: { + ...state.session, + id: action.sessionId, + hasActiveSession: action.hasActiveSession, + }, + }; + + case 'SESSION_CLEAR': + return { + ...state, + session: { + ...state.session, + id: null, + hasActiveSession: false, + }, + approval: null, + approvalQueue: [], + ui: { + ...state.ui, + activeOverlay: 'none', + }, + }; + + case 'MODEL_UPDATE': + return { + ...state, + session: { + ...state.session, + modelName: action.modelName, + }, + }; + + case 'CONVERSATION_RESET': + return { + ...state, + approval: null, + approvalQueue: [], + ui: { + ...state.ui, + activeOverlay: 'none', + }, + }; + + // Approval actions + case 'APPROVAL_REQUEST': + // Dedupe: skip if this approval ID is already pending or queued + if (state.approval?.approvalId === action.approval.approvalId) { + return state; + } + if (state.approvalQueue.some((r) => r.approvalId === action.approval.approvalId)) { + return state; + } + // If there's already a pending approval, queue this one + if (state.approval !== null) { + return { + ...state, + approvalQueue: [...state.approvalQueue, action.approval], + }; + } + // Otherwise, show it immediately + return { + ...state, + approval: action.approval, + ui: { + ...state.ui, + activeOverlay: 'approval', + }, + }; + + case 'APPROVAL_COMPLETE': + // Check if there are queued approvals + if (state.approvalQueue.length > 0) { + // Show the next approval from the queue + const nextApproval = state.approvalQueue[0]!; + const remainingQueue = state.approvalQueue.slice(1); + return { + ...state, + approval: nextApproval, + approvalQueue: remainingQueue, + ui: { + ...state.ui, + activeOverlay: 'approval', + }, + }; + } + // No more approvals, clear everything + return { + ...state, + approval: null, + ui: { + ...state.ui, + activeOverlay: 'none', + }, + }; + + // Exit warning actions (for double Ctrl+C to exit) + case 'EXIT_WARNING_SHOW': + return { + ...state, + ui: { + ...state.ui, + exitWarningShown: true, + exitWarningTimestamp: Date.now(), + }, + }; + + case 'EXIT_WARNING_CLEAR': + return { + ...state, + ui: { + ...state.ui, + exitWarningShown: false, + exitWarningTimestamp: null, + }, + }; + + // Copy mode actions (for text selection in alternate buffer) + case 'COPY_MODE_ENABLE': + return { + ...state, + ui: { + ...state.ui, + copyModeEnabled: true, + }, + }; + + case 'COPY_MODE_DISABLE': + return { + ...state, + ui: { + ...state.ui, + copyModeEnabled: false, + }, + }; + + default: + return state; + } +} diff --git a/dexto/packages/cli/src/cli/ink-cli/state/streaming-state.ts b/dexto/packages/cli/src/cli/ink-cli/state/streaming-state.ts new file mode 100644 index 00000000..390d526a --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/state/streaming-state.ts @@ -0,0 +1,46 @@ +/** + * Streaming state manager for CLI + * + * Simple module-level state that can be accessed by both React components + * and command handlers. Uses a subscription pattern for React integration. + */ + +type StreamingListener = (enabled: boolean) => void; + +let streamingEnabled = false; +const listeners = new Set<StreamingListener>(); + +/** + * Get current streaming state + */ +export function isStreamingEnabled(): boolean { + return streamingEnabled; +} + +/** + * Set streaming state and notify listeners + */ +export function setStreamingEnabled(enabled: boolean): void { + if (streamingEnabled !== enabled) { + streamingEnabled = enabled; + listeners.forEach((listener) => listener(enabled)); + } +} + +/** + * Toggle streaming state + * @returns New streaming state + */ +export function toggleStreaming(): boolean { + setStreamingEnabled(!streamingEnabled); + return streamingEnabled; +} + +/** + * Subscribe to streaming state changes + * @returns Unsubscribe function + */ +export function subscribeToStreaming(listener: StreamingListener): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/state/types.ts b/dexto/packages/cli/src/cli/ink-cli/state/types.ts new file mode 100644 index 00000000..964b8260 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/state/types.ts @@ -0,0 +1,446 @@ +/** + * Core state types for Ink CLI + * Central type definitions for the CLI state machine + */ + +import type { ApprovalRequest } from '../components/ApprovalPrompt.js'; +import type { ToolDisplayData, ContentPart, McpConnectionStatus, McpServerType } from '@dexto/core'; + +/** + * Update information for version check + */ +export interface UpdateInfo { + current: string; + latest: string; + updateCommand: string; +} + +/** + * Startup information displayed in CLI header + */ +export interface StartupInfo { + connectedServers: { count: number; names: string[] }; + failedConnections: string[]; + toolCount: number; + logFile: string | null; + /** Update info if a newer version is available */ + updateInfo?: UpdateInfo | undefined; + /** True if installed agents differ from bundled and user should sync */ + needsAgentSync?: boolean | undefined; +} + +/** + * Tool call status for visual feedback + * - pending: Tool call received, checking if approval needed (static gray dot) + * - pending_approval: Waiting for user approval (static orange dot) + * - running: Actually executing (animated green/teal spinner) + * - finished: Completed (green dot success, red dot error) + */ +export type ToolStatus = 'pending' | 'pending_approval' | 'running' | 'finished'; + +/** + * Styled message types for rich command output + */ +export type StyledMessageType = + | 'config' + | 'stats' + | 'help' + | 'session-list' + | 'session-history' + | 'log-config' + | 'run-summary' + | 'prompts' + | 'sysprompt' + | 'shortcuts'; + +/** + * Structured data for styled messages + */ +export interface ConfigStyledData { + configFilePath: string | null; + provider: string; + model: string; + maxTokens: number | null; + temperature: number | null; + toolConfirmationMode: string; + maxSessions: string; + sessionTTL: string; + mcpServers: string[]; + promptsCount: number; + pluginsEnabled: string[]; +} + +export interface StatsStyledData { + sessions: { + total: number; + inMemory: number; + maxAllowed: number; + }; + mcp: { + connected: number; + failed: number; + toolCount: number; + }; + tokenUsage?: { + inputTokens: number; + outputTokens: number; + reasoningTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + totalTokens: number; + }; + estimatedCost?: number; +} + +export interface HelpStyledData { + commands: Array<{ + name: string; + description: string; + category: string; + }>; +} + +export interface SessionListStyledData { + sessions: Array<{ + id: string; + messageCount: number; + lastActive: string; + isCurrent: boolean; + }>; + total: number; +} + +export interface SessionHistoryStyledData { + sessionId: string; + messages: Array<{ + role: string; + content: string; + timestamp: string; + }>; + total: number; +} + +export interface LogConfigStyledData { + currentLevel: string; + logFile: string | null; + availableLevels: string[]; +} + +export interface RunSummaryStyledData { + /** Duration in milliseconds */ + durationMs: number; + /** Total tokens used (lastInput + cumulativeOutput) */ + totalTokens: number; +} + +export interface PromptsStyledData { + mcpPrompts: Array<{ + name: string; + title?: string; + description?: string; + args?: string[]; + }>; + configPrompts: Array<{ + name: string; + title?: string; + description?: string; + }>; + customPrompts: Array<{ + name: string; + title?: string; + description?: string; + }>; + total: number; +} + +export interface SysPromptStyledData { + content: string; +} + +export interface ShortcutsStyledData { + categories: Array<{ + name: string; + shortcuts: Array<{ + keys: string; + description: string; + }>; + }>; +} + +export type StyledData = + | ConfigStyledData + | StatsStyledData + | HelpStyledData + | SessionListStyledData + | SessionHistoryStyledData + | LogConfigStyledData + | RunSummaryStyledData + | PromptsStyledData + | SysPromptStyledData + | ShortcutsStyledData; + +/** + * Sub-agent progress data for spawn_agent tool calls + */ +export interface SubAgentProgress { + /** Short task description */ + task: string; + /** Agent ID (e.g., 'explore-agent') */ + agentId: string; + /** Number of tools called by the sub-agent */ + toolsCalled: number; + /** Current tool being executed */ + currentTool: string; + /** Current tool arguments (optional) */ + currentArgs?: Record<string, unknown> | undefined; + /** Cumulative token usage from the sub-agent (updated on each llm:response) */ + tokenUsage?: { + input: number; + output: number; + total: number; + }; +} + +/** + * Todo status for workflow tracking + */ +export type TodoStatus = 'pending' | 'in_progress' | 'completed'; + +/** + * Todo item for workflow tracking + */ +export interface TodoItem { + id: string; + sessionId: string; + content: string; + activeForm: string; + status: TodoStatus; + position: number; + createdAt: Date | string; + updatedAt: Date | string; +} + +/** + * Message in the chat interface + * + * TODO: Consolidate with InternalMessage from @dexto/core. Currently we have two + * message types: InternalMessage (core, ContentPart[] content) and Message (CLI, + * string content + UI fields). Consider extending InternalMessage or extracting + * shared role type to reduce duplication and type confusion. + */ +export interface Message { + id: string; + role: 'user' | 'assistant' | 'system' | 'tool'; + content: string; + timestamp: Date; + isStreaming?: boolean; + toolResult?: string; // Tool result preview (first 4-5 lines) + toolStatus?: ToolStatus; // Status for tool messages (running/finished) + isError?: boolean; // True if tool execution failed + styledType?: StyledMessageType; // Type of styled rendering (if any) + styledData?: StyledData; // Structured data for styled rendering + /** True for split messages that continue a previous message (no role indicator) */ + isContinuation?: boolean; + /** True for messages that are queued while agent is processing */ + isQueued?: boolean; + /** Queue position (1-indexed) for queued messages */ + queuePosition?: number; + /** Structured display data for tool-specific rendering (diffs, shell output, etc.) */ + toolDisplayData?: ToolDisplayData; + /** Content parts for tool result rendering */ + toolContent?: ContentPart[]; + /** Sub-agent progress data (for spawn_agent tool calls) */ + subAgentProgress?: SubAgentProgress; +} + +/** + * Streaming message state + */ +export interface StreamingMessage { + id: string; + content: string; +} + +/** + * Pending image attachment + */ +export interface PendingImage { + /** Unique ID for tracking/removal */ + id: string; + /** Base64-encoded image data */ + data: string; + /** MIME type of the image */ + mimeType: string; + /** Placeholder text shown in input (e.g., "[Image 1]") */ + placeholder: string; +} + +/** + * Pasted content block (for collapsible paste feature) + */ +export interface PastedBlock { + /** Unique ID for tracking */ + id: string; + /** Sequential number for display (Paste 1, Paste 2, etc.) */ + number: number; + /** The full original pasted text */ + fullText: string; + /** Line count for display */ + lineCount: number; + /** Whether this block is currently collapsed */ + isCollapsed: boolean; + /** The placeholder text when collapsed (e.g., "[Paste 1: ~32 lines]") */ + placeholder: string; +} + +/** + * Input state management + */ +export interface InputState { + value: string; + history: string[]; + historyIndex: number; + draftBeforeHistory: string; + /** Pending images to be sent with the next message */ + images: PendingImage[]; + /** Pasted content blocks (collapsed/expandable) */ + pastedBlocks: PastedBlock[]; + /** Counter for generating sequential paste numbers */ + pasteCounter: number; +} + +/** + * Available overlay types + */ +export type OverlayType = + | 'none' + | 'slash-autocomplete' + | 'resource-autocomplete' + | 'model-selector' + | 'custom-model-wizard' + | 'session-selector' + | 'mcp-server-list' + | 'mcp-server-actions' + | 'mcp-add-choice' + | 'mcp-add-selector' + | 'mcp-custom-type-selector' + | 'mcp-custom-wizard' + | 'log-level-selector' + | 'stream-selector' + | 'session-subcommand-selector' + | 'api-key-input' + | 'search' + | 'approval' + | 'tool-browser' + | 'prompt-list' + | 'prompt-add-choice' + | 'prompt-add-wizard' + | 'prompt-delete-selector' + | 'session-rename' + | 'context-stats' + | 'export-wizard' + | 'plugin-manager' + | 'plugin-list' + | 'plugin-actions' + | 'marketplace-browser' + | 'marketplace-add'; + +/** + * MCP server type for custom wizard (null = not yet selected) + */ +export type McpWizardServerType = McpServerType | null; + +/** + * MCP server info for actions screen + */ +export interface SelectedMcpServer { + name: string; + enabled: boolean; + status: McpConnectionStatus; + type: McpServerType; +} + +/** + * Pending model switch info (when waiting for API key input) + */ +export interface PendingModelSwitch { + provider: string; + model: string; + displayName?: string; +} + +/** + * Prompt add wizard state + */ +export type PromptAddScope = 'agent' | 'shared'; + +export interface PromptAddWizardState { + scope: PromptAddScope; + step: 'name' | 'title' | 'description' | 'content'; + name: string; + title: string; + description: string; + content: string; +} + +/** + * History search state (Ctrl+R reverse search) + */ +export interface HistorySearchState { + isActive: boolean; + query: string; + matchIndex: number; // Index into filtered matches (0 = most recent match) + originalInput: string; // Cached input to restore on Escape + lastMatch: string; // Last valid match (preserved when no results) +} + +/** + * UI state management + */ +export interface UIState { + isProcessing: boolean; + isCancelling: boolean; // True when cancellation is in progress + isThinking: boolean; // True when LLM is thinking (before streaming starts) + isCompacting: boolean; // True when context is being compacted + activeOverlay: OverlayType; + exitWarningShown: boolean; // True when first Ctrl+C was pressed (pending second to exit) + exitWarningTimestamp: number | null; // Timestamp of first Ctrl+C for timeout + mcpWizardServerType: McpWizardServerType; // Server type for MCP custom wizard + copyModeEnabled: boolean; // True when copy mode is active (mouse events disabled for text selection) + pendingModelSwitch: PendingModelSwitch | null; // Pending model switch waiting for API key + selectedMcpServer: SelectedMcpServer | null; // Selected server for MCP actions screen + historySearch: HistorySearchState; // Ctrl+R reverse history search + promptAddWizard: PromptAddWizardState | null; // Prompt add wizard state + autoApproveEdits: boolean; // True when edit mode is on (auto-approve edit_file/write_file) + todoExpanded: boolean; // True when todo list is expanded (shows all tasks), false when collapsed (shows current task only) + // Plan mode state (Shift+Tab toggle) + planModeActive: boolean; // True when plan mode indicator is shown + planModeInitialized: boolean; // True after first message sent in plan mode (prevents re-injection) +} + +/** + * Session state management + */ +export interface SessionState { + id: string | null; + hasActiveSession: boolean; + modelName: string; // Current model name +} + +/** + * Root CLI state (UI state only - messages handled separately via useState) + */ +export interface CLIState { + // Input state + input: InputState; + + // UI state + ui: UIState; + + // Session state + session: SessionState; + + // Approval state + approval: ApprovalRequest | null; + approvalQueue: ApprovalRequest[]; // Queue for pending approvals +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/bracketedPaste.ts b/dexto/packages/cli/src/cli/ink-cli/utils/bracketedPaste.ts new file mode 100644 index 00000000..e40b8620 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/bracketedPaste.ts @@ -0,0 +1,30 @@ +/** + * Bracketed Paste Mode utilities + * + * Bracketed paste mode tells the terminal to wrap pasted text with escape sequences: + * - Start: \x1b[200~ + * - End: \x1b[201~ + * + * This allows the application to distinguish between typed and pasted text, + * which is essential for handling multi-line pastes correctly (e.g., not + * treating newlines in pasted text as submit commands). + */ + +const ENABLE_BRACKETED_PASTE = '\x1b[?2004h'; +const DISABLE_BRACKETED_PASTE = '\x1b[?2004l'; + +/** + * Enable bracketed paste mode + * Call this when starting the CLI to ensure paste detection works + */ +export function enableBracketedPaste(): void { + process.stdout.write(ENABLE_BRACKETED_PASTE); +} + +/** + * Disable bracketed paste mode + * Call this when exiting the CLI to restore normal terminal behavior + */ +export function disableBracketedPaste(): void { + process.stdout.write(DISABLE_BRACKETED_PASTE); +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/clipboardUtils.ts b/dexto/packages/cli/src/cli/ink-cli/utils/clipboardUtils.ts new file mode 100644 index 00000000..95c9d763 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/clipboardUtils.ts @@ -0,0 +1,397 @@ +/** + * Clipboard utilities for reading images from system clipboard + * + * Supports macOS, Windows/WSL, and Linux (Wayland + X11). + * + */ + +import { spawn } from 'node:child_process'; +import { platform, release } from 'node:os'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import * as os from 'node:os'; + +export interface ClipboardImageContent { + /** Base64-encoded image data */ + data: string; + /** MIME type of the image */ + mimeType: string; +} + +/** + * Execute a command and return stdout as a buffer + * @param command - The command to execute + * @param args - Command arguments + * @param timeoutMs - Timeout in milliseconds (default: 10000) + */ +async function execCommand( + command: string, + args: string[], + timeoutMs: number = 10000 +): Promise<{ stdout: Buffer; stderr: string; exitCode: number }> { + return new Promise((resolve) => { + let timedOut = false; + const proc = spawn(command, args); + const stdoutChunks: Buffer[] = []; + let stderr = ''; + + const timer = setTimeout(() => { + timedOut = true; + proc.kill(); + resolve({ + stdout: Buffer.concat(stdoutChunks), + stderr: stderr + '\nCommand timed out', + exitCode: 1, + }); + }, timeoutMs); + + proc.stdout.on('data', (chunk: Buffer) => { + stdoutChunks.push(chunk); + }); + + proc.stderr.on('data', (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + proc.on('close', (exitCode) => { + if (timedOut) return; + clearTimeout(timer); + resolve({ + stdout: Buffer.concat(stdoutChunks), + stderr, + exitCode: exitCode ?? 1, + }); + }); + + proc.on('error', () => { + if (timedOut) return; + clearTimeout(timer); + resolve({ + stdout: Buffer.concat(stdoutChunks), + stderr, + exitCode: 1, + }); + }); + }); +} + +/** + * Execute an osascript command and return stdout as string + */ +async function execOsascript(script: string): Promise<{ stdout: string; success: boolean }> { + const result = await execCommand('osascript', ['-e', script]); + return { + stdout: result.stdout.toString().trim(), + success: result.exitCode === 0, + }; +} + +/** + * Read image from clipboard on macOS + * Uses osascript to save clipboard image to temp file, reads it, then cleans up + */ +async function readClipboardImageMacOS(): Promise<ClipboardImageContent | undefined> { + const tmpFile = path.join(os.tmpdir(), `dexto-clipboard-${Date.now()}-${process.pid}.png`); + + try { + // AppleScript to save clipboard image as PNG to temp file + const script = ` + set imageData to the clipboard as "PNGf" + set fileRef to open for access POSIX file "${tmpFile}" with write permission + set eof fileRef to 0 + write imageData to fileRef + close access fileRef + `; + + const result = await execOsascript(script); + if (!result.success) { + return undefined; + } + + // Read the temp file + const buffer = await fs.readFile(tmpFile); + if (buffer.length === 0) { + return undefined; + } + + return { + data: buffer.toString('base64'), + mimeType: 'image/png', + }; + } catch { + return undefined; + } finally { + // Clean up temp file + try { + await fs.unlink(tmpFile); + } catch { + // Ignore cleanup errors + } + } +} + +/** + * Read image from clipboard on Windows (including WSL) + * Uses PowerShell to get clipboard image and convert to base64 + */ +async function readClipboardImageWindows(): Promise<ClipboardImageContent | undefined> { + try { + // PowerShell script to get clipboard image as base64 PNG + const script = ` + Add-Type -AssemblyName System.Windows.Forms + $img = [System.Windows.Forms.Clipboard]::GetImage() + if ($img) { + $ms = New-Object System.IO.MemoryStream + $img.Save($ms, [System.Drawing.Imaging.ImageFormat]::Png) + [System.Convert]::ToBase64String($ms.ToArray()) + } + `.trim(); + + // Use powershell.exe for both native Windows and WSL (works on both) + const powershellCmd = 'powershell.exe'; + const result = await execCommand(powershellCmd, ['-command', script]); + + const base64 = result.stdout.toString().trim(); + if (!base64 || result.exitCode !== 0) { + return undefined; + } + + // Validate it's actually base64 data + const buffer = Buffer.from(base64, 'base64'); + if (buffer.length === 0) { + return undefined; + } + + return { + data: base64, + mimeType: 'image/png', + }; + } catch { + return undefined; + } +} + +/** + * Check if current environment is WSL (Windows Subsystem for Linux) + * Checks for both 'wsl' (WSL2) and 'microsoft' (WSL1) in kernel release + */ +function isWSL(): boolean { + if (platform() !== 'linux') return false; + const rel = release().toLowerCase(); + return rel.includes('wsl') || rel.includes('microsoft'); +} + +/** + * Read image from clipboard on Linux + * Tries Wayland (wl-paste) first, then X11 (xclip) + */ +async function readClipboardImageLinux(): Promise<ClipboardImageContent | undefined> { + // Try Wayland first (wl-paste) + try { + const result = await execCommand('wl-paste', ['-t', 'image/png']); + if (result.exitCode === 0 && result.stdout.length > 0) { + return { + data: result.stdout.toString('base64'), + mimeType: 'image/png', + }; + } + } catch { + // wl-paste not available or failed, try xclip + } + + // Try X11 (xclip) + try { + const result = await execCommand('xclip', [ + '-selection', + 'clipboard', + '-t', + 'image/png', + '-o', + ]); + if (result.exitCode === 0 && result.stdout.length > 0) { + return { + data: result.stdout.toString('base64'), + mimeType: 'image/png', + }; + } + } catch { + // xclip not available or failed + } + + return undefined; +} + +/** + * Read image from system clipboard + * + * @returns ClipboardImageContent if clipboard contains an image, undefined otherwise + * + * @example + * ```typescript + * const image = await readClipboardImage(); + * if (image) { + * console.log(`Got ${image.mimeType} image, ${image.data.length} bytes base64`); + * } + * ``` + */ +export async function readClipboardImage(): Promise<ClipboardImageContent | undefined> { + const os = platform(); + + if (os === 'darwin') { + return readClipboardImageMacOS(); + } + + if (os === 'win32' || isWSL()) { + return readClipboardImageWindows(); + } + + if (os === 'linux') { + return readClipboardImageLinux(); + } + + return undefined; +} + +/** + * Write text to system clipboard + * + * @param text - The text to copy to clipboard + * @returns true if successful, false otherwise + * + * @example + * ```typescript + * const success = await writeToClipboard('Hello, World!'); + * if (success) { + * console.log('Copied to clipboard'); + * } + * ``` + */ +export async function writeToClipboard(text: string): Promise<boolean> { + const os = platform(); + + try { + if (os === 'darwin') { + // macOS: use pbcopy + const proc = spawn('pbcopy'); + proc.stdin.write(text); + proc.stdin.end(); + return new Promise((resolve) => { + proc.on('close', (code) => resolve(code === 0)); + proc.on('error', () => resolve(false)); + }); + } + + if (os === 'win32' || isWSL()) { + // Windows/WSL: use clip.exe (simpler than PowerShell) + const proc = spawn('clip.exe'); + proc.stdin.write(text); + proc.stdin.end(); + return new Promise((resolve) => { + proc.on('close', (code) => resolve(code === 0)); + proc.on('error', () => resolve(false)); + }); + } + + if (os === 'linux') { + // Try Wayland first (wl-copy), fall back to X11 (xclip) + const tryClipboardTool = (cmd: string, args: string[] = []): Promise<boolean> => { + return new Promise((resolve) => { + const proc = spawn(cmd, args); + let errorOccurred = false; + + proc.on('error', () => { + errorOccurred = true; + resolve(false); + }); + + proc.stdin.write(text); + proc.stdin.end(); + + proc.on('close', (code) => { + if (!errorOccurred) { + resolve(code === 0); + } + }); + }); + }; + + // Try wl-copy first, then xclip + const wlResult = await tryClipboardTool('wl-copy'); + if (wlResult) return true; + + return tryClipboardTool('xclip', ['-selection', 'clipboard']); + } + + return false; + } catch { + return false; + } +} + +/** + * Check if clipboard contains an image (without reading it) + * This is a lighter-weight check that doesn't require reading the full image data. + * + * @returns true if clipboard likely contains an image + */ +export async function clipboardHasImage(): Promise<boolean> { + const os = platform(); + + if (os === 'darwin') { + try { + const result = await execOsascript('clipboard info'); + // Check for image types in clipboard info + const imageRegex = + /«class PNGf»|TIFF picture|JPEG picture|GIF picture|«class JPEG»|«class TIFF»/; + return imageRegex.test(result.stdout); + } catch { + return false; + } + } + + if (os === 'win32' || isWSL()) { + try { + // Quick check if clipboard has an image + const script = ` + Add-Type -AssemblyName System.Windows.Forms + [System.Windows.Forms.Clipboard]::ContainsImage() + `.trim(); + + const powershellCmd = 'powershell.exe'; + const result = await execCommand(powershellCmd, ['-command', script]); + return result.stdout.toString().trim().toLowerCase() === 'true'; + } catch { + return false; + } + } + + if (os === 'linux') { + // Try Wayland first + try { + const result = await execCommand('wl-paste', ['--list-types']); + if (result.exitCode === 0 && result.stdout.toString().includes('image/')) { + return true; + } + } catch { + // Try X11 + } + + // Try X11 + try { + const result = await execCommand('xclip', [ + '-selection', + 'clipboard', + '-t', + 'TARGETS', + '-o', + ]); + if (result.exitCode === 0 && result.stdout.toString().includes('image/')) { + return true; + } + } catch { + // xclip not available + } + } + + return false; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/commandOverlays.ts b/dexto/packages/cli/src/cli/ink-cli/utils/commandOverlays.ts new file mode 100644 index 00000000..a7025ba3 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/commandOverlays.ts @@ -0,0 +1,91 @@ +/** + * Command Overlay Registry + * + * Central source of truth for command-to-overlay mappings. + * When adding a new interactive command, add it here - no other files need changes. + */ + +import type { OverlayType } from '../state/types.js'; + +/** + * Commands that ALWAYS show an overlay when invoked. + * These commands have no handler logic - the overlay IS the functionality. + */ +const ALWAYS_OVERLAY: Record<string, OverlayType> = { + search: 'search', + find: 'search', // alias + model: 'model-selector', + resume: 'session-selector', + switch: 'session-selector', + stream: 'stream-selector', + tools: 'tool-browser', + mcp: 'mcp-server-list', + rename: 'session-rename', + context: 'context-stats', + ctx: 'context-stats', // alias + tokens: 'context-stats', // alias + export: 'export-wizard', + plugin: 'plugin-manager', +}; + +/** + * Commands that show an overlay ONLY when invoked without arguments. + * With arguments, they execute their handler instead. + */ +const NO_ARGS_OVERLAY: Record<string, OverlayType> = { + session: 'session-subcommand-selector', + log: 'log-level-selector', + prompts: 'prompt-list', +}; + +/** + * Get the overlay for a command submission (with parsed args). + * Used by InputContainer.handleSubmit + * + * @param command - The base command name (e.g., 'mcp', 'search') + * @param args - Arguments passed to the command + * @returns Overlay type to show, or null to execute command handler + */ +export function getCommandOverlay(command: string, args: string[]): OverlayType | null { + // Commands that always show overlay + const alwaysOverlay = ALWAYS_OVERLAY[command]; + if (alwaysOverlay) return alwaysOverlay; + + // Commands that show overlay only when no args + if (args.length === 0) { + const noArgsOverlay = NO_ARGS_OVERLAY[command]; + if (noArgsOverlay) return noArgsOverlay; + } + + return null; +} + +/** + * Get the overlay for a command selected from autocomplete. + * Used by OverlayContainer.handleSystemCommandSelect + * + * When selecting from autocomplete, there are never args - + * we just need to know if this command has an overlay. + * + * @param command - The command name selected from autocomplete + * @returns Overlay type to show, or null to execute command + */ +export function getCommandOverlayForSelect(command: string): OverlayType | null { + // Check "always overlay" commands first + const alwaysOverlay = ALWAYS_OVERLAY[command]; + if (alwaysOverlay) return alwaysOverlay; + + // Check "no args overlay" commands (selecting = no args) + const noArgsOverlay = NO_ARGS_OVERLAY[command]; + if (noArgsOverlay) return noArgsOverlay; + + return null; +} + +/** + * Check if a command has any overlay behavior. + * Useful for determining if a command should be handled specially. + */ +export function isInteractiveCommand(command: string): boolean { + return command in ALWAYS_OVERLAY || command in NO_ARGS_OVERLAY; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/debugLog.ts b/dexto/packages/cli/src/cli/ink-cli/utils/debugLog.ts new file mode 100644 index 00000000..4ad7c499 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/debugLog.ts @@ -0,0 +1,81 @@ +/** + * Debug Logging Utility + * + * Cross-platform debug logging that writes to temp directory. + * Controlled via environment variables for easy enable/disable. + * + * Usage: + * import { createDebugLogger } from './debugLog.js'; + * const debug = createDebugLogger('stream'); + * debug.log('EVENT', { foo: 'bar' }); + * + * Enable via environment: + * DEXTO_DEBUG_STREAM=true dexto + * DEXTO_DEBUG_ALL=true dexto (enables all debug loggers) + */ + +import { appendFileSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +export interface DebugLogger { + /** Log a message with optional data */ + log: (msg: string, data?: Record<string, unknown>) => void; + /** Check if debug logging is enabled */ + isEnabled: () => boolean; + /** Get the log file path */ + getLogPath: () => string; + /** Clear the log file and write a header */ + reset: (header?: string) => void; +} + +/** + * Creates a debug logger for a specific component. + * + * @param name Component name (used in env var and filename) + * @returns Debug logger instance + * + * @example + * const debug = createDebugLogger('stream'); + * // Enabled via DEXTO_DEBUG_STREAM=true or DEXTO_DEBUG_ALL=true + * // Writes to: {tmpdir}/dexto-debug-stream.log + */ +export function createDebugLogger(name: string): DebugLogger { + const envVar = `DEXTO_DEBUG_${name.toUpperCase()}`; + const logPath = join(tmpdir(), `dexto-debug-${name.toLowerCase()}.log`); + + const isEnabled = (): boolean => { + return process.env[envVar] === 'true' || process.env.DEXTO_DEBUG_ALL === 'true'; + }; + + const log = (msg: string, data?: Record<string, unknown>): void => { + if (!isEnabled()) return; + + try { + const timestamp = new Date().toISOString().split('T')[1]; + const dataStr = data ? ` ${JSON.stringify(data)}` : ''; + const line = `[${timestamp}] ${msg}${dataStr}\n`; + appendFileSync(logPath, line); + } catch { + // Silently ignore serialization and write errors in debug logging + } + }; + + const reset = (header?: string): void => { + if (!isEnabled()) return; + + const defaultHeader = `=== DEXTO DEBUG [${name.toUpperCase()}] ${new Date().toISOString()} ===`; + try { + writeFileSync(logPath, `${header ?? defaultHeader}\n`); + } catch { + // Silently ignore write errors in debug logging + } + }; + + return { + log, + isEnabled, + getLogPath: () => logPath, + reset, + }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/idGenerator.ts b/dexto/packages/cli/src/cli/ink-cli/utils/idGenerator.ts new file mode 100644 index 00000000..3e3cd508 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/idGenerator.ts @@ -0,0 +1,14 @@ +/** + * Utility for generating unique IDs + */ + +import { randomUUID } from 'crypto'; + +/** + * Generate a unique message ID with a type prefix + * @param type - The message type (user, system, error, tool, assistant, command) + * @returns A unique ID string + */ +export function generateMessageId(type: string): string { + return `${type}-${randomUUID()}`; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/index.ts b/dexto/packages/cli/src/cli/ink-cli/utils/index.ts new file mode 100644 index 00000000..8488cee4 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/index.ts @@ -0,0 +1,43 @@ +/** + * Utils module exports + */ + +// Input parsing +export { + type AutocompleteType, + detectAutocompleteType, + extractSlashQuery, + extractResourceQuery, + findActiveAtIndex, +} from './inputParsing.js'; + +// Command overlays (central registry) +export { + getCommandOverlay, + getCommandOverlayForSelect, + isInteractiveCommand, +} from './commandOverlays.js'; + +// Message formatting +export { + createUserMessage, + createSystemMessage, + createErrorMessage, + createToolMessage, + createStreamingMessage, + convertHistoryToUIMessages, + getStartupInfo, +} from './messageFormatting.js'; + +// ID generation +export { generateMessageId } from './idGenerator.js'; + +// Sound notifications +export { + playNotificationSound, + SoundNotificationService, + initializeSoundService, + getSoundService, + type SoundType, + type SoundConfig, +} from './soundNotification.js'; diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/input.ts b/dexto/packages/cli/src/cli/ink-cli/utils/input.ts new file mode 100644 index 00000000..d0fb0154 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/input.ts @@ -0,0 +1,58 @@ +/** + * Input Utilities + * Constants and helpers for parsing terminal input sequences. + */ + +export const ESC = '\u001B'; +export const SGR_EVENT_PREFIX = `${ESC}[<`; +export const X11_EVENT_PREFIX = `${ESC}[M`; + +// Focus tracking sequences +export const FOCUS_IN = `${ESC}[I`; +export const FOCUS_OUT = `${ESC}[O`; + +// Mouse event regex patterns +// Using ESC constant to avoid control-character lint warnings +const ESC_CHAR = String.fromCharCode(0x1b); +export const SGR_MOUSE_REGEX = new RegExp(`^${ESC_CHAR}\\[<(\\d+);(\\d+);(\\d+)([mM])`); +export const X11_MOUSE_REGEX = new RegExp(`^${ESC_CHAR}\\[M([\\s\\S]{3})`); + +/** + * Check if buffer could be a SGR mouse sequence (or prefix thereof) + */ +export function couldBeSGRMouseSequence(buffer: string): boolean { + if (buffer.length === 0) return true; + // Check if buffer is a prefix of a mouse sequence starter + if (SGR_EVENT_PREFIX.startsWith(buffer)) return true; + // Check if buffer is a mouse sequence prefix + if (buffer.startsWith(SGR_EVENT_PREFIX)) return true; + return false; +} + +/** + * Check if buffer could be any mouse sequence (or prefix thereof) + */ +export function couldBeMouseSequence(buffer: string): boolean { + if (buffer.length === 0) return true; + + // Check SGR prefix + if (SGR_EVENT_PREFIX.startsWith(buffer) || buffer.startsWith(SGR_EVENT_PREFIX)) return true; + // Check X11 prefix + if (X11_EVENT_PREFIX.startsWith(buffer) || buffer.startsWith(X11_EVENT_PREFIX)) return true; + + return false; +} + +/** + * Get the length of a complete mouse sequence at the start of buffer. + * Returns 0 if no complete sequence found. + */ +export function getMouseSequenceLength(buffer: string): number { + const sgrMatch = buffer.match(SGR_MOUSE_REGEX); + if (sgrMatch) return sgrMatch[0].length; + + const x11Match = buffer.match(X11_MOUSE_REGEX); + if (x11Match) return x11Match[0].length; + + return 0; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/inputParsing.ts b/dexto/packages/cli/src/cli/ink-cli/utils/inputParsing.ts new file mode 100644 index 00000000..61f06078 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/inputParsing.ts @@ -0,0 +1,80 @@ +/** + * Input parsing utilities + * Helpers for detecting autocomplete triggers and parsing input + */ + +import { parseInput } from '../../commands/interactive-commands/command-parser.js'; + +/** + * Autocomplete type + */ +export type AutocompleteType = 'none' | 'slash' | 'resource'; + +/** + * Detects what type of autocomplete should be shown based on input + */ +export function detectAutocompleteType(input: string): AutocompleteType { + if (input.startsWith('/')) { + // Only show slash autocomplete if user hasn't started typing arguments + // Once there's a space after the command, hide autocomplete + const afterSlash = input.slice(1).trim(); + if (afterSlash.includes(' ')) { + return 'none'; // User is typing arguments, hide autocomplete + } + return 'slash'; + } + + // Check for @ mention (at start or after space) + const atIndex = findActiveAtIndex(input, input.length); + if (atIndex >= 0) { + return 'resource'; + } + + return 'none'; +} + +/** + * Extracts the query string for slash command autocomplete + */ +export function extractSlashQuery(input: string): string { + if (!input.startsWith('/')) return ''; + return input.slice(1).trim(); +} + +/** + * Extracts the query string for resource autocomplete + */ +export function extractResourceQuery(input: string): string { + const atIndex = findActiveAtIndex(input, input.length); + if (atIndex < 0) return ''; + return input.slice(atIndex + 1).trim(); +} + +/** + * Finds the active @ mention position (at start or after space) + * Returns -1 if no valid @ found + */ +export function findActiveAtIndex(value: string, caret: number): number { + // Walk backwards from caret to find an '@' + for (let i = caret - 1; i >= 0; i--) { + const ch = value[i]; + if (ch === '@') { + // Check if @ is at start or preceded by whitespace + if (i === 0) { + return i; // @ at start is valid + } + const prev = value[i - 1]; + if (prev && /\s/.test(prev)) { + return i; // @ after whitespace is valid + } + return -1; // @ in middle of text (like email) - ignore + } + if (ch && /\s/.test(ch)) break; // stop at whitespace + } + return -1; +} + +/** + * Re-export parseInput for convenience + */ +export { parseInput }; diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/messageFormatting.ts b/dexto/packages/cli/src/cli/ink-cli/utils/messageFormatting.ts new file mode 100644 index 00000000..3cdad498 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/messageFormatting.ts @@ -0,0 +1,750 @@ +/** + * Message formatting utilities + * Helpers for creating and formatting messages + */ + +import path from 'path'; +import os from 'os'; +import type { DextoAgent, InternalMessage, ContentPart, ToolCall } from '@dexto/core'; +import { isTextPart, isAssistantMessage, isToolMessage } from '@dexto/core'; +import type { Message } from '../state/types.js'; +import { generateMessageId } from './idGenerator.js'; + +/** + * Regex to detect skill invocation messages. + * Matches: <skill-invocation>...skill: "config:skill-name"...</skill-invocation> + * Works for both fork and inline skills. + */ +const SKILL_INVOCATION_REGEX = + /<skill-invocation>[\s\S]*?skill:\s*"(?:config:)?([^"]+)"[\s\S]*?<\/skill-invocation>/; + +/** + * Formats a skill invocation message for clean display. + * Converts verbose <skill-invocation> blocks to clean /skill-name format. + * Works for both fork skills (just the tag) and inline skills (tag + content). + * + * @param content - The message content to check and format + * @returns Formatted content if it's a skill invocation, original content otherwise + */ +export function formatSkillInvocationMessage(content: string): string { + const match = content.match(SKILL_INVOCATION_REGEX); + if (match) { + const skillName = match[1]; + // Extract task context if present + const contextMatch = content.match(/Task context:\s*(.+?)(?:\n|$)/); + if (contextMatch) { + return `/${skillName} ${contextMatch[1]}`; + } + return `/${skillName}`; + } + return content; +} + +/** + * Checks if a message content is a skill invocation. + */ +export function isSkillInvocationMessage(content: string): boolean { + return SKILL_INVOCATION_REGEX.test(content); +} + +/** + * Convert absolute path to display-friendly relative path. + * Strategy: + * 1. If path is under cwd → relative from cwd (e.g., "src/file.ts") + * 2. If path is under home → use tilde (e.g., "~/Projects/file.ts") + * 3. Otherwise → return absolute path + */ +export function makeRelativePath(absolutePath: string, cwd: string = process.cwd()): string { + // Normalize paths for comparison + const normalizedPath = path.normalize(absolutePath); + const normalizedCwd = path.normalize(cwd); + const homeDir = os.homedir(); + + // If under cwd, return relative path + if (normalizedPath.startsWith(normalizedCwd + path.sep) || normalizedPath === normalizedCwd) { + const relative = path.relative(normalizedCwd, normalizedPath); + return relative || '.'; + } + + // If under home directory, use tilde + if (normalizedPath.startsWith(homeDir + path.sep) || normalizedPath === homeDir) { + return '~' + normalizedPath.slice(homeDir.length); + } + + // Return absolute path as-is + return absolutePath; +} + +/** + * Format a path for display: relative + center-truncate if needed. + * @param absolutePath - The absolute file path + * @param maxWidth - Maximum display width (default 60) + * @param cwd - Current working directory for relative path calculation + */ +export function formatPathForDisplay( + absolutePath: string, + maxWidth: number = 60, + cwd: string = process.cwd() +): string { + // First convert to relative + const relativePath = makeRelativePath(absolutePath, cwd); + + // If fits, return as-is + if (relativePath.length <= maxWidth) { + return relativePath; + } + + // Apply center-truncation + return centerTruncatePath(relativePath, maxWidth); +} + +/** + * Center-truncate a file path to keep the filename visible. + * e.g., "/Users/karaj/Projects/very/long/path/to/file.ts" → "/Users/karaj/…/to/file.ts" + * + * Strategy: + * 1. If path fits within maxWidth, return as-is + * 2. Keep first segment (root/home) and last 2 segments (parent + filename) + * 3. Add "…" in the middle + */ +export function centerTruncatePath(filePath: string, maxWidth: number): string { + if (filePath.length <= maxWidth) { + return filePath; + } + + const sep = path.sep; + const segments = filePath.split(sep).filter(Boolean); + + if (segments.length <= 3) { + // Too few segments to center-truncate, just end-truncate + return filePath.slice(0, maxWidth - 1) + '…'; + } + + // Keep first segment and last 2 segments + const first = filePath.startsWith(sep) ? sep + segments[0] : segments[0]; + const lastTwo = segments.slice(-2).join(sep); + + const truncated = `${first}${sep}…${sep}${lastTwo}`; + + if (truncated.length <= maxWidth) { + return truncated; + } + + // Still too long - try with just the filename + const filename = segments[segments.length - 1] || ''; + const withJustFilename = `…${sep}${filename}`; + + if (withJustFilename.length <= maxWidth) { + return withJustFilename; + } + + // Filename itself is too long, end-truncate it + return filename.slice(0, maxWidth - 1) + '…'; +} + +/** + * Tool-specific display configuration. + * Controls how each tool is displayed in the UI - name, which args to show, etc. + */ +interface ToolDisplayConfig { + /** User-friendly display name */ + displayName: string; + /** Which args to display, in order */ + argsToShow: string[]; + /** Primary arg shown without key name (first in argsToShow) */ + primaryArg?: string; +} + +/** + * Per-tool display configurations. + * Each tool specifies exactly which arguments to show and how. + */ +const TOOL_CONFIGS: Record<string, ToolDisplayConfig> = { + // File tools - show file_path as primary + read_file: { displayName: 'Read', argsToShow: ['file_path'], primaryArg: 'file_path' }, + write_file: { displayName: 'Write', argsToShow: ['file_path'], primaryArg: 'file_path' }, + edit_file: { displayName: 'Update', argsToShow: ['file_path'], primaryArg: 'file_path' }, + + // Search tools - show pattern as primary, path as secondary + glob_files: { + displayName: 'Find files', + argsToShow: ['pattern', 'path'], + primaryArg: 'pattern', + }, + grep_content: { + displayName: 'Search files', + argsToShow: ['pattern', 'path'], + primaryArg: 'pattern', + }, + + // Bash - show command only, skip description + bash_exec: { displayName: 'Bash', argsToShow: ['command'], primaryArg: 'command' }, + bash_output: { + displayName: 'BashOutput', + argsToShow: ['process_id'], + primaryArg: 'process_id', + }, + kill_process: { displayName: 'Kill', argsToShow: ['process_id'], primaryArg: 'process_id' }, + + // User interaction + ask_user: { displayName: 'Ask', argsToShow: ['question'], primaryArg: 'question' }, + + // Agent spawning - handled specially in formatToolHeader for dynamic agentId + spawn_agent: { displayName: 'Agent', argsToShow: ['task'], primaryArg: 'task' }, + + // Skill invocation - handled specially in formatToolHeader to show clean skill name + invoke_skill: { displayName: 'Skill', argsToShow: ['skill'], primaryArg: 'skill' }, + + todo_write: { displayName: 'UpdateTasks', argsToShow: [] }, +}; + +/** + * Gets the display config for a tool. + * Handles internal-- prefix by stripping it before lookup. + */ +function getToolConfig(toolName: string): ToolDisplayConfig | undefined { + // Try direct lookup first + if (TOOL_CONFIGS[toolName]) { + return TOOL_CONFIGS[toolName]; + } + // Strip internal-- prefix and try again + if (toolName.startsWith('internal--')) { + const baseName = toolName.replace('internal--', ''); + return TOOL_CONFIGS[baseName]; + } + + // Strip "custom--" prefix and try again + if (toolName.startsWith('custom--')) { + const baseName = toolName.replace('custom--', ''); + return TOOL_CONFIGS[baseName]; + } + return undefined; +} + +/** + * Gets a user-friendly display name for a tool. + * Returns the friendly name if known, otherwise returns the original name + * with any "internal--" prefix stripped. + * MCP tools keep their server prefix for clarity (e.g., "mcp_server__tool"). + */ +export function getToolDisplayName(toolName: string): string { + const config = getToolConfig(toolName); + if (config) { + return config.displayName; + } + // Strip "internal--" prefix for unknown internal tools + if (toolName.startsWith('internal--')) { + return toolName.replace('internal--', ''); + } + // Strip "custom--" prefix for custom tools + if (toolName.startsWith('custom--')) { + return toolName.replace('custom--', ''); + } + // MCP tools: strip mcp-- or mcp__ prefix and server name for clean display + if (toolName.startsWith('mcp--')) { + const parts = toolName.split('--'); + if (parts.length >= 3) { + return parts.slice(2).join('--'); + } + return toolName.substring(5); + } + if (toolName.startsWith('mcp__')) { + const parts = toolName.substring(5).split('__'); + if (parts.length >= 2) { + return parts.slice(1).join('__'); + } + return toolName.substring(5); + } + return toolName; +} + +/** + * Gets the tool type badge for display. + * Returns: 'internal', MCP server name, or 'custom' + */ +export function getToolTypeBadge(toolName: string): string { + // Internal tools + if (toolName.startsWith('internal--') || toolName.startsWith('internal__')) { + return 'internal'; + } + + // MCP tools with server name + if (toolName.startsWith('mcp--')) { + const parts = toolName.split('--'); + if (parts.length >= 3 && parts[1]) { + return `MCP: ${parts[1]}`; // Format: 'MCP: github', 'MCP: postgres' + } + return 'MCP'; + } + + if (toolName.startsWith('mcp__')) { + const parts = toolName.substring(5).split('__'); + if (parts.length >= 2 && parts[0]) { + return `MCP: ${parts[0]}`; // Format: 'MCP: servername' + } + return 'MCP'; + } + + // Custom tools + if (toolName.startsWith('custom--')) { + return 'custom'; + } + + // Unknown - likely custom + return 'custom'; +} + +/** + * Result of formatting a tool header for display + */ +export interface FormattedToolHeader { + /** User-friendly display name (e.g., "Explore", "Read") */ + displayName: string; + /** Formatted arguments string (e.g., "file.ts" or "pattern, path: /src") */ + argsFormatted: string; + /** Tool type badge (e.g., "internal", "custom", "MCP: github") */ + badge: string; + /** Full formatted header string (e.g., "Explore(task) [custom]") */ + header: string; +} + +/** + * Formats a tool call header for consistent display across CLI. + * Used by both tool messages and approval prompts. + * + * Handles special cases like spawn_agent (uses agentId as display name). + * + * @param toolName - Raw tool name (may include prefixes like "custom--") + * @param args - Tool arguments object + * @returns Formatted header components and full string + */ +export function formatToolHeader( + toolName: string, + args: Record<string, unknown> = {} +): FormattedToolHeader { + let displayName = getToolDisplayName(toolName); + const argsFormatted = formatToolArgsForDisplay(toolName, args); + const badge = getToolTypeBadge(toolName); + + // Normalize tool name to handle all prefixes (internal--, custom--) + const normalizedToolName = toolName.replace(/^(?:internal--|custom--)/, ''); + + // Special handling for spawn_agent: use agentId as display name + const isSpawnAgent = normalizedToolName === 'spawn_agent'; + if (isSpawnAgent && args.agentId) { + const agentId = String(args.agentId); + const agentLabel = agentId.replace(/-agent$/, ''); + displayName = agentLabel.charAt(0).toUpperCase() + agentLabel.slice(1); + } + + // Special handling for invoke_skill: show skill as /skill-name + const isInvokeSkill = normalizedToolName === 'invoke_skill'; + if (isInvokeSkill && args.skill) { + const skillName = String(args.skill); + // Extract display name from skill identifier (e.g., "config:test-fork" -> "test-fork") + const colonIndex = skillName.indexOf(':'); + const displaySkillName = colonIndex >= 0 ? skillName.slice(colonIndex + 1) : skillName; + // Override args display to show clean slash command format + return { + displayName: 'Skill', + argsFormatted: `/${displaySkillName}`, + badge, + header: `Skill(/${displaySkillName})`, + }; + } + + // Only show badge for MCP tools (external tools worth distinguishing) + const isMcpTool = badge.startsWith('MCP'); + const badgeSuffix = isMcpTool ? ` [${badge}]` : ''; + + // Format: DisplayName(args) [badge] (badge only for MCP) + const header = argsFormatted + ? `${displayName}(${argsFormatted})${badgeSuffix}` + : `${displayName}()${badgeSuffix}`; + + return { + displayName, + argsFormatted, + badge, + header, + }; +} + +/** + * Fallback primary argument names for unknown tools. + * Used when we don't have a specific config for a tool. + */ +const FALLBACK_PRIMARY_ARGS = new Set([ + 'file_path', + 'path', + 'pattern', + 'command', + 'query', + 'question', + 'url', +]); + +/** + * Arguments that are file paths and should use relative path formatting. + * These get converted to relative paths and center-truncated if needed. + */ +const PATH_ARGS = new Set(['file_path', 'path']); + +/** + * Arguments that should never be truncated (urls, task descriptions, etc.) + * These provide important context that users need to see in full. + * Note: 'command' is handled specially - single-line commands are not truncated, + * but multi-line commands (heredocs) are truncated to first line only. + */ +const NEVER_TRUNCATE_ARGS = new Set(['url', 'task', 'pattern', 'question']); + +/** + * Formats tool arguments for display. + * Format: ToolName(primary_arg) or ToolName(primary_arg, key: value) + * + * Uses tool-specific config to determine which args to show. + * - File paths: converted to relative paths, center-truncated if needed + * - Commands/URLs: shown in full (never truncated) + * - Other args: truncated at 40 chars + */ +export function formatToolArgsForDisplay(toolName: string, args: Record<string, unknown>): string { + const entries = Object.entries(args); + if (entries.length === 0) return ''; + + const config = getToolConfig(toolName); + const parts: string[] = []; + + /** + * Format a single argument value for display + */ + const formatArgValue = (argName: string, value: unknown): string => { + const strValue = typeof value === 'string' ? value : JSON.stringify(value); + + // File paths: use relative path (no truncation) + if (PATH_ARGS.has(argName)) { + return makeRelativePath(strValue); + } + + // Commands: show single-line in full, truncate multi-line (heredocs) to first line + if (argName === 'command') { + const newlineIndex = strValue.indexOf('\n'); + if (newlineIndex === -1) { + // Single-line command: show in full (useful for complex pipes) + return strValue; + } + // Multi-line command (heredoc): show first line only + return strValue.slice(0, newlineIndex) + '...'; + } + + // URLs: never truncate + if (NEVER_TRUNCATE_ARGS.has(argName)) { + return strValue; + } + + // Other args: simple truncation + return strValue.length > 40 ? strValue.slice(0, 37) + '...' : strValue; + }; + + if (config) { + // Use tool-specific config + for (const argName of config.argsToShow) { + if (!(argName in args)) continue; + if (argName === 'description') continue; // Skip description field + if (parts.length >= 3) break; + + const formattedValue = formatArgValue(argName, args[argName]); + + if (argName === config.primaryArg) { + // Primary arg without key name + parts.unshift(formattedValue); + } else { + // Secondary args with key name + parts.push(`${argName}: ${formattedValue}`); + } + } + } else { + // Fallback for unknown tools (MCP, etc.) + for (const [key, value] of entries) { + if (key === 'description') continue; // Skip description field + if (parts.length >= 3) break; + + const formattedValue = formatArgValue(key, value); + + if (FALLBACK_PRIMARY_ARGS.has(key) || PATH_ARGS.has(key)) { + // Primary arg without key name + parts.unshift(formattedValue); + } else { + // Other args with key name + parts.push(`${key}: ${formattedValue}`); + } + } + } + + return parts.join(', '); +} + +/** + * Formats tool arguments for display (compact preview). + * @deprecated Use formatToolArgsForDisplay instead + */ +export function formatToolArgsPreview( + args: Record<string, unknown>, + maxLength: number = 60 +): string { + const entries = Object.entries(args); + if (entries.length === 0) return ''; + + // Show key parameters in a compact format + const preview = entries + .slice(0, 3) // Max 3 params + .map(([key, value]) => { + const strValue = typeof value === 'string' ? value : JSON.stringify(value); + const truncated = strValue.length > 30 ? strValue.slice(0, 27) + '...' : strValue; + return `${key}: "${truncated}"`; + }) + .join(', '); + + return preview.length > maxLength ? preview.slice(0, maxLength - 3) + '...' : preview; +} + +/** + * Creates a user message + */ +export function createUserMessage(content: string): Message { + return { + id: generateMessageId('user'), + role: 'user', + content, + timestamp: new Date(), + }; +} + +/** + * Creates a queued user message (shown when message is queued while processing) + */ +export function createQueuedUserMessage(content: string, queuePosition: number): Message { + return { + id: generateMessageId('user-queued'), + role: 'user', + content, + timestamp: new Date(), + isQueued: true, + queuePosition, + }; +} + +/** + * Creates a system message + */ +export function createSystemMessage(content: string): Message { + return { + id: generateMessageId('system'), + role: 'system', + content, + timestamp: new Date(), + }; +} + +/** + * Creates an error message + */ +export function createErrorMessage(error: Error | string): Message { + const content = error instanceof Error ? error.message : error; + return { + id: generateMessageId('error'), + role: 'system', + content: `❌ Error: ${content}`, + timestamp: new Date(), + }; +} + +/** + * Creates a tool call message + */ +export function createToolMessage(toolName: string): Message { + return { + id: generateMessageId('tool'), + role: 'tool', + content: `🔧 Calling tool: ${toolName}`, + timestamp: new Date(), + }; +} + +/** + * Formats a streaming placeholder message + */ +export function createStreamingMessage(): Message { + return { + id: generateMessageId('assistant'), + role: 'assistant', + content: '', + timestamp: new Date(), + isStreaming: true, + }; +} + +/** + * Extracts text content from message content (handles ContentPart array or null) + */ +function extractTextContent(content: ContentPart[] | null): string { + if (!content) { + return ''; + } + + return content + .filter(isTextPart) + .map((part) => part.text) + .join('\n'); +} + +/** + * Generates a preview of tool result content for display + */ +function generateToolResultPreview(content: ContentPart[]): string { + const textContent = extractTextContent(content); + if (!textContent) return ''; + + const lines = textContent.split('\n'); + const previewLines = lines.slice(0, 5); + let preview = previewLines.join('\n'); + + // Truncate if too long + if (preview.length > 400) { + preview = preview.slice(0, 397) + '...'; + } else if (lines.length > 5) { + preview += '\n...'; + } + + return preview; +} + +/** + * Converts session history messages to UI messages + */ +export function convertHistoryToUIMessages( + history: InternalMessage[], + sessionId: string +): Message[] { + const uiMessages: Message[] = []; + + // Build a map of toolCallId -> ToolCall for looking up tool call args + const toolCallMap = new Map<string, ToolCall>(); + for (const msg of history) { + if (isAssistantMessage(msg) && msg.toolCalls) { + for (const toolCall of msg.toolCalls) { + toolCallMap.set(toolCall.id, toolCall); + } + } + } + + history.forEach((msg, index) => { + const timestamp = new Date(msg.timestamp ?? Date.now() - (history.length - index) * 1000); + + // Handle tool messages specially + if (isToolMessage(msg)) { + // Look up the original tool call to get args + const toolCall = toolCallMap.get(msg.toolCallId); + + // Format tool name + const displayName = getToolDisplayName(msg.name); + + // Format args if we have them + let toolContent = displayName; + if (toolCall) { + try { + const args = JSON.parse(toolCall.function.arguments || '{}'); + const argsFormatted = formatToolArgsForDisplay(msg.name, args); + if (argsFormatted) { + toolContent = `${displayName}(${argsFormatted})`; + } + } catch { + // Ignore JSON parse errors + } + } + + // Add tool type badge (only for MCP tools) + const badge = getToolTypeBadge(msg.name); + if (badge.startsWith('MCP')) { + toolContent = `${toolContent} [${badge}]`; + } + + // Generate result preview + const resultPreview = generateToolResultPreview(msg.content); + + uiMessages.push({ + id: `session-${sessionId}-${index}`, + role: 'tool', + content: toolContent, + timestamp, + toolStatus: 'finished', + toolResult: resultPreview, + isError: msg.success === false, + // Store content parts for potential rich rendering + toolContent: msg.content, + // Restore structured display data for rich rendering (diffs, shell output, etc.) + ...(msg.displayData !== undefined && { + toolDisplayData: msg.displayData, + }), + }); + return; + } + + // Handle assistant messages - skip those with only tool calls (no text content) + if (isAssistantMessage(msg)) { + const textContent = extractTextContent(msg.content); + + // Skip if no text content (message was just tool calls) + if (!textContent) return; + + uiMessages.push({ + id: `session-${sessionId}-${index}`, + role: 'assistant', + content: textContent, + timestamp, + }); + return; + } + + // Handle other messages (user, system) + let textContent = extractTextContent(msg.content); + + // Skip empty messages + if (!textContent) return; + + // Format skill invocation messages for cleaner display + if (msg.role === 'user') { + textContent = formatSkillInvocationMessage(textContent); + } + + uiMessages.push({ + id: `session-${sessionId}-${index}`, + role: msg.role, + content: textContent, + timestamp, + }); + }); + + return uiMessages; +} + +/** + * Collects startup information for display in header + */ +export async function getStartupInfo(agent: DextoAgent) { + const connectedServers = agent.mcpManager.getClients(); + const failedConnections = agent.mcpManager.getFailedConnections(); + const tools = await agent.getAllTools(); + const toolCount = Object.keys(tools).length; + // Use agent's logger which has the correct per-agent log path from enriched config + const logFile = agent.logger.getLogFilePath(); + + return { + connectedServers: { + count: connectedServers.size, + names: Array.from(connectedServers.keys()), + }, + failedConnections: Object.keys(failedConnections), + toolCount, + logFile, + }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/mouse.ts b/dexto/packages/cli/src/cli/ink-cli/utils/mouse.ts new file mode 100644 index 00000000..8acad2a7 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/mouse.ts @@ -0,0 +1,262 @@ +/** + * Mouse Event Utilities + * Provides trackpad/mouse scroll support. + */ + +import { + ESC, + SGR_MOUSE_REGEX, + X11_MOUSE_REGEX, + SGR_EVENT_PREFIX, + X11_EVENT_PREFIX, + couldBeMouseSequence, +} from './input.js'; + +export type MouseEventName = + | 'scroll-up' + | 'scroll-down' + | 'scroll-left' + | 'scroll-right' + | 'left-press' + | 'left-release' + | 'right-press' + | 'right-release' + | 'middle-press' + | 'middle-release' + | 'move'; + +export interface MouseEvent { + name: MouseEventName; + col: number; + row: number; + shift: boolean; + meta: boolean; + ctrl: boolean; + button: 'left' | 'middle' | 'right' | 'none'; +} + +export type MouseHandler = (event: MouseEvent) => void | boolean; + +/** + * Enable mouse events in the terminal. + * Uses SGR extended mouse mode for better coordinate handling. + */ +export function enableMouseEvents(): void { + // ?1002h = button event tracking (clicks + drags + scroll wheel) + // ?1006h = SGR extended mouse mode (better coordinate handling) + process.stdout.write('\u001b[?1002h\u001b[?1006h'); +} + +/** + * Disable mouse events in the terminal. + */ +export function disableMouseEvents(): void { + process.stdout.write('\u001b[?1006l\u001b[?1002l'); +} + +/** + * Get mouse event name from button code + * + * NOTE: X11 mouse protocol limitation - all button releases report the same code (3), + * so we cannot distinguish between left-release, middle-release, and right-release. + * This function returns 'left-release' for all releases in X11 mode. + * Consumers needing accurate release identification should track press state themselves. + */ +export function getMouseEventName(buttonCode: number, isRelease: boolean): MouseEventName | null { + const isMove = (buttonCode & 32) !== 0; + + // Horizontal scroll (less common) + if (buttonCode === 66) { + return 'scroll-left'; + } else if (buttonCode === 67) { + return 'scroll-right'; + } + + // Vertical scroll + if ((buttonCode & 64) === 64) { + if ((buttonCode & 1) === 0) { + return 'scroll-up'; + } else { + return 'scroll-down'; + } + } + + if (isMove) { + return 'move'; + } + + const button = buttonCode & 3; + const type = isRelease ? 'release' : 'press'; + switch (button) { + case 0: + return `left-${type}` as MouseEventName; + case 1: + return `middle-${type}` as MouseEventName; + case 2: + return `right-${type}` as MouseEventName; + default: + return null; + } +} + +/** + * Get button type from button code + */ +function getButtonFromCode(code: number): MouseEvent['button'] { + const button = code & 3; + switch (button) { + case 0: + return 'left'; + case 1: + return 'middle'; + case 2: + return 'right'; + default: + return 'none'; + } +} + +/** + * Parse SGR format mouse event + */ +export function parseSGRMouseEvent(buffer: string): { event: MouseEvent; length: number } | null { + const match = buffer.match(SGR_MOUSE_REGEX); + + if (match) { + const buttonCode = parseInt(match[1]!, 10); + const col = parseInt(match[2]!, 10); + const row = parseInt(match[3]!, 10); + const action = match[4]; + const isRelease = action === 'm'; + + const shift = (buttonCode & 4) !== 0; + const meta = (buttonCode & 8) !== 0; + const ctrl = (buttonCode & 16) !== 0; + + const name = getMouseEventName(buttonCode, isRelease); + + if (name) { + return { + event: { + name, + ctrl, + meta, + shift, + col, + row, + button: getButtonFromCode(buttonCode), + }, + length: match[0].length, + }; + } + } + + return null; +} + +/** + * Parse X11 format mouse event + */ +export function parseX11MouseEvent(buffer: string): { event: MouseEvent; length: number } | null { + const match = buffer.match(X11_MOUSE_REGEX); + if (!match) return null; + + const b = match[1]!.charCodeAt(0) - 32; + const col = match[1]!.charCodeAt(1) - 32; + const row = match[1]!.charCodeAt(2) - 32; + + const shift = (b & 4) !== 0; + const meta = (b & 8) !== 0; + const ctrl = (b & 16) !== 0; + const isMove = (b & 32) !== 0; + const isWheel = (b & 64) !== 0; + + let name: MouseEventName | null = null; + + if (isWheel) { + const button = b & 3; + switch (button) { + case 0: + name = 'scroll-up'; + break; + case 1: + name = 'scroll-down'; + break; + } + } else if (isMove) { + name = 'move'; + } else { + const button = b & 3; + if (button === 3) { + // X11 reports 'release' (3) for all button releases + name = 'left-release'; + } else { + switch (button) { + case 0: + name = 'left-press'; + break; + case 1: + name = 'middle-press'; + break; + case 2: + name = 'right-press'; + break; + } + } + } + + if (name) { + let button = getButtonFromCode(b); + if (name === 'left-release' && button === 'none') { + button = 'left'; + } + + return { + event: { + name, + ctrl, + meta, + shift, + col, + row, + button, + }, + length: match[0].length, + }; + } + return null; +} + +/** + * Parse mouse event from buffer (tries both SGR and X11 formats) + */ +export function parseMouseEvent(buffer: string): { event: MouseEvent; length: number } | null { + return parseSGRMouseEvent(buffer) || parseX11MouseEvent(buffer); +} + +/** + * Check if buffer could be an incomplete mouse sequence + */ +export function isIncompleteMouseSequence(buffer: string): boolean { + if (!couldBeMouseSequence(buffer)) return false; + + // If it matches a complete sequence, it's not incomplete + if (parseMouseEvent(buffer)) return false; + + if (buffer.startsWith(X11_EVENT_PREFIX)) { + // X11 needs exactly 3 bytes after prefix + return buffer.length < X11_EVENT_PREFIX.length + 3; + } + + if (buffer.startsWith(SGR_EVENT_PREFIX)) { + // SGR sequences end with 'm' or 'M' + // Add a reasonable max length check to fail early on garbage + return !/[mM]/.test(buffer) && buffer.length < 50; + } + + // It's a prefix of the prefix (e.g. "ESC" or "ESC [") + return true; +} + +// Re-export ESC for convenience +export { ESC }; diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/soundNotification.test.ts b/dexto/packages/cli/src/cli/ink-cli/utils/soundNotification.test.ts new file mode 100644 index 00000000..e0d1dda3 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/soundNotification.test.ts @@ -0,0 +1,119 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { exec } from 'child_process'; +import { SoundNotificationService, type SoundConfig } from './soundNotification.js'; + +// Mock child_process exec +vi.mock('child_process', () => ({ + exec: vi.fn((_command, _options, callback) => { + // Handle both (command, callback) and (command, options, callback) signatures + const cb = typeof _options === 'function' ? _options : callback; + if (cb) cb(null, '', ''); + }), +})); + +const mockedExec = vi.mocked(exec); + +// Full config for testing (mirrors defaults from PreferenceSoundsSchema) +const TEST_CONFIG: SoundConfig = { + enabled: true, + onApprovalRequired: true, + onTaskComplete: true, +}; + +describe('SoundNotificationService', () => { + let originalStdoutWrite: typeof process.stdout.write; + let writeSpy: ReturnType<typeof vi.fn>; + + beforeEach(() => { + originalStdoutWrite = process.stdout.write; + writeSpy = vi.fn(); + process.stdout.write = writeSpy as unknown as typeof process.stdout.write; + }); + + afterEach(() => { + process.stdout.write = originalStdoutWrite; + }); + + describe('constructor', () => { + it('should accept full config', () => { + const service = new SoundNotificationService(TEST_CONFIG); + expect(service.getConfig()).toEqual(TEST_CONFIG); + }); + + it('should allow disabling specific sounds', () => { + const config: SoundConfig = { + enabled: true, + onApprovalRequired: false, + onTaskComplete: true, + }; + const service = new SoundNotificationService(config); + expect(service.getConfig()).toEqual(config); + }); + }); + + describe('setConfig', () => { + it('should update config', () => { + const service = new SoundNotificationService({ ...TEST_CONFIG, enabled: false }); + service.setConfig({ enabled: true }); + expect(service.getConfig().enabled).toBe(true); + }); + }); + + describe('playApprovalSound', () => { + it('should not play when sounds disabled', () => { + const service = new SoundNotificationService({ ...TEST_CONFIG, enabled: false }); + service.playApprovalSound(); + // No bell should be written when disabled + expect(writeSpy).not.toHaveBeenCalled(); + }); + + it('should not play when approval sounds disabled', () => { + const service = new SoundNotificationService({ + ...TEST_CONFIG, + onApprovalRequired: false, + }); + service.playApprovalSound(); + // The service checks onApprovalRequired before playing + expect(writeSpy).not.toHaveBeenCalled(); + }); + + it('should attempt to play sound when enabled', () => { + const service = new SoundNotificationService(TEST_CONFIG); + mockedExec.mockClear(); + writeSpy.mockClear(); + service.playApprovalSound(); + // Should either call exec (platform sound) or write bell (fallback) + const soundAttempted = + mockedExec.mock.calls.length > 0 || writeSpy.mock.calls.length > 0; + expect(soundAttempted).toBe(true); + }); + }); + + describe('playCompleteSound', () => { + it('should not play when sounds disabled', () => { + const service = new SoundNotificationService({ ...TEST_CONFIG, enabled: false }); + service.playCompleteSound(); + expect(writeSpy).not.toHaveBeenCalled(); + }); + + it('should not play when complete sounds disabled', () => { + const service = new SoundNotificationService({ + ...TEST_CONFIG, + onTaskComplete: false, + }); + service.playCompleteSound(); + expect(writeSpy).not.toHaveBeenCalled(); + }); + + it('should attempt to play sound when enabled', () => { + const service = new SoundNotificationService(TEST_CONFIG); + mockedExec.mockClear(); + writeSpy.mockClear(); + service.playCompleteSound(); + // Should either call exec (platform sound) or write bell (fallback) + const soundAttempted = + mockedExec.mock.calls.length > 0 || writeSpy.mock.calls.length > 0; + expect(soundAttempted).toBe(true); + }); + }); +}); diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/soundNotification.ts b/dexto/packages/cli/src/cli/ink-cli/utils/soundNotification.ts new file mode 100644 index 00000000..02457757 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/soundNotification.ts @@ -0,0 +1,243 @@ +/** + * Sound Notification Utility + * + * Plays system sounds for CLI notifications like approval requests and task completion. + * Uses platform-specific commands with fallback to terminal bell. + * + * Sound files should be placed in ~/.dexto/sounds/ or use system defaults. + */ + +import { exec } from 'child_process'; +import { existsSync } from 'fs'; +import { join } from 'path'; +import { homedir, platform } from 'os'; + +export type SoundType = 'approval' | 'complete'; + +/** + * Platform-specific default sound paths + */ +const PLATFORM_SOUNDS: Record<string, Record<SoundType, string>> = { + darwin: { + // macOS system sounds + approval: '/System/Library/Sounds/Blow.aiff', + complete: '/System/Library/Sounds/Glass.aiff', + }, + linux: { + // Common Linux sound paths (freedesktop) + approval: '/usr/share/sounds/freedesktop/stereo/message-new-instant.oga', + complete: '/usr/share/sounds/freedesktop/stereo/complete.oga', + }, + win32: { + // Windows system sounds (handled differently via PowerShell) + approval: 'SystemAsterisk', + complete: 'SystemHand', + }, +}; + +/** + * Get custom sound path from ~/.dexto/sounds/ + */ +function getCustomSoundPath(soundType: SoundType): string | null { + const dextoSoundsDir = join(homedir(), '.dexto', 'sounds'); + const extensions = ['.wav', '.mp3', '.ogg', '.aiff', '.m4a']; + + for (const ext of extensions) { + const customPath = join(dextoSoundsDir, `${soundType}${ext}`); + if (existsSync(customPath)) { + return customPath; + } + } + + return null; +} + +/** + * Play a sound file using platform-specific command + */ +function playSound(soundPath: string): void { + const currentPlatform = platform(); + + let command: string; + + switch (currentPlatform) { + case 'darwin': + // macOS: use afplay + command = `afplay "${soundPath}"`; + break; + + case 'linux': + // Linux: try paplay (PulseAudio), then aplay (ALSA) + if (soundPath.endsWith('.oga') || soundPath.endsWith('.ogg')) { + command = `paplay "${soundPath}" 2>/dev/null || ogg123 -q "${soundPath}" 2>/dev/null`; + } else { + command = `paplay "${soundPath}" 2>/dev/null || aplay -q "${soundPath}" 2>/dev/null`; + } + break; + + case 'win32': + // Windows: use PowerShell with System.Media.SystemSounds + // For Windows system sounds, soundPath is the sound name + if ( + ['SystemAsterisk', 'SystemHand', 'SystemExclamation', 'SystemQuestion'].includes( + soundPath + ) + ) { + command = `powershell -c "[System.Media.SystemSounds]::${soundPath.replace('System', '')}.Play()"`; + } else { + // For custom files, use SoundPlayer + command = `powershell -c "(New-Object System.Media.SoundPlayer '${soundPath}').PlaySync()"`; + } + break; + + default: + // Fallback: try to use terminal bell + playTerminalBell(); + return; + } + + // Execute sound command asynchronously (fire and forget) + // Use 5s timeout to prevent hanging processes (e.g., audio device issues) + exec(command, { timeout: 5000 }, (error) => { + if (error) { + // Silently fall back to terminal bell on error + playTerminalBell(); + } + }); +} + +/** + * Play terminal bell (works in most terminals) + */ +function playTerminalBell(): void { + process.stdout.write('\x07'); +} + +/** + * Play a notification sound + * + * @param soundType - Type of sound to play ('approval' or 'complete') + * + * @example + * ```typescript + * // Play approval required sound + * playNotificationSound('approval'); + * + * // Play task complete sound + * playNotificationSound('complete'); + * ``` + */ +export function playNotificationSound(soundType: SoundType): void { + const currentPlatform = platform(); + + // Check for custom sound first + const customSound = getCustomSoundPath(soundType); + if (customSound) { + playSound(customSound); + return; + } + + // Use platform default + const platformSounds = PLATFORM_SOUNDS[currentPlatform]; + if (platformSounds) { + const defaultSound = platformSounds[soundType]; + if (defaultSound) { + // For macOS and Linux, check if file exists + if (currentPlatform !== 'win32') { + if (existsSync(defaultSound)) { + playSound(defaultSound); + } else { + // File doesn't exist, use bell + playTerminalBell(); + } + } else { + // Windows uses system sound names + playSound(defaultSound); + } + return; + } + } + + // Fallback to terminal bell + playTerminalBell(); +} + +/** + * Sound configuration interface + * + * Note: Default values are defined in PreferenceSoundsSchema (packages/agent-management/src/preferences/schemas.ts) + * which is the single source of truth for sound preferences. + */ +export interface SoundConfig { + enabled: boolean; + onApprovalRequired: boolean; + onTaskComplete: boolean; +} + +/** + * Sound notification service that respects configuration + * + * All fields are required - callers should provide complete config from preferences. + * Default values come from PreferenceSoundsSchema in @dexto/agent-management. + */ +export class SoundNotificationService { + private config: SoundConfig; + + constructor(config: SoundConfig) { + this.config = { ...config }; + } + + /** + * Update configuration + */ + setConfig(config: Partial<SoundConfig>): void { + this.config = { ...this.config, ...config }; + } + + /** + * Get current configuration + */ + getConfig(): SoundConfig { + return { ...this.config }; + } + + /** + * Play approval required sound if enabled + */ + playApprovalSound(): void { + if (this.config.enabled && this.config.onApprovalRequired) { + playNotificationSound('approval'); + } + } + + /** + * Play task complete sound if enabled + */ + playCompleteSound(): void { + if (this.config.enabled && this.config.onTaskComplete) { + playNotificationSound('complete'); + } + } +} + +/** + * Singleton instance for global use + * Initialize with loadGlobalPreferences() in CLI startup + */ +let globalSoundService: SoundNotificationService | null = null; + +/** + * Get the global sound notification service + * @returns The global service, or null if not initialized + */ +export function getSoundService(): SoundNotificationService | null { + return globalSoundService; +} + +/** + * Initialize the global sound service with configuration + */ +export function initializeSoundService(config: SoundConfig): SoundNotificationService { + globalSoundService = new SoundNotificationService(config); + return globalSoundService; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/streamSplitter.ts b/dexto/packages/cli/src/cli/ink-cli/utils/streamSplitter.ts new file mode 100644 index 00000000..75ec15f7 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/streamSplitter.ts @@ -0,0 +1,182 @@ +/** + * Stream Splitter Utilities + * + * Provides markdown-aware splitting for streaming content to reduce flickering. + * + * Approach: + * - Find safe split points at paragraph boundaries (\n\n) + * - Protect code blocks from being split mid-block + * - Allow progressive finalization during streaming + */ + +/** + * Checks if a given index is inside a fenced code block (``` ... ```) + */ +function isIndexInsideCodeBlock(content: string, indexToTest: number): boolean { + let fenceCount = 0; + let searchPos = 0; + + while (searchPos < content.length) { + const nextFence = content.indexOf('```', searchPos); + if (nextFence === -1 || nextFence >= indexToTest) { + break; + } + fenceCount++; + searchPos = nextFence + 3; + } + + return fenceCount % 2 === 1; +} + +/** + * Finds the starting index of the code block that encloses the given index. + * Returns -1 if the index is not inside a code block. + */ +function findEnclosingCodeBlockStart(content: string, index: number): number { + if (!isIndexInsideCodeBlock(content, index)) { + return -1; + } + + let currentSearchPos = 0; + while (currentSearchPos < index) { + const blockStartIndex = content.indexOf('```', currentSearchPos); + if (blockStartIndex === -1 || blockStartIndex >= index) { + break; + } + + const blockEndIndex = content.indexOf('```', blockStartIndex + 3); + if (blockStartIndex < index) { + if (blockEndIndex === -1 || index < blockEndIndex + 3) { + return blockStartIndex; + } + } + + if (blockEndIndex === -1) break; + currentSearchPos = blockEndIndex + 3; + } + + return -1; +} + +/** + * Find the last safe split point in the content. + * + * Safe split points are: + * 1. After paragraph breaks (\n\n) that are not inside code blocks + * 2. Before code blocks if we're inside one + * + * Returns the index to split at, or content.length if no split is needed. + */ +export function findLastSafeSplitPoint(content: string): number { + // If we're inside a code block at the end, split before it + const enclosingBlockStart = findEnclosingCodeBlockStart(content, content.length); + if (enclosingBlockStart !== -1) { + return enclosingBlockStart; + } + + // Search for the last double newline (\n\n) not in a code block + let searchStartIndex = content.length; + while (searchStartIndex >= 0) { + const dnlIndex = content.lastIndexOf('\n\n', searchStartIndex); + if (dnlIndex === -1) { + break; + } + + const potentialSplitPoint = dnlIndex + 2; + if (!isIndexInsideCodeBlock(content, potentialSplitPoint)) { + return potentialSplitPoint; + } + + // If potentialSplitPoint was inside a code block, + // search before the \n\n we just found + searchStartIndex = dnlIndex - 1; + } + + // No safe split point found, return full length + return content.length; +} + +/** + * Find the last newline that's not inside a code block. + * Used for line-based batching. + */ +export function findLastSafeNewline(content: string): number { + let searchPos = content.length; + + while (searchPos > 0) { + const nlIndex = content.lastIndexOf('\n', searchPos - 1); + if (nlIndex === -1) { + break; + } + + if (!isIndexInsideCodeBlock(content, nlIndex)) { + return nlIndex + 1; // Return position after the newline + } + + searchPos = nlIndex; + } + + return -1; // No safe newline found +} + +/** + * Minimum content length before considering a paragraph split (\n\n). + */ +const MIN_PARAGRAPH_SPLIT_LENGTH = 200; + +/** + * Maximum content length before forcing a line-based split. + * This prevents excessive accumulation that causes flickering. + * Roughly 3-4 lines of terminal width (~80 chars each). + */ +const MAX_PENDING_LENGTH = 300; + +/** + * Determines if content should be split for progressive finalization. + * + * Strategy: + * 1. First try paragraph splits (\n\n) for clean breaks + * 2. If content exceeds MAX_PENDING_LENGTH with no paragraph break, + * fall back to line-based splits (\n) to prevent flickering + * + * Returns: + * - { shouldSplit: false } if no split needed + * - { shouldSplit: true, splitIndex, before, after } if split found + */ +export function checkForSplit(content: string): { + shouldSplit: boolean; + splitIndex?: number; + before?: string; + after?: string; +} { + // Don't split very small content + if (content.length < MIN_PARAGRAPH_SPLIT_LENGTH) { + return { shouldSplit: false }; + } + + // Try paragraph-based split first (cleaner breaks) + const paragraphSplitPoint = findLastSafeSplitPoint(content); + if (paragraphSplitPoint > 80 && paragraphSplitPoint < content.length - 20) { + return { + shouldSplit: true, + splitIndex: paragraphSplitPoint, + before: content.substring(0, paragraphSplitPoint), + after: content.substring(paragraphSplitPoint), + }; + } + + // If content is getting too long, force a line-based split to reduce flickering + if (content.length > MAX_PENDING_LENGTH) { + const lineSplitPoint = findLastSafeNewline(content); + if (lineSplitPoint > 80 && lineSplitPoint < content.length - 20) { + return { + shouldSplit: true, + splitIndex: lineSplitPoint, + before: content.substring(0, lineSplitPoint), + after: content.substring(lineSplitPoint), + }; + } + } + + return { shouldSplit: false }; +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/textUtils.ts b/dexto/packages/cli/src/cli/ink-cli/utils/textUtils.ts new file mode 100644 index 00000000..5ce4ecc9 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/textUtils.ts @@ -0,0 +1,289 @@ +/** + * Unicode-aware text utilities for terminal rendering + * + * These utilities work at the code-point level rather than UTF-16 code units, + * so that surrogate-pair emoji count as one "column". + */ + +import stripAnsi from 'strip-ansi'; +import stringWidth from 'string-width'; + +// Cache for code points to reduce GC pressure +const codePointsCache = new Map<string, string[]>(); +const MAX_STRING_LENGTH_TO_CACHE = 1000; + +/** + * Convert a string to an array of code points (handles surrogate pairs correctly) + */ +export function toCodePoints(str: string): string[] { + // ASCII fast path - check if all chars are ASCII (0-127) + let isAscii = true; + for (let i = 0; i < str.length; i++) { + if (str.charCodeAt(i) > 127) { + isAscii = false; + break; + } + } + if (isAscii) { + return str.split(''); + } + + // Cache short strings + // We return a copy ([...cached]) to prevent callers from mutating the cache. + // This has O(n) overhead but n is bounded by MAX_STRING_LENGTH_TO_CACHE (1000). + if (str.length <= MAX_STRING_LENGTH_TO_CACHE) { + const cached = codePointsCache.get(str); + if (cached) { + return [...cached]; + } + } + + const result = Array.from(str); + + // Cache result + if (str.length <= MAX_STRING_LENGTH_TO_CACHE) { + codePointsCache.set(str, result); + } + + return result; +} + +/** + * Get the code-point length of a string + */ +export function cpLen(str: string): number { + return toCodePoints(str).length; +} + +/** + * Slice a string by code-point indices + */ +export function cpSlice(str: string, start: number, end?: number): string { + const arr = toCodePoints(str).slice(start, end); + return arr.join(''); +} + +/** + * Strip characters that can break terminal rendering. + * + * Characters stripped: + * - ANSI escape sequences + * - C0 control chars (0x00-0x1F) except CR/LF/TAB + * - C1 control chars (0x80-0x9F) + * + * Characters preserved: + * - All printable Unicode including emojis + * - CR/LF (0x0D/0x0A) - needed for line breaks + * - TAB (0x09) + */ +export function stripUnsafeCharacters(str: string): string { + const strippedAnsi = stripAnsi(str); + + return toCodePoints(strippedAnsi) + .filter((char) => { + const code = char.codePointAt(0); + if (code === undefined) return false; + + // Preserve CR/LF/TAB for line handling + if (code === 0x0a || code === 0x0d || code === 0x09) return true; + + // Remove C0 control chars (except CR/LF/TAB) + if (code >= 0x00 && code <= 0x1f) return false; + + // Remove DEL control char + if (code === 0x7f) return false; + + // Remove C1 control chars (0x80-0x9f) + if (code >= 0x80 && code <= 0x9f) return false; + + // Preserve all other characters including Unicode/emojis + return true; + }) + .join(''); +} + +// String width caching for performance optimization +const stringWidthCache = new Map<string, number>(); + +/** + * Cached version of stringWidth function for better performance + */ +export function getCachedStringWidth(str: string): number { + // ASCII printable chars have width 1 + if (/^[\x20-\x7E]*$/.test(str)) { + return str.length; + } + + if (stringWidthCache.has(str)) { + return stringWidthCache.get(str)!; + } + + const width = stringWidth(str); + stringWidthCache.set(str, width); + + return width; +} + +/** + * Clear the string width cache + */ +export function clearStringWidthCache(): void { + stringWidthCache.clear(); +} + +// Word character detection helpers +// These accept string | undefined to safely handle array bounds access (chars[i] may be undefined) +export const isWordCharStrict = (char: string | undefined): boolean => + char !== undefined && /[\w\p{L}\p{N}]/u.test(char); + +export const isWhitespace = (char: string | undefined): boolean => + char !== undefined && /\s/.test(char); + +export const isCombiningMark = (char: string | undefined): boolean => + char !== undefined && /\p{M}/u.test(char); + +export const isWordCharWithCombining = (char: string | undefined): boolean => + isWordCharStrict(char) || isCombiningMark(char); + +// Get the script of a character (simplified for common scripts) +export const getCharScript = (char: string | undefined): string => { + if (!char) return 'other'; + if (/[\p{Script=Latin}]/u.test(char)) return 'latin'; + if (/[\p{Script=Han}]/u.test(char)) return 'han'; + if (/[\p{Script=Arabic}]/u.test(char)) return 'arabic'; + if (/[\p{Script=Hiragana}]/u.test(char)) return 'hiragana'; + if (/[\p{Script=Katakana}]/u.test(char)) return 'katakana'; + if (/[\p{Script=Cyrillic}]/u.test(char)) return 'cyrillic'; + return 'other'; +}; + +export const isDifferentScript = ( + char1: string | undefined, + char2: string | undefined +): boolean => { + if (!isWordCharStrict(char1) || !isWordCharStrict(char2)) return false; + return getCharScript(char1) !== getCharScript(char2); +}; + +// Initialize segmenter for word boundary detection +const segmenter = new Intl.Segmenter(undefined, { granularity: 'word' }); + +/** + * Find previous word boundary in a line + */ +export function findPrevWordBoundary(line: string, cursorCol: number): number { + const codePoints = toCodePoints(line); + // Convert cursorCol (CP index) to string index + const prefix = codePoints.slice(0, cursorCol).join(''); + const cursorIdx = prefix.length; + + let targetIdx = 0; + + for (const seg of segmenter.segment(line)) { + if (seg.index >= cursorIdx) break; + if (seg.isWordLike) { + targetIdx = seg.index; + } + } + + return toCodePoints(line.slice(0, targetIdx)).length; +} + +/** + * Find next word boundary in a line + */ +export function findNextWordBoundary(line: string, cursorCol: number): number { + const codePoints = toCodePoints(line); + const prefix = codePoints.slice(0, cursorCol).join(''); + const cursorIdx = prefix.length; + + let targetIdx = line.length; + + for (const seg of segmenter.segment(line)) { + const segEnd = seg.index + seg.segment.length; + if (segEnd > cursorIdx) { + if (seg.isWordLike) { + targetIdx = segEnd; + break; + } + } + } + + return toCodePoints(line.slice(0, targetIdx)).length; +} + +/** + * Find next word start within a line, starting from col + */ +export function findNextWordStartInLine(line: string, col: number): number | null { + const chars = toCodePoints(line); + let i = col; + + if (i >= chars.length) return null; + + const currentChar = chars[i]!; + + // Skip current word/sequence based on character type + if (isWordCharStrict(currentChar)) { + while (i < chars.length && isWordCharWithCombining(chars[i]!)) { + if ( + i + 1 < chars.length && + isWordCharStrict(chars[i + 1]!) && + isDifferentScript(chars[i]!, chars[i + 1]!) + ) { + i++; + break; + } + i++; + } + } else if (!isWhitespace(currentChar)) { + while (i < chars.length && !isWordCharStrict(chars[i]!) && !isWhitespace(chars[i]!)) { + i++; + } + } + + // Skip whitespace + while (i < chars.length && isWhitespace(chars[i]!)) { + i++; + } + + return i < chars.length ? i : null; +} + +/** + * Find previous word start within a line + */ +export function findPrevWordStartInLine(line: string, col: number): number | null { + const chars = toCodePoints(line); + let i = col; + + if (i <= 0) return null; + + i--; + + // Skip whitespace moving backwards + while (i >= 0 && isWhitespace(chars[i]!)) { + i--; + } + + if (i < 0) return null; + + if (isWordCharStrict(chars[i]!)) { + while (i >= 0 && isWordCharStrict(chars[i]!)) { + if ( + i - 1 >= 0 && + isWordCharStrict(chars[i - 1]!) && + isDifferentScript(chars[i]!, chars[i - 1]!) + ) { + return i; + } + i--; + } + return i + 1; + } else { + while (i >= 0 && !isWordCharStrict(chars[i]!) && !isWhitespace(chars[i]!)) { + i--; + } + return i + 1; + } +} diff --git a/dexto/packages/cli/src/cli/ink-cli/utils/toolUtils.ts b/dexto/packages/cli/src/cli/ink-cli/utils/toolUtils.ts new file mode 100644 index 00000000..4fea5237 --- /dev/null +++ b/dexto/packages/cli/src/cli/ink-cli/utils/toolUtils.ts @@ -0,0 +1,39 @@ +/** + * Tool utility functions for CLI + */ + +/** + * Check if a tool is an edit or write file tool. + * Used for "accept all edits" mode which auto-approves these tools. + */ +export function isEditWriteTool(toolName: string | undefined): boolean { + return ( + toolName === 'internal--edit_file' || + toolName === 'internal--write_file' || + toolName === 'custom--write_file' || + toolName === 'custom--edit_file' || + toolName === 'edit_file' || + toolName === 'write_file' + ); +} + +/** + * Check if a tool is a plan update tool. + * Used for "accept all edits" mode which also auto-approves plan updates + * (typically marking tasks as complete during implementation). + */ +export function isPlanUpdateTool(toolName: string | undefined): boolean { + return ( + toolName === 'plan_update' || + toolName === 'custom--plan_update' || + toolName === 'internal--plan_update' + ); +} + +/** + * Check if a tool should be auto-approved in "accept all edits" mode. + * Includes file edit/write tools and plan update tools. + */ +export function isAutoApprovableInEditMode(toolName: string | undefined): boolean { + return isEditWriteTool(toolName) || isPlanUpdateTool(toolName); +} diff --git a/dexto/packages/cli/src/cli/utils/api-key-setup.ts b/dexto/packages/cli/src/cli/utils/api-key-setup.ts new file mode 100644 index 00000000..cc96df63 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/api-key-setup.ts @@ -0,0 +1,530 @@ +// packages/cli/src/cli/utils/api-key-setup.ts + +import * as p from '@clack/prompts'; +import chalk from 'chalk'; +import type { LLMProvider } from '@dexto/core'; +import { logger, getExecutionContext } from '@dexto/core'; +import { saveProviderApiKey } from '@dexto/agent-management'; +import { applyLayeredEnvironmentLoading } from '../../utils/env.js'; +import { + getProviderDisplayName, + validateApiKeyFormat, + getProviderInstructions, + openApiKeyUrl, + getProviderInfo, + getProviderEnvVar, +} from './provider-setup.js'; +import { verifyApiKey } from './api-key-verification.js'; + +export interface ApiKeySetupResult { + success: boolean; + apiKey?: string; + cancelled?: boolean; + skipped?: boolean; + error?: string; +} + +/** + * Interactively prompts the user to set up an API key for a specific provider. + * Includes browser auto-open, format validation with hints, and API key verification. + * + * @param provider - The specific provider that needs API key setup + * @param options - Configuration options + * @returns Result indicating success, cancellation, or error + */ +export async function interactiveApiKeySetup( + provider: LLMProvider, + options: { + exitOnCancel?: boolean; + skipVerification?: boolean; + model?: string; + } = {} +): Promise<ApiKeySetupResult> { + const { exitOnCancel = true, skipVerification = false, model } = options; + + try { + p.intro(chalk.cyan('🔑 API Key Setup')); + + const instructions = getProviderInstructions(provider); + const providerInfo = getProviderInfo(provider); + + // Show targeted message for the required provider + if (instructions) { + p.note( + `Your configuration requires a ${getProviderDisplayName(provider)} API key.\n\n` + + instructions.content, + chalk.bold(instructions.title) + ); + } + + // Ask what they want to do - with "Open in browser" option + const hasApiKeyUrl = Boolean(providerInfo?.apiKeyUrl); + const actionOptions = [ + ...(hasApiKeyUrl + ? [ + { + value: 'open-browser' as const, + label: `${chalk.green('→')} Get API key (opens browser)`, + hint: "We'll wait while you grab one", + }, + ] + : []), + { + value: 'setup' as const, + label: `${chalk.cyan('●')} Paste existing key`, + hint: 'I already have an API key', + }, + { + value: 'skip' as const, + label: `${chalk.gray('○')} Set up later`, + hint: 'Continue without key for now', + }, + ]; + + const action = await p.select({ + message: 'What would you like to do?', + options: actionOptions, + }); + + if (p.isCancel(action)) { + p.cancel('Setup cancelled'); + if (exitOnCancel) process.exit(0); + return { success: false, cancelled: true }; + } + + if (action === 'skip') { + const envVar = getProviderEnvVar(provider); + p.note( + `You can configure your API key later:\n\n` + + `${chalk.cyan('Option 1:')} Run ${chalk.bold('dexto setup')}\n\n` + + `${chalk.cyan('Option 2:')} Set environment variable manually:\n` + + ` ${chalk.dim(`export ${envVar}=your-key-here`)}\n` + + ` ${chalk.dim('Add to ~/.bashrc or ~/.zshrc to persist')}`, + 'Setup Later' + ); + return { success: true, skipped: true }; + } + + // Open browser if requested + if (action === 'open-browser') { + const spinner = p.spinner(); + spinner.start('Opening browser...'); + const opened = await openApiKeyUrl(provider); + if (opened) { + spinner.stop('Browser opened! Get your API key and paste it below.'); + } else { + spinner.stop(`Could not open browser. Visit: ${providerInfo?.apiKeyUrl}`); + } + } + + // Prompt for API key with improved validation + const apiKey = await promptForApiKey(provider); + if (!apiKey) { + if (exitOnCancel) process.exit(0); + return { success: false, cancelled: true }; + } + + // Verify API key works (unless skipped) + if (!skipVerification) { + const verificationResult = await verifyApiKeyWithSpinner(provider, apiKey, model); + if (!verificationResult.success) { + p.log.error(verificationResult.error || 'API key verification failed'); + + const retryAction = await p.select({ + message: 'What would you like to do?', + options: [ + { + value: 'retry' as const, + label: 'Try a different API key', + hint: 'Enter a new API key', + }, + { + value: 'save-anyway' as const, + label: 'Save anyway', + hint: 'Key might still work - save and continue', + }, + { + value: 'skip' as const, + label: 'Skip for now', + hint: 'Continue without saving - configure later', + }, + ], + }); + + if (p.isCancel(retryAction)) { + if (exitOnCancel) process.exit(1); + const result: ApiKeySetupResult = { success: false, cancelled: true }; + return result; + } + + if (retryAction === 'retry') { + // Recursive retry + return interactiveApiKeySetup(provider, options); + } + + if (retryAction === 'skip') { + p.log.warn( + 'Skipping API key setup. You can configure it later with: dexto setup' + ); + return { success: true, skipped: true }; + } + + // retryAction === 'save-anyway' - fall through to save + p.log.info('Saving API key despite verification failure...'); + } + } + + // Save API key + const saveResult = await saveApiKeyWithSpinner(provider, apiKey); + if (!saveResult.success) { + showManualSaveInstructions(provider, apiKey); + p.log.warn('You can configure API key later with: dexto setup'); + // Don't fail - return success with the API key for in-memory use + return { success: true, apiKey, skipped: true }; + } + + p.outro(chalk.green('✨ API key setup complete!')); + return { success: true, apiKey }; + } catch (error) { + if (p.isCancel(error)) { + p.cancel('Setup cancelled'); + if (exitOnCancel) process.exit(0); + return { success: false, cancelled: true }; + } + + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error(`API key setup failed: ${errorMessage}`); + + if (exitOnCancel) { + console.error(chalk.red('\n❌ API key setup required to continue.')); + process.exit(1); + } + + return { success: false, error: errorMessage }; + } +} + +/** + * Prompt for API key with format validation and hints + */ +async function promptForApiKey(provider: LLMProvider): Promise<string | null> { + const providerInfo = getProviderInfo(provider); + const formatHint = providerInfo?.apiKeyPrefix + ? chalk.gray(` (starts with ${providerInfo.apiKeyPrefix})`) + : ''; + + const apiKey = await p.password({ + message: `Enter your ${getProviderDisplayName(provider)} API key${formatHint}`, + mask: '*', + validate: (value) => { + const result = validateApiKeyFormat(value, provider); + if (!result.valid) { + return result.error; + } + return undefined; + }, + }); + + if (p.isCancel(apiKey)) { + p.cancel('Setup cancelled'); + return null; + } + + return apiKey.trim(); +} + +/** + * Verify API key with a spinner + */ +async function verifyApiKeyWithSpinner( + provider: LLMProvider, + apiKey: string, + model?: string +): Promise<{ success: boolean; error?: string }> { + const spinner = p.spinner(); + spinner.start('Verifying API key...'); + + try { + const result = await verifyApiKey(provider, apiKey, model); + + if (result.success) { + spinner.stop(chalk.green('✓ API key verified successfully!')); + return { success: true }; + } else { + spinner.stop(chalk.red('✗ API key verification failed')); + const response: { success: boolean; error?: string } = { success: false }; + if (result.error) response.error = result.error; + return response; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + spinner.stop(chalk.red('✗ Verification failed')); + return { success: false, error: errorMessage }; + } +} + +/** + * Save API key with a spinner + */ +async function saveApiKeyWithSpinner( + provider: LLMProvider, + apiKey: string +): Promise<{ success: boolean; error?: string; path?: string }> { + const spinner = p.spinner(); + spinner.start('Saving API key...'); + + try { + const meta = await saveProviderApiKey(provider, apiKey, process.cwd()); + spinner.stop(chalk.green(`✓ API key saved to ${meta.targetEnvPath}`)); + + // Reload environment variables + await applyLayeredEnvironmentLoading(); + + return { success: true, path: meta.targetEnvPath }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + spinner.stop(chalk.red('✗ Failed to save API key')); + logger.error(`Failed to save API key: ${errorMessage}`); + return { success: false, error: errorMessage }; + } +} + +/** + * Show manual setup instructions when automatic save fails + */ +function showManualSaveInstructions(provider: LLMProvider, apiKey: string): void { + const envVar = getProviderEnvVar(provider); + const maskedKey = apiKey.slice(0, 8) + '...' + apiKey.slice(-4); + + const instructions = + getExecutionContext() === 'global-cli' + ? [ + `1. Create or edit: ${chalk.cyan('~/.dexto/.env')}`, + `2. Add this line: ${chalk.rgb(255, 165, 0)(`${envVar}=${maskedKey}`)}`, + `3. Run ${chalk.cyan('dexto')} again`, + ] + : [ + `1. Create or edit: ${chalk.cyan('.env')} in your project`, + `2. Add this line: ${chalk.rgb(255, 165, 0)(`${envVar}=your_api_key`)}`, + `3. Run ${chalk.cyan('dexto')} again`, + ]; + + p.note(instructions.join('\n'), chalk.rgb(255, 165, 0)('Manual Setup Required')); +} + +/** + * Quick check if an API key is already configured for a provider + */ +export function hasApiKeyConfigured(provider: LLMProvider): boolean { + const envVar = getProviderEnvVar(provider); + return Boolean(process.env[envVar]); +} + +/** + * Result when user is prompted about pending API key setup + */ +export interface PendingSetupPromptResult { + action: 'setup' | 'skip' | 'cancel'; + apiKey?: string; +} + +/** + * Prompts user about pending API key setup from a previous incomplete setup. + * This is shown when user previously skipped API key setup and is now trying to run dexto. + * + * @param provider - The provider that needs API key setup + * @param model - The model configured for the provider + * @returns Result indicating which action the user chose + */ +export async function promptForPendingApiKey( + provider: LLMProvider, + model: string +): Promise<PendingSetupPromptResult> { + const providerName = getProviderDisplayName(provider); + + p.intro(chalk.cyan('🔧 Complete Your Setup')); + + p.note( + `You previously set up Dexto with ${chalk.bold(providerName)} but skipped API key configuration.\n\n` + + `To use ${providerName}/${model}, you'll need to add your API key.`, + 'Setup Incomplete' + ); + + const action = await p.select({ + message: 'What would you like to do?', + options: [ + { + value: 'setup' as const, + label: `Set up ${providerName} API key now`, + hint: 'Recommended - complete your setup', + }, + { + value: 'skip' as const, + label: 'Skip for now', + hint: 'Continue without API key - will show error when trying to use LLM', + }, + { + value: 'cancel' as const, + label: 'Exit', + hint: 'Exit and set up later with: dexto setup', + }, + ], + }); + + if (p.isCancel(action)) { + p.cancel('Cancelled'); + return { action: 'cancel' }; + } + + if (action === 'cancel') { + return { action: 'cancel' }; + } + + if (action === 'skip') { + p.log.warn('Continuing without API key. You may see errors when the LLM is invoked.'); + return { action: 'skip' }; + } + + // action === 'setup' - use the existing API key setup flow + const setupResult = await interactiveApiKeySetup(provider, { + exitOnCancel: false, + model, + }); + + if (setupResult.cancelled) { + return { action: 'cancel' }; + } + + if (setupResult.skipped) { + p.log.warn('API key setup skipped. You can configure it later with: dexto setup'); + return { action: 'skip' }; + } + + if (setupResult.success && setupResult.apiKey) { + return { action: 'setup', apiKey: setupResult.apiKey }; + } + + return { action: 'skip' }; +} + +/** + * Result when user is prompted about missing API key for a specific agent + */ +export interface AgentApiKeyPromptResult { + action: 'add-key' | 'use-default' | 'cancel'; + apiKey?: string; // Only set if action is 'add-key' and key was successfully added +} + +/** + * Prompts user when loading a specific agent that requires an API key they don't have. + * Offers options to add the API key, continue with their default LLM, or cancel. + * + * @param agentProvider - The provider the agent requires + * @param agentModel - The model the agent uses + * @param userProvider - The user's default provider (from preferences) + * @param userModel - The user's default model (from preferences) + * @returns Result indicating which action the user chose + */ +export async function promptForMissingAgentApiKey( + agentProvider: LLMProvider, + agentModel: string, + userProvider: LLMProvider, + userModel: string +): Promise<AgentApiKeyPromptResult> { + const agentProviderName = getProviderDisplayName(agentProvider); + const userProviderName = getProviderDisplayName(userProvider); + const userDefault = `${userProviderName}/${userModel}`; + + p.intro(chalk.cyan('🔧 Agent Configuration')); + + p.note( + `This agent is configured for ${chalk.bold(`${agentProviderName}/${agentModel}`)}\n` + + `but you don't have an API key set up for ${agentProviderName}.\n\n` + + `Your default LLM is ${chalk.green(userDefault)}.`, + 'Missing API Key' + ); + + const action = await p.select({ + message: 'What would you like to do?', + options: [ + { + value: 'use-default' as const, + label: `Continue with ${userDefault}`, + hint: 'Agent will use your default LLM instead (behavior may differ)', + }, + { + value: 'add-key' as const, + label: `Set up ${agentProviderName} API key`, + hint: `Configure ${agentProviderName} to run agent as intended`, + }, + { + value: 'cancel' as const, + label: 'Cancel', + hint: 'Exit without running the agent', + }, + ], + }); + + if (p.isCancel(action)) { + p.cancel('Cancelled'); + return { action: 'cancel' }; + } + + if (action === 'cancel') { + return { action: 'cancel' }; + } + + if (action === 'use-default') { + p.log.warn( + `Using ${userDefault} instead of ${agentProviderName}/${agentModel}. ` + + `Agent behavior may differ from intended.` + ); + return { action: 'use-default' }; + } + + // action === 'add-key' - use the existing API key setup flow + const setupResult = await interactiveApiKeySetup(agentProvider, { + exitOnCancel: false, + model: agentModel, + }); + + if (setupResult.cancelled) { + return { action: 'cancel' }; + } + + if (setupResult.skipped) { + // User skipped in the API key setup - ask again what they want to do + const skipAction = await p.select({ + message: 'API key setup was skipped. What would you like to do?', + options: [ + { + value: 'use-default' as const, + label: `Continue with ${userDefault}`, + hint: 'Agent will use your default LLM instead', + }, + { + value: 'cancel' as const, + label: 'Cancel', + hint: 'Exit without running the agent', + }, + ], + }); + + if (p.isCancel(skipAction) || skipAction === 'cancel') { + return { action: 'cancel' }; + } + + p.log.warn( + `Using ${userDefault} instead of ${agentProviderName}/${agentModel}. ` + + `Agent behavior may differ from intended.` + ); + return { action: 'use-default' }; + } + + if (setupResult.success && setupResult.apiKey) { + return { action: 'add-key', apiKey: setupResult.apiKey }; + } + + // Shouldn't reach here, but default to cancel + return { action: 'cancel' }; +} diff --git a/dexto/packages/cli/src/cli/utils/api-key-verification.ts b/dexto/packages/cli/src/cli/utils/api-key-verification.ts new file mode 100644 index 00000000..fe77f786 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/api-key-verification.ts @@ -0,0 +1,252 @@ +// packages/cli/src/cli/utils/api-key-verification.ts + +import type { LLMProvider } from '@dexto/core'; +import { logger } from '@dexto/core'; + +export interface VerificationResult { + success: boolean; + error?: string; + modelUsed?: string; +} + +/** + * Verify an API key by making a minimal test request to the provider. + * Uses provider-specific endpoints for efficient validation. + * + * @param provider - The LLM provider + * @param apiKey - The API key to verify + * @param model - Optional specific model to test with + * @returns Verification result + */ +export async function verifyApiKey( + provider: LLMProvider, + apiKey: string, + _model?: string +): Promise<VerificationResult> { + try { + switch (provider) { + case 'openai': + return await verifyOpenAI(apiKey); + case 'anthropic': + return await verifyAnthropic(apiKey); + case 'google': + return await verifyGoogle(apiKey); + case 'groq': + return await verifyGroq(apiKey); + case 'xai': + return await verifyXAI(apiKey); + case 'cohere': + return await verifyCohere(apiKey); + case 'openrouter': + return await verifyOpenRouter(apiKey); + case 'glama': + return await verifyGlama(apiKey); + case 'openai-compatible': + case 'litellm': + // For custom endpoints, we can't verify without a baseURL + // Just do basic format check + return { success: true, modelUsed: 'custom' }; + case 'vertex': + case 'bedrock': + // These use cloud credentials, not API keys + // Skip verification + return { success: true, modelUsed: 'cloud-auth' }; + default: + // Unknown provider - skip verification + return { success: true, modelUsed: 'unknown' }; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + logger.debug(`API key verification failed for ${provider}: ${errorMessage}`); + return { success: false, error: errorMessage }; + } +} + +/** + * Verify OpenAI API key using the models endpoint + */ +async function verifyOpenAI(apiKey: string): Promise<VerificationResult> { + const response = await fetch('https://api.openai.com/v1/models', { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + if (response.ok) { + return { success: true, modelUsed: 'models-list' }; + } + + const error = await parseErrorResponse(response); + return { success: false, error }; +} + +/** + * Verify Anthropic API key using a minimal messages request + */ +async function verifyAnthropic(apiKey: string): Promise<VerificationResult> { + // Anthropic doesn't have a models endpoint, so we make a minimal request + // that will fail fast if the key is invalid + const response = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify({ + model: 'claude-3-5-haiku-20241022', + max_tokens: 1, + messages: [{ role: 'user', content: 'hi' }], + }), + }); + + // 200 = success, 400 = bad request (but key is valid), 401/403 = invalid key + if (response.ok || response.status === 400) { + return { success: true, modelUsed: 'claude-3-5-haiku-20241022' }; + } + + const error = await parseErrorResponse(response); + return { success: false, error }; +} + +/** + * Verify Google AI API key using the models endpoint + */ +async function verifyGoogle(apiKey: string): Promise<VerificationResult> { + const response = await fetch( + `https://generativelanguage.googleapis.com/v1/models?key=${apiKey}`, + { + method: 'GET', + } + ); + + if (response.ok) { + return { success: true, modelUsed: 'models-list' }; + } + + const error = await parseErrorResponse(response); + return { success: false, error }; +} + +/** + * Verify Groq API key using the models endpoint + */ +async function verifyGroq(apiKey: string): Promise<VerificationResult> { + const response = await fetch('https://api.groq.com/openai/v1/models', { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + if (response.ok) { + return { success: true, modelUsed: 'models-list' }; + } + + const error = await parseErrorResponse(response); + return { success: false, error }; +} + +/** + * Verify xAI API key using the models endpoint + */ +async function verifyXAI(apiKey: string): Promise<VerificationResult> { + const response = await fetch('https://api.x.ai/v1/models', { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + if (response.ok) { + return { success: true, modelUsed: 'models-list' }; + } + + const error = await parseErrorResponse(response); + return { success: false, error }; +} + +/** + * Verify Cohere API key using a check-api-key endpoint or models list + */ +async function verifyCohere(apiKey: string): Promise<VerificationResult> { + const response = await fetch('https://api.cohere.com/v2/models', { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + if (response.ok) { + return { success: true, modelUsed: 'models-list' }; + } + + const error = await parseErrorResponse(response); + return { success: false, error }; +} + +/** + * Verify OpenRouter API key using the auth endpoint + */ +async function verifyOpenRouter(apiKey: string): Promise<VerificationResult> { + const response = await fetch('https://openrouter.ai/api/v1/auth/key', { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + if (response.ok) { + return { success: true, modelUsed: 'auth-check' }; + } + + const error = await parseErrorResponse(response); + return { success: false, error }; +} + +/** + * Verify Glama API key using the models endpoint + */ +async function verifyGlama(apiKey: string): Promise<VerificationResult> { + const response = await fetch('https://glama.ai/api/gateway/openai/v1/models', { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + if (response.ok) { + return { success: true, modelUsed: 'models-list' }; + } + + const error = await parseErrorResponse(response); + return { success: false, error }; +} + +/** + * Parse error response from provider API + */ +async function parseErrorResponse(response: Response): Promise<string> { + const status = response.status; + + // Common status codes + if (status === 401) { + return 'Invalid API key - authentication failed'; + } + if (status === 403) { + return 'API key does not have permission to access this resource'; + } + if (status === 429) { + return 'Rate limit exceeded - but API key appears valid'; + } + + // Try to parse JSON error + try { + const json = await response.json(); + const message = json.error?.message || json.message || json.detail || JSON.stringify(json); + return `API error (${status}): ${message}`; + } catch { + return `API error (${status}): ${response.statusText}`; + } +} diff --git a/dexto/packages/cli/src/cli/utils/config-validation.ts b/dexto/packages/cli/src/cli/utils/config-validation.ts new file mode 100644 index 00000000..4fb3dc13 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/config-validation.ts @@ -0,0 +1,534 @@ +import { z } from 'zod'; +import chalk from 'chalk'; +import * as p from '@clack/prompts'; +import { + AgentConfigSchema, + createAgentConfigSchema, + type AgentConfig, + type ValidatedAgentConfig, + type LLMValidationOptions, +} from '@dexto/core'; +import { interactiveApiKeySetup } from './api-key-setup.js'; +import { LLMErrorCode } from '@dexto/core'; +import type { LLMProvider } from '@dexto/core'; +import { logger } from '@dexto/core'; +import { + getGlobalPreferencesPath, + loadGlobalPreferences, + saveGlobalPreferences, +} from '@dexto/agent-management'; + +export interface ValidationResult { + success: boolean; + config?: ValidatedAgentConfig; + errors?: string[]; + skipped?: boolean; +} + +/** + * Validates agent config with optional interactive fixes for user experience. + * Uses schema parsing to detect API key issues and provides targeted setup. + * Returns validated config with all defaults applied. + * + * IMPORTANT: This function NEVER exits the process. It always returns a result + * that allows the caller to decide what to do next. + * + * @param config - The agent configuration to validate + * @param interactive - Whether to allow interactive prompts to fix issues + * @param validationOptions - Validation strictness options + * @param validationOptions.strict - When true (default), enforces API key requirements. + * When false, allows missing credentials for interactive config. + */ +export async function validateAgentConfig( + config: AgentConfig, + interactive: boolean = false, + validationOptions?: LLMValidationOptions +): Promise<ValidationResult> { + // Use appropriate schema based on validation options + // Default to strict validation unless explicitly relaxed + const schema = + validationOptions?.strict === false + ? createAgentConfigSchema({ strict: false }) + : AgentConfigSchema; + + // Parse with schema to detect issues + const parseResult = schema.safeParse(config); + + if (parseResult.success) { + return { success: true, config: parseResult.data }; + } + + // Validation failed - handle based on mode + logger.debug(`Agent config validation error: ${JSON.stringify(parseResult.error)}`); + const errors = formatZodErrors(parseResult.error); + + if (!interactive) { + // Non-interactive mode: show errors and next steps, but don't exit + showValidationErrors(errors); + showNextSteps(); + return { success: false, errors }; + } + + // Interactive mode: try to help the user fix the issue + // Check for API key errors first + const apiKeyError = findApiKeyError(parseResult.error, config); + if (apiKeyError) { + return await handleApiKeyError(apiKeyError.provider, config, errors, validationOptions); + } + + // Check for baseURL errors next + const baseURLError = findBaseURLError(parseResult.error, config); + if (baseURLError) { + return await handleBaseURLError(baseURLError.provider, config, errors, validationOptions); + } + + // Other validation errors - show options + return await handleOtherErrors(errors); +} + +/** + * Handle API key validation errors interactively + */ +async function handleApiKeyError( + provider: LLMProvider, + config: AgentConfig, + errors: string[], + validationOptions?: LLMValidationOptions +): Promise<ValidationResult> { + console.log(chalk.rgb(255, 165, 0)(`\n🔑 API key issue detected for ${provider} provider\n`)); + + const action = await p.select({ + message: 'How would you like to proceed?', + options: [ + { + value: 'setup' as const, + label: 'Set up API key now', + hint: 'Configure the API key interactively', + }, + { + value: 'skip' as const, + label: 'Continue anyway', + hint: 'Try to start without fixing (may fail)', + }, + { + value: 'edit' as const, + label: 'Edit configuration manually', + hint: 'Show file path and instructions', + }, + ], + }); + + if (p.isCancel(action)) { + showNextSteps(); + return { success: false, errors, skipped: true }; + } + + if (action === 'setup') { + const result = await interactiveApiKeySetup(provider, { exitOnCancel: false }); + if (result.success && !result.skipped) { + // Retry validation after API key setup + return validateAgentConfig(config, true, validationOptions); + } + // Setup was skipped or cancelled - let them continue anyway + return { success: false, errors, skipped: true }; + } + + if (action === 'edit') { + showManualEditInstructions(); + return { success: false, errors, skipped: true }; + } + + // 'skip' - continue anyway + p.log.warn('Continuing with validation errors - some features may not work correctly'); + return { success: false, errors, skipped: true }; +} + +/** + * Handle baseURL validation errors interactively + */ +async function handleBaseURLError( + provider: LLMProvider, + config: AgentConfig, + errors: string[], + validationOptions?: LLMValidationOptions +): Promise<ValidationResult> { + console.log(chalk.rgb(255, 165, 0)(`\n🌐 Base URL required for ${provider} provider\n`)); + + const providerExamples: Record<string, string> = { + 'openai-compatible': 'http://localhost:11434/v1 (Ollama)', + litellm: 'http://localhost:4000 (LiteLLM proxy)', + }; + + const example = providerExamples[provider] || 'http://localhost:8080/v1'; + + p.note( + [ + `The ${provider} provider requires a base URL to connect to your`, + `local or custom LLM endpoint.`, + ``, + `${chalk.gray('Example:')} ${example}`, + ].join('\n'), + 'Base URL Required' + ); + + const action = await p.select({ + message: 'How would you like to proceed?', + options: [ + { + value: 'setup' as const, + label: 'Enter base URL now', + hint: 'Configure the base URL interactively', + }, + { + value: 'skip' as const, + label: 'Continue anyway', + hint: 'Try to start without fixing (may fail)', + }, + { + value: 'edit' as const, + label: 'Edit configuration manually', + hint: 'Show file path and instructions', + }, + ], + }); + + if (p.isCancel(action)) { + showNextSteps(); + return { success: false, errors, skipped: true }; + } + + if (action === 'setup') { + const result = await interactiveBaseURLSetup(provider, config.llm?.baseURL); + if (result.success && !result.skipped && result.baseURL && config.llm) { + // Update config with the new baseURL for retry validation + const updatedConfig = { + ...config, + llm: { ...config.llm, baseURL: result.baseURL }, + }; + // Retry validation after baseURL setup + return validateAgentConfig(updatedConfig, true, validationOptions); + } + // Setup was skipped or cancelled + return { success: false, errors, skipped: true }; + } + + if (action === 'edit') { + showManualEditInstructions(); + return { success: false, errors, skipped: true }; + } + + // 'skip' - continue anyway + p.log.warn('Continuing with validation errors - some features may not work correctly'); + return { success: false, errors, skipped: true }; +} + +/** + * Interactive baseURL setup + */ +async function interactiveBaseURLSetup( + provider: LLMProvider, + existingBaseURL?: string +): Promise<{ success: boolean; baseURL?: string; skipped?: boolean }> { + const providerDefaults: Record<string, string> = { + 'openai-compatible': 'http://localhost:11434/v1', + litellm: 'http://localhost:4000', + }; + + // Use existing baseURL if available, otherwise fall back to provider defaults + const defaultURL = existingBaseURL || providerDefaults[provider] || ''; + + const baseURL = await p.text({ + message: `Enter base URL for ${provider}`, + placeholder: defaultURL, + initialValue: defaultURL, + validate: (value) => { + if (!value.trim()) { + return 'Base URL is required'; + } + try { + new URL(value.trim()); + return undefined; + } catch { + return 'Please enter a valid URL (e.g., http://localhost:11434/v1)'; + } + }, + }); + + if (p.isCancel(baseURL)) { + p.log.warn('Skipping base URL setup. You can configure it later with: dexto setup'); + return { success: false, skipped: true }; + } + + const trimmedURL = baseURL.trim(); + + // Save to preferences + const spinner = p.spinner(); + spinner.start('Saving base URL to preferences...'); + + try { + const preferences = await loadGlobalPreferences(); + + // Update the LLM section with baseURL (complete replacement as per schema design) + const updatedPreferences = { + ...preferences, + llm: { + ...preferences.llm, + baseURL: trimmedURL, + }, + }; + + await saveGlobalPreferences(updatedPreferences); + spinner.stop(chalk.green('✓ Base URL saved to preferences')); + + return { success: true, baseURL: trimmedURL }; + } catch (error) { + spinner.stop(chalk.red('✗ Failed to save base URL')); + logger.error( + `Failed to save baseURL: ${error instanceof Error ? error.message : String(error)}` + ); + + // Show manual instructions + p.note( + [ + `Add this to your preferences file:`, + ``, + ` ${chalk.cyan('baseURL:')} ${trimmedURL}`, + ``, + `File: ${getGlobalPreferencesPath()}`, + ].join('\n'), + chalk.rgb(255, 165, 0)('Manual Setup Required') + ); + + // Still return success with the URL for in-memory use + return { success: true, baseURL: trimmedURL, skipped: true }; + } +} + +/** + * Handle non-API-key validation errors interactively + */ +async function handleOtherErrors(errors: string[]): Promise<ValidationResult> { + console.log(chalk.rgb(255, 165, 0)('\n⚠️ Configuration issues detected:\n')); + for (const error of errors) { + console.log(chalk.red(` • ${error}`)); + } + console.log(''); + + const action = await p.select({ + message: 'How would you like to proceed?', + options: [ + { + value: 'skip' as const, + label: 'Continue anyway', + hint: 'Try to start despite errors (may fail)', + }, + { + value: 'edit' as const, + label: 'Edit configuration manually', + hint: 'Show file path and instructions', + }, + { + value: 'setup' as const, + label: 'Run setup again', + hint: 'Reconfigure from scratch', + }, + ], + }); + + if (p.isCancel(action)) { + showNextSteps(); + return { success: false, errors, skipped: true }; + } + + if (action === 'edit') { + showManualEditInstructions(); + return { success: false, errors, skipped: true }; + } + + if (action === 'setup') { + p.log.info('Run: dexto setup --force'); + return { success: false, errors, skipped: true }; + } + + // 'skip' - continue anyway + p.log.warn('Continuing with validation errors - some features may not work correctly'); + return { success: false, errors, skipped: true }; +} + +/** + * Show validation errors in a user-friendly way + */ +function showValidationErrors(errors: string[]): void { + console.log(chalk.rgb(255, 165, 0)('\n⚠️ Configuration issues detected:\n')); + for (const error of errors) { + console.log(chalk.red(` • ${error}`)); + } + console.log(''); +} + +/** + * Show next steps after validation failure + */ +function showNextSteps(): void { + const prefsPath = getGlobalPreferencesPath(); + console.log(chalk.bold('\nNext steps:')); + console.log(` • Run ${chalk.cyan('dexto setup')} to reconfigure interactively`); + console.log(` • Edit ${chalk.cyan(prefsPath)} directly`); + console.log(` • Check your environment variables\n`); +} + +/** + * Show manual edit instructions + */ +function showManualEditInstructions(): void { + const prefsPath = getGlobalPreferencesPath(); + + p.note( + [ + `Your configuration files:`, + ``, + ` ${chalk.cyan('Global preferences:')} ${prefsPath}`, + ` ${chalk.cyan('Agent configs:')} ~/.dexto/agents/*/`, + ``, + `Edit the appropriate file and run dexto again.`, + ``, + chalk.gray('Example commands:'), + chalk.gray(` code ${prefsPath} # Open in VS Code`), + chalk.gray(` nano ${prefsPath} # Edit in terminal`), + ].join('\n'), + 'Manual Configuration' + ); +} + +/** + * Extract API key error details from Zod validation error + */ +function findApiKeyError( + error: z.ZodError, + configData: AgentConfig +): { provider: LLMProvider } | null { + for (const issue of error.issues) { + // Check for our custom LLM_MISSING_API_KEY error code in params + if (issue.code === 'custom' && hasErrorCode(issue.params, LLMErrorCode.API_KEY_MISSING)) { + // Extract provider from error params (added by our schema) + const provider = getProviderFromParams(issue.params); + if (provider) { + return { provider }; + } + } + + // Fallback: check for apiKey path errors and extract provider from config + if (issue.path.includes('apiKey') && issue.message.includes('Missing API key')) { + const provider = configData.llm?.provider; + if (provider) { + return { provider }; + } + } + } + return null; +} + +/** + * Extract baseURL error details from Zod validation error + */ +function findBaseURLError( + error: z.ZodError, + configData: AgentConfig +): { provider: LLMProvider } | null { + for (const issue of error.issues) { + // Check for our custom BASE_URL_MISSING error code in params + if (issue.code === 'custom' && hasErrorCode(issue.params, LLMErrorCode.BASE_URL_MISSING)) { + const provider = getProviderFromParams(issue.params) || configData.llm?.provider; + if (provider) { + return { provider }; + } + } + + // Fallback: check for baseURL path errors + if (issue.path.includes('baseURL') && issue.message.includes('requires')) { + const provider = configData.llm?.provider; + if (provider) { + return { provider }; + } + } + } + return null; +} + +/** + * Type guard to check if params contains the expected error code + */ +function hasErrorCode(params: unknown, expectedCode: LLMErrorCode): boolean { + return ( + typeof params === 'object' && + params !== null && + 'code' in params && + params.code === expectedCode + ); +} + +/** + * Extract provider from Zod issue params + */ +function getProviderFromParams(params: unknown): LLMProvider | null { + if ( + typeof params === 'object' && + params !== null && + 'provider' in params && + typeof params.provider === 'string' + ) { + return params.provider as LLMProvider; + } + return null; +} + +/** + * Format Zod validation errors in a user-friendly way + */ +function formatZodErrors(error: z.ZodError): string[] { + return error.issues.map((issue) => { + const path = issue.path.length > 0 ? issue.path.join('.') : 'config'; + return `${path}: ${issue.message}`; + }); +} + +/** + * Legacy function for backwards compatibility + * @deprecated Use validateAgentConfig with result handling instead + */ +export async function validateAgentConfigOrExit( + config: AgentConfig, + interactive: boolean = false +): Promise<ValidatedAgentConfig> { + const result = await validateAgentConfig(config, interactive); + + if (result.success && result.config) { + return result.config; + } + + // If validation failed but was skipped, return config as-is with defaults applied + // This allows the app to launch in a limited capacity (e.g., web UI for configuration) + // Runtime errors will occur when actually trying to use the LLM + if (result.skipped) { + logger.warn('Starting with validation warnings - some features may not work'); + + // Apply defaults manually without strict validation + // This allows launching web UI for interactive configuration + // Use unknown cast to bypass branded type checking since we're intentionally + // returning a partially valid config that the user acknowledged + const configWithDefaults = { + ...config, + llm: { + ...config.llm, + maxIterations: config.llm?.maxIterations ?? 50, + }, + } as unknown as ValidatedAgentConfig; + + return configWithDefaults; + } + + // Last resort: exit with helpful message + console.log(chalk.rgb(255, 165, 0)('\nUnable to start with current configuration.')); + showNextSteps(); + process.exit(1); +} diff --git a/dexto/packages/cli/src/cli/utils/dexto-auth-check.ts b/dexto/packages/cli/src/cli/utils/dexto-auth-check.ts new file mode 100644 index 00000000..dc7466fc --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/dexto-auth-check.ts @@ -0,0 +1,125 @@ +// packages/cli/src/cli/utils/dexto-auth-check.ts +/** + * Dexto Authentication Check + * + * Validates that users configured to use Dexto credits are properly authenticated. + * This prevents the confusing state where user is logged out but still has + * Dexto credits configured (which would fail at runtime). + * + * ## When This Check Runs + * + * - On CLI startup, after loading preferences + * - Before attempting to use the LLM + * + * ## Why This Exists + * + * When a user runs `dexto logout` while configured with `provider: dexto`, + * their preferences.yml still points to Dexto. Without this check, the CLI + * would attempt to use Dexto credits and fail with "Invalid API key" errors. + * + * Instead, we catch this state early and offer clear options: + * - Log back in to continue using Dexto credits + * - Run setup to configure a different provider (BYOK) + * + * @module dexto-auth-check + */ + +import chalk from 'chalk'; +import * as p from '@clack/prompts'; +import { isAuthenticated } from '../auth/index.js'; +import { getEffectiveLLMConfig } from '../../config/effective-llm.js'; + +export interface DextoAuthCheckResult { + shouldContinue: boolean; + action?: 'login' | 'setup' | 'cancel'; +} + +/** + * Check if user is configured to use Dexto credits but is not authenticated. + * This can happen if user logged out after setting up with Dexto credits. + * + * Uses getEffectiveLLMConfig() to determine the actual LLM that will be used, + * considering all config layers (local, preferences, bundled). + * + * @param interactive Whether to show interactive prompts + * @param agentId Agent to check config for (default: 'coding-agent') + * @returns Whether to continue startup and what action was taken + * + * @example + * ```typescript + * const result = await checkDextoAuthState(true, 'coding-agent'); + * if (!result.shouldContinue) { + * if (result.action === 'login') { + * await handleLoginCommand(); + * } else if (result.action === 'setup') { + * await handleSetupCommand(); + * } + * } + * ``` + */ +export async function checkDextoAuthState( + interactive: boolean = true, + agentId: string = 'coding-agent' +): Promise<DextoAuthCheckResult> { + // Get the effective LLM config considering all layers + const effectiveLLM = await getEffectiveLLMConfig({ agentId }); + + // Not using dexto provider - nothing to check + if (!effectiveLLM || effectiveLLM.provider !== 'dexto') { + return { shouldContinue: true }; + } + + // Using dexto provider - check if authenticated + const authenticated = await isAuthenticated(); + if (authenticated) { + return { shouldContinue: true }; + } + + // User is configured for Dexto credits but not logged in + if (!interactive) { + // Non-interactive mode - just show error and exit + console.log(chalk.red('\n❌ You are configured to use Dexto credits but not logged in.\n')); + console.log(chalk.dim('Your preferences have provider: dexto, but no active session.\n')); + console.log(chalk.bold('To fix this:')); + console.log( + chalk.cyan(' • dexto login') + chalk.dim(' - Log back in to use Dexto credits') + ); + console.log( + chalk.cyan(' • dexto setup') + chalk.dim(' - Configure a different provider (BYOK)') + ); + console.log(); + return { shouldContinue: false, action: 'cancel' }; + } + + // Interactive mode - prompt user + console.log(chalk.yellow('\n⚠️ Dexto Authentication Required\n')); + console.log(chalk.dim('You are configured to use Dexto credits (provider: dexto)')); + console.log(chalk.dim('but you are not currently logged in.\n')); + + const action = await p.select({ + message: 'How would you like to proceed?', + options: [ + { + value: 'login' as const, + label: 'Log in to Dexto', + hint: 'Authenticate to use Dexto credits', + }, + { + value: 'setup' as const, + label: 'Configure a different provider', + hint: 'Set up your own API key (BYOK)', + }, + { + value: 'cancel' as const, + label: 'Exit', + hint: 'Cancel and exit', + }, + ], + }); + + if (p.isCancel(action) || action === 'cancel') { + return { shouldContinue: false, action: 'cancel' }; + } + + return { shouldContinue: false, action }; +} diff --git a/dexto/packages/cli/src/cli/utils/dexto-setup.ts b/dexto/packages/cli/src/cli/utils/dexto-setup.ts new file mode 100644 index 00000000..81555f72 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/dexto-setup.ts @@ -0,0 +1,19 @@ +// packages/cli/src/cli/utils/dexto-setup.ts + +import { getDextoApiKey, isAuthenticated } from '../auth/index.js'; + +/** + * Check if user can use Dexto provider. + * Requires BOTH: + * 1. User is authenticated (valid auth token from dexto login) + * 2. Has DEXTO_API_KEY (from auth config or environment) + */ +export async function canUseDextoProvider(): Promise<boolean> { + const authenticated = await isAuthenticated(); + if (!authenticated) return false; + + const apiKey = await getDextoApiKey(); + if (!apiKey) return false; + + return true; +} diff --git a/dexto/packages/cli/src/cli/utils/execute.ts b/dexto/packages/cli/src/cli/utils/execute.ts new file mode 100644 index 00000000..75f4099a --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/execute.ts @@ -0,0 +1,62 @@ +import { spawn } from 'child_process'; +import { logger } from '@dexto/core'; + +// Default timeout for spawned processes (in milliseconds) +const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes + +/** + * Reusable helper to run any shell command with a timeout. Uses spawn to run the command. + * @param command - The command to execute + * @param args - The arguments to pass to the command + * @param options - The options for the command + * @returns A promise that resolves when the command has finished executing + */ +export function executeWithTimeout( + command: string, + args: string[], + options: { cwd: string; timeoutMs?: number } +): Promise<void> { + return new Promise<void>((resolve, reject) => { + const { cwd, timeoutMs: timeout = DEFAULT_TIMEOUT_MS } = options; + const child = spawn(command, args, { cwd }); + let stdout = ''; + let stderr = ''; + + // Kill process if it takes too long + const timer = setTimeout(() => { + logger.error(`Process timed out after ${timeout}ms, killing process`); + child.kill(); + reject(new Error(`Process timed out after ${timeout}ms`)); + }, timeout); + + child.stdout.on('data', (data: Buffer) => { + const text = data.toString(); + stdout += text; + logger.debug(text); + }); + + child.stderr.on('data', (data: Buffer) => { + const text = data.toString(); + stderr += text; + // Not logging to avoid spamming the console with installation warnings + // logger.error(text); + }); + + child.on('error', (error: Error) => { + clearTimeout(timer); + logger.error(`Error spawning process: ${error.message}`); + reject(error); + }); + + child.on('close', (code: number) => { + clearTimeout(timer); + if (code !== 0) { + logger.error(`Process exited with code ${code}\n${stderr}`); + reject(new Error(`Process exited with code ${code}`)); + } else { + logger.debug(`${command} ${args.join(' ')} stdout: ${stdout}`); + resolve(); + } + }); + }); +} diff --git a/dexto/packages/cli/src/cli/utils/local-model-setup.ts b/dexto/packages/cli/src/cli/utils/local-model-setup.ts new file mode 100644 index 00000000..79964f37 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/local-model-setup.ts @@ -0,0 +1,863 @@ +// packages/cli/src/cli/utils/local-model-setup.ts + +/** + * Interactive setup flow for local AI models. + * + * This module provides the setup experience when a user selects + * 'local' or 'ollama' as their provider during `dexto setup`. + */ + +import chalk from 'chalk'; +import * as p from '@clack/prompts'; +import * as fs from 'fs'; +import * as path from 'path'; +import { + getRecommendedLocalModels, + getAllLocalModels, + getLocalModelById, + detectGPU, + formatGPUInfo, + downloadModel, + checkOllamaStatus, + listOllamaModels, + isOllamaModelAvailable, + pullOllamaModel, + isNodeLlamaCppInstalled, + type ModelDownloadProgress, +} from '@dexto/core'; +import { spawn } from 'child_process'; +import { + getAllInstalledModels, + setActiveModel, + addInstalledModel, + getModelsDirectory, + modelFileExists, + getModelFileSize, + formatSize, + saveCustomModel, + getDextoGlobalPath, + type InstalledModel, +} from '@dexto/agent-management'; + +/** + * Result of local model setup + */ +export interface LocalModelSetupResult { + /** Whether setup completed successfully */ + success: boolean; + + /** Selected model ID */ + modelId?: string; + + /** Whether user cancelled */ + cancelled?: boolean; + + /** Whether user wants to go back to provider selection */ + back?: boolean; + + /** Whether user skipped model selection */ + skipped?: boolean; +} + +/** + * Type guard: Check if local model setup result has a selected model. + * Use this before proceeding with model configuration. + * + * Returns false for: cancelled, back, skipped, or missing modelId + * Returns true only when: success=true AND modelId is present + */ +export function hasSelectedModel( + result: LocalModelSetupResult +): result is LocalModelSetupResult & { modelId: string } { + return ( + result.success && !result.cancelled && !result.back && !result.skipped && !!result.modelId + ); +} + +/** + * Install node-llama-cpp to the global deps directory (~/.dexto/deps). + * This compiles native bindings for the user's system. + * Installing globally ensures it's available for CLI, WebUI, and all projects. + */ +async function installNodeLlamaCpp(): Promise<boolean> { + const depsDir = getDextoGlobalPath('deps'); + + // Ensure deps directory exists + if (!fs.existsSync(depsDir)) { + fs.mkdirSync(depsDir, { recursive: true }); + } + + // Initialize package.json if it doesn't exist (required for npm install) + const packageJsonPath = path.join(depsDir, 'package.json'); + if (!fs.existsSync(packageJsonPath)) { + fs.writeFileSync( + packageJsonPath, + JSON.stringify( + { + name: 'dexto-deps', + version: '1.0.0', + private: true, + description: 'Native dependencies for Dexto', + }, + null, + 2 + ) + ); + } + + return new Promise((resolve) => { + // Install to global deps directory using npm + const child = spawn('npm', ['install', 'node-llama-cpp'], { + stdio: ['ignore', 'pipe', 'pipe'], + cwd: depsDir, + shell: true, + }); + + let stderr = ''; + child.stderr?.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('close', (code) => { + if (code === 0) { + resolve(true); + } else { + console.error(chalk.gray(stderr)); + resolve(false); + } + }); + + child.on('error', () => { + resolve(false); + }); + }); +} + +/** + * Check and install node-llama-cpp if needed. + * Returns true if ready to use, false if installation failed/cancelled. + */ +async function ensureNodeLlamaCpp(): Promise<boolean> { + const isInstalled = await isNodeLlamaCppInstalled(); + if (isInstalled) { + return true; + } + + p.note( + 'Local model execution requires node-llama-cpp.\n' + + 'This will compile native bindings for your system.\n\n' + + chalk.gray('Installation may take 1-2 minutes.'), + 'Dependency Required' + ); + + const shouldInstall = await p.confirm({ + message: 'Install node-llama-cpp now?', + initialValue: true, + }); + + if (p.isCancel(shouldInstall) || !shouldInstall) { + return false; + } + + const spinner = p.spinner(); + spinner.start('Installing node-llama-cpp (compiling native bindings)...'); + + const success = await installNodeLlamaCpp(); + + if (success) { + spinner.stop(chalk.green('✓ node-llama-cpp installed successfully')); + return true; + } else { + spinner.stop(chalk.red('✗ Installation failed')); + p.log.error( + 'Failed to install node-llama-cpp. You can try manually:\n' + + chalk.gray(' npm install node-llama-cpp') + ); + return false; + } +} + +/** + * Interactive local model setup for 'local' provider. + * + * Shows available models, offers to download, and sets up the active model. + * Uses node-llama-cpp for native GGUF model execution. + */ +export async function setupLocalModels(): Promise<LocalModelSetupResult> { + console.log(chalk.cyan('\n🤖 Local Model Setup\n')); + + // Ensure node-llama-cpp is installed first + const dependencyReady = await ensureNodeLlamaCpp(); + if (!dependencyReady) { + p.log.warn('Setup cancelled - node-llama-cpp is required for local models.'); + return { success: false, cancelled: true }; + } + + // Get installed models first - if already installed, we can skip other checks + const installed = await getAllInstalledModels(); + const installedIds = new Set(installed.map((m) => m.id)); + + // Check if any models are already installed - offer quick path + if (installed.length > 0) { + const useExisting = await p.confirm({ + message: `You have ${installed.length} model(s) installed. Use an existing model?`, + initialValue: true, + }); + + if (p.isCancel(useExisting)) { + return { success: false, cancelled: true }; + } + + if (useExisting) { + // Let user select from installed models - no additional setup needed + const selected = await selectInstalledModel(installed); + if (selected.cancelled) { + return { success: false, cancelled: true }; + } + if (selected.customGGUF) { + // User wants to use a custom GGUF file + return setupCustomGGUF(); + } + if (selected.modelId) { + await setActiveModel(selected.modelId); + p.log.success(`Using ${selected.modelId} as active model`); + return { success: true, modelId: selected.modelId }; + } + } + } + + // Only detect GPU if we're going to show model recommendations + const gpuInfo = await detectGPU(); + console.log(chalk.gray(`GPU detected: ${formatGPUInfo(gpuInfo)}\n`)); + + // Get recommended models + const recommendedModels = getRecommendedLocalModels(); + + // Build options with install status + const modelOptions = recommendedModels.map((model) => { + const isInstalled = installedIds.has(model.id); + const statusIcon = isInstalled ? chalk.green('✓') : chalk.gray('○'); + const vramHint = model.minVRAM ? `${model.minVRAM}GB+ VRAM` : 'CPU OK'; + + return { + value: model.id, + label: `${statusIcon} ${model.name}`, + hint: `${formatSize(model.sizeBytes)} | ${vramHint}${isInstalled ? ' (installed)' : ''}`, + }; + }); + + // Add option to see all models + modelOptions.push({ + value: '_all_models', + label: `${chalk.blue('...')} Show all available models`, + hint: `${getAllLocalModels().length} models available`, + }); + + // Add option to use custom GGUF file + modelOptions.push({ + value: '_custom_gguf', + label: `${chalk.blue('...')} Use custom GGUF file`, + hint: 'For GGUF files not in registry', + }); + + // Add option to skip + modelOptions.push({ + value: '_skip', + label: `${chalk.rgb(255, 165, 0)('→')} Skip for now`, + hint: 'Configure later with: dexto setup', + }); + + // Add back option + modelOptions.push({ + value: '_back', + label: chalk.gray('← Back'), + hint: 'Choose a different provider', + }); + + p.note( + 'Local models run completely on your machine - free, private, and offline.\n' + + 'Select a model to download (or use an existing one).', + 'Local AI' + ); + + const selected = await p.select({ + message: 'Choose a model to use', + options: modelOptions, + }); + + if (p.isCancel(selected)) { + return { success: false, cancelled: true }; + } + + if (selected === '_skip') { + p.log.info(chalk.gray('Skipped model selection. Use `dexto setup` to configure later.')); + return { success: true, skipped: true }; + } + + if (selected === '_back') { + return { success: false, back: true }; + } + + if (selected === '_all_models') { + // Show all models + return await showAllModelsSelection(installedIds); + } + + if (selected === '_custom_gguf') { + // Use custom GGUF file + return setupCustomGGUF(); + } + + const modelId = selected as string; + + // Check if already installed + if (installedIds.has(modelId)) { + await setActiveModel(modelId); + p.log.success(`Using ${modelId} as active model`); + return { success: true, modelId }; + } + + // Download the model + const downloadResult = await downloadModelInteractive(modelId); + if (!downloadResult.success) { + if (downloadResult.cancelled) { + return { success: false, cancelled: true }; + } + return { success: false }; + } + + // Set as active + await setActiveModel(modelId); + return { success: true, modelId }; +} + +/** + * Check if Ollama model is available, offer to pull if not. + * Returns true if model is ready to use, false if user declined pull or pull failed. + */ +async function ensureOllamaModelAvailable(modelName: string): Promise<boolean> { + // Check if model is already available + const isAvailable = await isOllamaModelAvailable(modelName); + if (isAvailable) { + return true; + } + + // Model not found - offer to pull it + console.log(chalk.rgb(255, 165, 0)(`\n⚠️ Model '${modelName}' is not available locally.\n`)); + + const shouldPull = await p.confirm({ + message: `Pull '${modelName}' from Ollama now?`, + initialValue: true, + }); + + if (p.isCancel(shouldPull) || !shouldPull) { + p.log.warn('Skipping model pull. You can pull it later with: ollama pull ' + modelName); + return false; + } + + // Pull the model with progress display + const spinner = p.spinner(); + spinner.start(`Pulling ${modelName} from Ollama...`); + + try { + await pullOllamaModel(modelName, undefined, (progress) => { + // Update spinner with progress (show percentage if available) + if (progress.completed && progress.total) { + const percent = Math.round((progress.completed / progress.total) * 100); + const sizeDownloaded = formatSize(progress.completed); + const sizeTotal = formatSize(progress.total); + spinner.message( + `Pulling ${modelName}... ${percent}% (${sizeDownloaded}/${sizeTotal}) - ${progress.status}` + ); + } else { + spinner.message(`Pulling ${modelName}... ${progress.status}`); + } + }); + + spinner.stop(chalk.green(`✓ Successfully pulled ${modelName}`)); + return true; + } catch (error) { + spinner.stop(chalk.red('✗ Failed to pull model')); + console.error( + chalk.red(`Error: ${error instanceof Error ? error.message : String(error)}`) + ); + p.log.warn('You can try pulling manually: ollama pull ' + modelName); + return false; + } +} + +/** + * Interactive Ollama model setup for 'ollama' provider. + */ +export async function setupOllamaModels(): Promise<LocalModelSetupResult> { + console.log(chalk.cyan('\n🦙 Ollama Setup\n')); + + // Check if Ollama is running + const status = await checkOllamaStatus(); + + if (!status.running) { + p.note( + chalk.rgb(255, 165, 0)('Ollama server is not running.\n\n') + + 'To use Ollama:\n' + + ' 1. Install Ollama: https://ollama.com/download\n' + + ' 2. Start the server: ollama serve\n' + + ' 3. Pull a model: ollama pull llama3.2', + 'Ollama Required' + ); + + const proceed = await p.confirm({ + message: 'Continue setup anyway? (You can configure Ollama later)', + initialValue: true, + }); + + if (p.isCancel(proceed)) { + return { success: false, cancelled: true }; + } + if (!proceed) { + return { success: false }; + } + + // Let them specify a model name even without Ollama running + const modelName = await p.text({ + message: 'Enter the Ollama model name to use', + placeholder: 'llama3.2', + initialValue: 'llama3.2', + }); + + if (p.isCancel(modelName)) { + return { success: false, cancelled: true }; + } + + return { success: true, modelId: modelName.trim() }; + } + + // Ollama is running - show available models + console.log(chalk.green(`✓ Ollama ${status.version || ''} running at ${status.url}\n`)); + + const ollamaModels = await listOllamaModels(); + + if (ollamaModels.length === 0) { + p.note( + 'No models found in Ollama.\n\n' + + 'To pull a model:\n' + + ' ollama pull llama3.2\n\n' + + 'Popular models:\n' + + ' • llama3.2 (3B/8B general)\n' + + ' • qwen2.5-coder (coding)\n' + + ' • mistral (7B general)', + 'No Models' + ); + + const modelName = await p.text({ + message: 'Enter the model name to pull', + placeholder: 'llama3.2', + initialValue: 'llama3.2', + }); + + if (p.isCancel(modelName)) { + return { success: false, cancelled: true }; + } + + const trimmedName = modelName.trim(); + const isReady = await ensureOllamaModelAvailable(trimmedName); + + if (!isReady) { + // User declined pull or pull failed + return { success: false }; + } + + return { success: true, modelId: trimmedName }; + } + + // Show available Ollama models + const modelOptions = ollamaModels.map((model) => ({ + value: model.name, + label: model.name, + hint: formatSize(model.size), + })); + + // Add option to enter custom name + modelOptions.push({ + value: '_custom', + label: `${chalk.blue('...')} Enter custom model name`, + hint: 'For models not yet pulled', + }); + + // Add back option + modelOptions.push({ + value: '_back', + label: chalk.gray('← Back'), + hint: 'Choose a different provider', + }); + + const selected = await p.select({ + message: 'Select an Ollama model', + options: modelOptions, + }); + + if (p.isCancel(selected)) { + return { success: false, cancelled: true }; + } + + if (selected === '_back') { + return { success: false, back: true }; + } + + if (selected === '_custom') { + const modelName = await p.text({ + message: 'Enter the Ollama model name', + placeholder: 'llama3.2:70b', + }); + + if (p.isCancel(modelName)) { + return { success: false, cancelled: true }; + } + + const trimmedName = modelName.trim(); + const isReady = await ensureOllamaModelAvailable(trimmedName); + + if (!isReady) { + // User declined pull or pull failed + return { success: false }; + } + + return { success: true, modelId: trimmedName }; + } + + return { success: true, modelId: selected as string }; +} + +/** + * Select from installed models + */ +async function selectInstalledModel( + installed: InstalledModel[] +): Promise<{ modelId?: string; cancelled?: boolean; customGGUF?: boolean }> { + const options = installed.map((model) => ({ + value: model.id, + label: model.id, + hint: formatSize(model.sizeBytes), + })); + + options.push({ + value: '_download_new', + label: `${chalk.blue('+')} Download a new model`, + hint: 'Browse available models', + }); + + options.push({ + value: '_custom_gguf', + label: `${chalk.blue('...')} Use custom GGUF file`, + hint: 'For GGUF files not in registry', + }); + + const selected = await p.select({ + message: 'Select a model', + options, + }); + + if (p.isCancel(selected)) { + return { cancelled: true }; + } + + if (selected === '_download_new') { + return {}; // Continue to download flow + } + + if (selected === '_custom_gguf') { + return { customGGUF: true }; + } + + return { modelId: selected as string }; +} + +/** + * Show all available models for selection + */ +async function showAllModelsSelection(installedIds: Set<string>): Promise<LocalModelSetupResult> { + const allModels = getAllLocalModels(); + + const modelOptions = allModels.map((model) => { + const isInstalled = installedIds.has(model.id); + const statusIcon = isInstalled ? chalk.green('✓') : chalk.gray('○'); + const category = model.categories?.[0] || 'general'; + const vramHint = model.minVRAM ? `${model.minVRAM}GB+` : 'CPU'; + + return { + value: model.id, + label: `${statusIcon} ${model.name}`, + hint: `${category} | ${formatSize(model.sizeBytes)} | ${vramHint}${isInstalled ? ' (installed)' : ''}`, + }; + }); + + modelOptions.push({ + value: '_back', + label: `${chalk.rgb(255, 165, 0)('←')} Back`, + hint: 'Return to recommended models', + }); + + const selected = await p.select({ + message: 'Select a model', + options: modelOptions, + }); + + if (p.isCancel(selected)) { + return { success: false, cancelled: true }; + } + + if (selected === '_back') { + // Recurse back to main setup + return setupLocalModels(); + } + + const modelId = selected as string; + + // Check if already installed + if (installedIds.has(modelId)) { + await setActiveModel(modelId); + p.log.success(`Using ${modelId} as active model`); + return { success: true, modelId }; + } + + // Download the model + const downloadResult = await downloadModelInteractive(modelId); + if (!downloadResult.success) { + if (downloadResult.cancelled) { + return { success: false, cancelled: true }; + } + return { success: false }; + } + + await setActiveModel(modelId); + return { success: true, modelId }; +} + +/** + * Download a model with interactive progress + */ +async function downloadModelInteractive( + modelId: string +): Promise<{ success: boolean; cancelled?: boolean }> { + const modelInfo = getLocalModelById(modelId); + if (!modelInfo) { + p.log.error(`Model '${modelId}' not found in registry`); + return { success: false }; + } + + // Check if model file already exists on disk (but not registered) + // First check the expected subdirectory, then fallback to root models dir + const fileExistsInSubdir = await modelFileExists(modelId, modelInfo.filename); + const rootFilePath = `${getModelsDirectory()}/${modelInfo.filename}`; + let actualFilePath: string | null = null; + let fileSize: number | null = null; + + if (fileExistsInSubdir) { + actualFilePath = `${getModelsDirectory()}/${modelId}/${modelInfo.filename}`; + fileSize = await getModelFileSize(modelId, modelInfo.filename); + } else { + // Check root models directory (legacy or manual placement) + try { + const fs = await import('fs/promises'); + const stats = await fs.stat(rootFilePath); + if (stats.isFile()) { + actualFilePath = rootFilePath; + fileSize = stats.size; + } + } catch { + // File doesn't exist in root either + } + } + + if (actualFilePath) { + p.log.info(chalk.green(`✓ Model file already exists on disk`)); + + // Register the existing model + const installedModel: InstalledModel = { + id: modelId, + filePath: actualFilePath, + sizeBytes: fileSize ?? modelInfo.sizeBytes, + downloadedAt: new Date().toISOString(), + source: 'huggingface', + filename: modelInfo.filename, + }; + + await addInstalledModel(installedModel); + p.log.success(`Model '${modelId}' registered successfully`); + return { success: true }; + } + + // Show model info and confirm + p.note( + `${modelInfo.name}\n` + + `${modelInfo.description}\n\n` + + `Size: ${formatSize(modelInfo.sizeBytes)}\n` + + `Context: ${modelInfo.contextLength.toLocaleString()} tokens\n` + + `Quantization: ${modelInfo.quantization}`, + 'Model Details' + ); + + const confirmed = await p.confirm({ + message: `Download ${modelInfo.name} (${formatSize(modelInfo.sizeBytes)})?`, + }); + + if (p.isCancel(confirmed) || !confirmed) { + return { success: false, cancelled: true }; + } + + // Start download with spinner + const spinner = p.spinner(); + spinner.start('Starting download...'); + + try { + const result = await downloadModel(modelId, { + targetDir: getModelsDirectory(), + events: { + onProgress: (progress: ModelDownloadProgress) => { + const pct = progress.percentage.toFixed(1); + const downloaded = formatSize(progress.bytesDownloaded); + const total = formatSize(progress.totalBytes); + const speedStr = progress.speed ? `${formatSize(progress.speed)}/s` : ''; + const etaStr = progress.eta ? `ETA: ${Math.round(progress.eta)}s` : ''; + + spinner.message(`${pct}% (${downloaded}/${total}) ${speedStr} ${etaStr}`); + }, + onComplete: () => { + spinner.stop(chalk.green(`✓ Downloaded ${modelInfo.name}`)); + }, + onError: (_modelId: string, error: Error) => { + spinner.stop(chalk.red(`✗ Download failed: ${error.message}`)); + }, + }, + }); + + // Register the installed model + const installedModel: InstalledModel = { + id: modelId, + filePath: result.filePath, + sizeBytes: result.sizeBytes, + downloadedAt: new Date().toISOString(), + source: 'huggingface', + filename: modelInfo.filename, + }; + + if (result.sha256) { + installedModel.sha256 = result.sha256; + } + + await addInstalledModel(installedModel); + + p.log.success(`Model '${modelId}' installed successfully`); + return { success: true }; + } catch (error) { + spinner.stop(chalk.red('Download failed')); + p.log.error( + `Failed to download: ${error instanceof Error ? error.message : String(error)}` + ); + return { success: false }; + } +} + +/** + * Setup a custom GGUF file. + * Prompts user for file path, validates it, and saves as a custom model. + * Mirrors the Ollama "Enter custom model name" pattern. + */ +async function setupCustomGGUF(): Promise<LocalModelSetupResult> { + // Prompt for file path + const filePath = await p.text({ + message: 'Enter path to GGUF file', + placeholder: '/path/to/model.gguf', + validate: (value) => { + if (!value.trim()) { + return 'File path is required'; + } + if (!value.endsWith('.gguf')) { + return 'File must have .gguf extension'; + } + if (!path.isAbsolute(value)) { + return 'Please enter an absolute path'; + } + return undefined; + }, + }); + + if (p.isCancel(filePath)) { + return { success: false, cancelled: true }; + } + + const trimmedPath = filePath.trim(); + + // Validate file exists + try { + const stats = fs.statSync(trimmedPath); + if (!stats.isFile()) { + p.log.error('Path is not a file'); + return { success: false }; + } + + const sizeBytes = stats.size; + const filename = path.basename(trimmedPath, '.gguf'); + + console.log( + chalk.green(`\n✓ Found: ${path.basename(trimmedPath)} (${formatSize(sizeBytes)})\n`) + ); + + // Prompt for display name (optional) + const displayName = await p.text({ + message: 'Display name (optional)', + placeholder: filename, + initialValue: filename, + }); + + if (p.isCancel(displayName)) { + return { success: false, cancelled: true }; + } + + // Note: Context length is auto-detected by node-llama-cpp from the GGUF file + + // Generate a model ID from the filename + // Convert to lowercase, replace spaces with dashes, remove special chars + let modelId = filename + .toLowerCase() + .replace(/\s+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .substring(0, 50); // Limit length + + // Fallback if modelId is empty after sanitization + if (!modelId) { + modelId = `custom-model-${Date.now()}`; + } + + // Save as custom model + await saveCustomModel({ + name: modelId, + provider: 'local', + filePath: trimmedPath, + displayName: displayName?.trim() || filename, + }); + + p.log.success(`Registered as '${modelId}'`); + + return { success: true, modelId }; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === 'ENOENT') { + p.log.error('File not found'); + } else if (nodeError.code === 'EACCES') { + p.log.error('Permission denied - file is not readable'); + } else { + p.log.error(`Error: ${error instanceof Error ? error.message : String(error)}`); + } + return { success: false }; + } +} + +/** + * Get the model name for preferences from a validated setup result. + * + * IMPORTANT: Only call this after validating with hasSelectedModel(). + * Throws if modelId is missing (indicates a bug in the calling code). + */ +export function getModelFromResult(result: LocalModelSetupResult & { modelId: string }): string { + return result.modelId; +} diff --git a/dexto/packages/cli/src/cli/utils/options.test.ts b/dexto/packages/cli/src/cli/utils/options.test.ts new file mode 100644 index 00000000..ddccb37f --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/options.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest'; +import { ZodError } from 'zod'; +import { validateCliOptions } from './options.js'; + +describe('validateCliOptions', () => { + it('does not throw for minimal valid options', () => { + const opts = { agent: 'config.yml', mode: 'cli', port: '8080' }; + expect(() => validateCliOptions(opts)).not.toThrow(); + }); + + it('does not throw for missing agent (now optional)', () => { + const opts = { mode: 'cli', port: '8080' }; + expect(() => validateCliOptions(opts)).not.toThrow(); + }); + + it('throws ZodError for empty agent string', () => { + const opts = { agent: '', mode: 'cli', port: '8080' }; + expect(() => validateCliOptions(opts)).toThrow(ZodError); + }); + + it('validates interactive flag correctly', () => { + const optsWithNoInteractive = { mode: 'cli', port: '8080', interactive: false }; + expect(() => validateCliOptions(optsWithNoInteractive)).not.toThrow(); + + const optsWithInteractive = { mode: 'cli', port: '8080', interactive: true }; + expect(() => validateCliOptions(optsWithInteractive)).not.toThrow(); + + const optsWithoutFlag = { mode: 'cli', port: '8080' }; + expect(() => validateCliOptions(optsWithoutFlag)).not.toThrow(); + }); +}); diff --git a/dexto/packages/cli/src/cli/utils/options.ts b/dexto/packages/cli/src/cli/utils/options.ts new file mode 100644 index 00000000..49c5a48b --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/options.ts @@ -0,0 +1,129 @@ +import { z } from 'zod'; +import { getSupportedProviders } from '@dexto/core'; +import chalk from 'chalk'; + +/** + * Validates the command-line options. + * @param opts - The command-line options object from commander. + * @throws {z.ZodError} If validation fails. + */ +export function validateCliOptions(opts: any): void { + const supportedProviders = getSupportedProviders().map((p) => p.toLowerCase()); + + // Base schema for primitive shape + const cliOptionShape = z + .object({ + agent: z.string().min(1, 'Agent name or path must not be empty').optional(), + strict: z.boolean().optional().default(false), + verbose: z.boolean().optional().default(true), + mode: z.enum(['web', 'cli', 'server', 'discord', 'telegram', 'mcp'], { + errorMap: () => ({ + message: + 'Mode must be one of "web", "cli", "server", "discord", "telegram", or "mcp"', + }), + }), + port: z + .string() + .refine( + (val) => { + const port = parseInt(val, 10); + return !isNaN(port) && port > 0 && port <= 65535; + }, + { message: 'Port must be a number between 1 and 65535' } + ) + .optional(), + autoApprove: z + .boolean() + .optional() + .default(false) + .describe('Automatically approve all tool executions when true'), + elicitation: z + .boolean() + .optional() + .default(true) + .describe('Enable elicitation (set to false with --no-elicitation)'), + provider: z.string().optional(), + model: z.string().optional(), + interactive: z + .boolean() + .optional() + .default(true) + .describe('Enable interactive prompts (set to false with --no-interactive)'), + }) + .strict(); + + // Basic semantic validation + const cliOptionSchema = cliOptionShape + // 1) provider must be one of the supported set if provided + .refine( + (data) => !data.provider || supportedProviders.includes(data.provider.toLowerCase()), + { + path: ['provider'], + message: `Provider must be one of: ${supportedProviders.join(', ')}`, + } + ) + // 2) Check for DISCORD_BOT_TOKEN if mode is discord + .refine( + (data) => { + if (data.mode === 'discord') { + return !!process.env.DISCORD_BOT_TOKEN; + } + return true; + }, + { + path: ['mode'], + message: + "DISCORD_BOT_TOKEN must be set in environment variables when mode is 'discord'", + } + ) + // 3) Check for TELEGRAM_BOT_TOKEN if mode is telegram + .refine( + (data) => { + if (data.mode === 'telegram') { + return !!process.env.TELEGRAM_BOT_TOKEN; + } + return true; + }, + { + path: ['mode'], + message: + "TELEGRAM_BOT_TOKEN must be set in environment variables when mode is 'telegram'", + } + ); + + // Execute validation + cliOptionSchema.parse({ + agent: opts.agent, + strict: opts.strict, + verbose: opts.verbose, + mode: opts.mode.toLowerCase(), + port: opts.port, + provider: opts.provider, + model: opts.model, + interactive: opts.interactive, + autoApprove: opts.autoApprove, + elicitation: opts.elicitation, + }); +} + +export function handleCliOptionsError(error: unknown): never { + if (error instanceof z.ZodError) { + console.error(chalk.red('❌ Invalid command-line options detected:')); + error.errors.forEach((err) => { + const fieldName = err.path.join('.') || 'Unknown Option'; + console.error(chalk.red(` • Option '${fieldName}': ${err.message}`)); + }); + console.error( + chalk.gray( + '\nPlease check your command-line arguments or run with --help for usage details.' + ) + ); + } else { + console.error( + chalk.red( + `❌ Validation error: ${error instanceof Error ? error.message : JSON.stringify(error)}` + ) + ); + } + process.exit(1); +} diff --git a/dexto/packages/cli/src/cli/utils/package-mgmt.ts b/dexto/packages/cli/src/cli/utils/package-mgmt.ts new file mode 100644 index 00000000..8e5c58dc --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/package-mgmt.ts @@ -0,0 +1,134 @@ +import { findPackageRoot, logger } from '@dexto/core'; +import fsExtra from 'fs-extra'; +import path from 'path'; +import { PackageJson } from 'type-fest'; + +/** + * Returns the install command for the given package manager + * @param pm - The package manager to use + * @returns The install command for the given package manager + */ +export function getPackageManagerInstallCommand(pm: string): string { + switch (pm) { + case 'npm': + return 'install'; + case 'yarn': + return 'add'; + case 'pnpm': + return 'add'; + case 'bun': + return 'add'; + default: + return 'install'; + } +} + +/** + * Returns the package manager for the given project + * @returns The package manager for the given project + */ +export function getPackageManager(): string { + const projectRoot = findPackageRoot(process.cwd()); + if (!projectRoot) { + return 'npm'; // Default to npm if no project root is found + } + // Check for specific lock files in this project + if (fsExtra.existsSync(path.join(projectRoot, 'pnpm-lock.yaml'))) { + return 'pnpm'; + } + if (fsExtra.existsSync(path.join(projectRoot, 'yarn.lock'))) { + return 'yarn'; + } + if ( + fsExtra.existsSync(path.join(projectRoot, 'bun.lockb')) || + fsExtra.existsSync(path.join(projectRoot, 'bun.lock')) + ) { + return 'bun'; + } + // Default to npm if no other lock file is found + return 'npm'; +} + +/** + * Goes all the way up to the root of the project and checks package.json to find the version of a project + * @returns The version of the given project + */ +export async function getPackageVersion(): Promise<string> { + const projectRoot = findPackageRoot(process.cwd()); + if (!projectRoot) { + throw new Error('Could not find project root'); + } + const pkgJsonPath = path.join(projectRoot, 'package.json'); + const content = (await fsExtra.readJSON(pkgJsonPath)) as PackageJson; + if (!content.version) { + throw new Error('Could not find version in package.json'); + } + return content.version; +} + +/** + * Adds scripts to the package.json file. + * Assumes that the package.json file is already present in the current directory + * @param scripts - The scripts to add to the package.json file + */ +export async function addScriptsToPackageJson(scripts: Record<string, string>) { + let packageJson; + try { + packageJson = await fsExtra.readJSON('package.json'); + } catch (err) { + throw new Error( + `Failed to read package.json: ${err instanceof Error ? err.message : String(err)}` + ); + } + + packageJson.scripts = { + ...packageJson.scripts, + ...scripts, + }; + + logger.debug(`Adding scripts to package.json: ${JSON.stringify(scripts, null, 2)}`); + + try { + logger.debug( + `Writing to package.json. \n Contents: ${JSON.stringify(packageJson, null, 2)}` + ); + await fsExtra.writeJSON('package.json', packageJson, { spaces: 4 }); + } catch (err) { + throw new Error( + `Failed to write to package.json: ${err instanceof Error ? err.message : String(err)}` + ); + } +} + +/** + * Checks for a package.json file in the current directory + * Useful to decide if we are in the right folder of a valid project + * @returns True if a package.json file is found, false otherwise + */ +export async function checkForFileInCurrentDirectory(fileName: string) { + const file = path.join(process.cwd(), fileName); + let isFilePresent = false; + + try { + await fsExtra.readJSON(file); + isFilePresent = true; + } catch { + isFilePresent = false; + } + + if (isFilePresent) { + return; + } + logger.debug(`${fileName} not found in the current directory.`); + throw new FileNotFoundError(`${fileName} not found in the current directory.`); +} + +/** + * Custom error class for when a required file is not found + */ +export class FileNotFoundError extends Error { + constructor(message: string) { + super(message); + this.name = 'FileNotFoundError'; + } +} diff --git a/dexto/packages/cli/src/cli/utils/project-utils.ts b/dexto/packages/cli/src/cli/utils/project-utils.ts new file mode 100644 index 00000000..a3bef9fb --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/project-utils.ts @@ -0,0 +1,26 @@ +// packages/cli/src/cli/utils/project-utils.ts + +import fs from 'node:fs/promises'; +import { parseDocument } from 'yaml'; +import { type LLMProvider, getDefaultModelForProvider, getPrimaryApiKeyEnvVar } from '@dexto/core'; + +/** + * Updates the LLM provider information in a dexto config file + * Used for project creation/initialization (modifies agent yml files) + * @param filepath Path to agent config file + * @param llmProvider LLM provider to configure + */ +export async function updateDextoConfigFile( + filepath: string, + llmProvider: LLMProvider +): Promise<void> { + const fileContent = await fs.readFile(filepath, 'utf8'); + const doc = parseDocument(fileContent); + doc.setIn(['llm', 'provider'], llmProvider); + doc.setIn(['llm', 'apiKey'], `$${getPrimaryApiKeyEnvVar(llmProvider)}`); + const defaultModel = getDefaultModelForProvider(llmProvider); + if (defaultModel) { + doc.setIn(['llm', 'model'], defaultModel); + } + await fs.writeFile(filepath, doc.toString(), 'utf8'); +} diff --git a/dexto/packages/cli/src/cli/utils/prompt-helpers.ts b/dexto/packages/cli/src/cli/utils/prompt-helpers.ts new file mode 100644 index 00000000..9def0ba2 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/prompt-helpers.ts @@ -0,0 +1,88 @@ +/** + * Consistent prompt helpers for CLI flows. + * + * These helpers wrap @clack/prompts with automatic cancel handling. + * Use for linear flows where cancel should exit the process. + */ + +import * as p from '@clack/prompts'; + +// ============================================================================= +// TYPE DEFINITIONS +// ============================================================================= + +type SelectOptions = Parameters<typeof p.select>[0]; +type TextOptions = Parameters<typeof p.text>[0]; +type ConfirmOptions = Parameters<typeof p.confirm>[0]; + +// ============================================================================= +// LINEAR FLOW HELPERS (cancel = exit) +// ============================================================================= + +/** + * Select prompt that exits on cancel. + * Use for linear flows where cancel should abort the entire operation. + * + * @example + * const choice = await selectOrExit<'a' | 'b'>({ + * message: 'Pick one', + * options: [{ value: 'a', label: 'A' }, { value: 'b', label: 'B' }] + * }, 'Operation cancelled'); + */ +export async function selectOrExit<T extends string>( + options: SelectOptions, + cancelMessage = 'Cancelled' +): Promise<T> { + const result = await p.select(options); + if (p.isCancel(result)) { + p.cancel(cancelMessage); + process.exit(0); + } + return result as T; +} + +/** + * Text prompt that exits on cancel. + * Use for linear flows where cancel should abort the entire operation. + * + * @example + * const name = await textOrExit({ + * message: 'Enter your name', + * placeholder: 'John Doe' + * }, 'Operation cancelled'); + */ +export async function textOrExit( + options: TextOptions, + cancelMessage = 'Cancelled' +): Promise<string> { + const result = await p.text(options); + if (p.isCancel(result)) { + p.cancel(cancelMessage); + process.exit(0); + } + return result; +} + +/** + * Confirm prompt that exits on cancel. + * Use for linear flows where cancel should abort the entire operation. + * + * Note: This only exits on cancel (Ctrl+C), not when user selects "No". + * For "must confirm or abort" behavior, check the return value separately. + * + * @example + * const confirmed = await confirmOrExit({ + * message: 'Are you sure?' + * }, 'Operation cancelled'); + */ +export async function confirmOrExit( + options: ConfirmOptions, + cancelMessage = 'Cancelled' +): Promise<boolean> { + const result = await p.confirm(options); + if (p.isCancel(result)) { + p.cancel(cancelMessage); + process.exit(0); + } + return result; +} diff --git a/dexto/packages/cli/src/cli/utils/provider-setup.ts b/dexto/packages/cli/src/cli/utils/provider-setup.ts new file mode 100644 index 00000000..91f0b414 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/provider-setup.ts @@ -0,0 +1,430 @@ +// packages/cli/src/cli/utils/provider-setup.ts + +import * as p from '@clack/prompts'; +import chalk from 'chalk'; +import open from 'open'; +import { + type LLMProvider, + LLM_PROVIDERS, + LLM_REGISTRY, + getDefaultModelForProvider, +} from '@dexto/core'; +import { getPrimaryApiKeyEnvVar } from '@dexto/agent-management'; + +/** + * Provider category for organizing the selection menu + */ +type ProviderCategory = 'recommended' | 'local' | 'cloud' | 'gateway' | 'enterprise'; + +/** + * Extended provider information for setup + */ +interface ProviderOption { + value: LLMProvider; + label: string; + hint: string; + category: ProviderCategory; + apiKeyUrl?: string; + apiKeyPrefix?: string; + apiKeyMinLength?: number; + requiresBaseURL?: boolean; + envVar: string; + free?: boolean; +} + +/** + * Provider configuration registry + * Organized by category for better UX + * + * Note: dexto is NOT included here - it's a transparent routing layer, + * not a user-selectable provider. When logged into Dexto, requests are + * automatically routed through the Dexto gateway. + */ +export const PROVIDER_REGISTRY: Partial<Record<LLMProvider, ProviderOption>> = { + google: { + value: 'google', + label: 'Google Gemini', + hint: 'Free tier, 1M+ context, multimodal', + category: 'recommended', + apiKeyUrl: 'https://aistudio.google.com/apikey', + apiKeyPrefix: 'AIza', + apiKeyMinLength: 20, + envVar: 'GOOGLE_API_KEY', + free: true, + }, + groq: { + value: 'groq', + label: 'Groq', + hint: 'Free tier, ultra-fast inference', + category: 'recommended', + apiKeyUrl: 'https://console.groq.com/keys', + apiKeyPrefix: 'gsk_', + apiKeyMinLength: 40, + envVar: 'GROQ_API_KEY', + free: true, + }, + // Local providers - run AI completely on your machine + local: { + value: 'local', + label: 'Local Models', + hint: 'Run Llama, Qwen, Mistral locally - Free, private, offline', + category: 'local', + envVar: '', // No API key required + free: true, + }, + ollama: { + value: 'ollama', + label: 'Ollama', + hint: 'Use Ollama server for local inference', + category: 'local', + envVar: '', // No API key required (optional OLLAMA_API_KEY for remote) + free: true, + }, + openai: { + value: 'openai', + label: 'OpenAI', + hint: 'GPT-4o, GPT-5, o1/o3 reasoning', + category: 'cloud', + apiKeyUrl: 'https://platform.openai.com/api-keys', + apiKeyPrefix: 'sk-', + apiKeyMinLength: 40, + envVar: 'OPENAI_API_KEY', + }, + anthropic: { + value: 'anthropic', + label: 'Anthropic', + hint: 'Claude 4.5, best for coding', + category: 'cloud', + apiKeyUrl: 'https://console.anthropic.com/settings/keys', + apiKeyPrefix: 'sk-ant-', + apiKeyMinLength: 40, + envVar: 'ANTHROPIC_API_KEY', + }, + xai: { + value: 'xai', + label: 'xAI', + hint: 'Grok models', + category: 'cloud', + apiKeyUrl: 'https://console.x.ai/team/default/api-keys', + envVar: 'XAI_API_KEY', + }, + cohere: { + value: 'cohere', + label: 'Cohere', + hint: 'Command models, RAG-optimized', + category: 'cloud', + apiKeyUrl: 'https://dashboard.cohere.com/api-keys', + envVar: 'COHERE_API_KEY', + }, + openrouter: { + value: 'openrouter', + label: 'OpenRouter', + hint: '100+ models, unified API', + category: 'gateway', + apiKeyUrl: 'https://openrouter.ai/keys', + apiKeyPrefix: 'sk-or-', + apiKeyMinLength: 40, + envVar: 'OPENROUTER_API_KEY', + }, + glama: { + value: 'glama', + label: 'Glama', + hint: 'OpenAI-compatible gateway', + category: 'gateway', + apiKeyUrl: 'https://glama.ai/settings/api-keys', + envVar: 'GLAMA_API_KEY', + }, + litellm: { + value: 'litellm', + label: 'LiteLLM', + hint: 'Self-hosted proxy for 100+ providers', + category: 'gateway', + requiresBaseURL: true, + envVar: 'LITELLM_API_KEY', + }, + 'openai-compatible': { + value: 'openai-compatible', + label: 'OpenAI-Compatible', + hint: 'Ollama, vLLM, LocalAI, or any OpenAI-format API', + category: 'gateway', + requiresBaseURL: true, + envVar: 'OPENAI_COMPATIBLE_API_KEY', + }, + vertex: { + value: 'vertex', + label: 'Google Vertex AI', + hint: 'GCP-hosted Gemini & Claude (uses ADC)', + category: 'enterprise', + apiKeyUrl: 'https://console.cloud.google.com/apis/credentials', + envVar: 'GOOGLE_VERTEX_PROJECT', + }, + bedrock: { + value: 'bedrock', + label: 'AWS Bedrock', + hint: 'AWS-hosted Claude & Nova (uses AWS creds)', + category: 'enterprise', + apiKeyUrl: 'https://console.aws.amazon.com/bedrock', + envVar: 'AWS_ACCESS_KEY_ID', + }, +}; + +/** + * Get providers organized by category + */ +function getProvidersByCategory(): Record<ProviderCategory, ProviderOption[]> { + const categories: Record<ProviderCategory, ProviderOption[]> = { + recommended: [], + local: [], + cloud: [], + gateway: [], + enterprise: [], + }; + + for (const provider of LLM_PROVIDERS) { + const option = PROVIDER_REGISTRY[provider]; + if (option) { + categories[option.category].push(option); + } + } + + return categories; +} + +/** + * Build provider selection options with categories + */ +function buildProviderOptions(): Array<{ value: LLMProvider; label: string; hint: string }> { + const categories = getProvidersByCategory(); + const options: Array<{ value: LLMProvider; label: string; hint: string }> = []; + + // Recommended (free) providers first + if (categories.recommended.length > 0) { + for (const p of categories.recommended) { + options.push({ + value: p.value, + label: `${chalk.green('●')} ${p.label}`, + hint: `${p.hint} ${chalk.green('(free)')}`, + }); + } + } + + // Local providers - run AI on your machine + if (categories.local.length > 0) { + for (const p of categories.local) { + options.push({ + value: p.value, + label: `${chalk.cyan('●')} ${p.label}`, + hint: `${p.hint} ${chalk.cyan('(local)')}`, + }); + } + } + + // Cloud providers + if (categories.cloud.length > 0) { + for (const p of categories.cloud) { + options.push({ + value: p.value, + label: `${chalk.blue('●')} ${p.label}`, + hint: p.hint, + }); + } + } + + // Gateway providers + if (categories.gateway.length > 0) { + for (const p of categories.gateway) { + const suffix = p.requiresBaseURL ? chalk.gray(' (requires URL)') : ''; + options.push({ + value: p.value, + label: `${chalk.rgb(255, 165, 0)('●')} ${p.label}`, + hint: `${p.hint}${suffix}`, + }); + } + } + + // Enterprise providers + if (categories.enterprise.length > 0) { + for (const p of categories.enterprise) { + options.push({ + value: p.value, + label: `${chalk.cyan('●')} ${p.label}`, + hint: p.hint, + }); + } + } + + return options; +} + +/** + * Interactive provider selection with back option. + * @returns Selected provider, '_back' if back selected, or null if cancelled + */ +export async function selectProvider(): Promise<LLMProvider | '_back' | null> { + const options = buildProviderOptions(); + + const choice = await p.select({ + message: 'Choose your AI provider', + options: [ + ...options, + { + value: '_back' as const, + label: chalk.gray('← Back'), + hint: 'Return to previous menu', + }, + ], + }); + + if (p.isCancel(choice)) { + return null; + } + + return choice as LLMProvider | '_back'; +} + +/** + * Get provider display name + */ +export function getProviderDisplayName(provider: LLMProvider): string { + return PROVIDER_REGISTRY[provider]?.label || provider; +} + +/** + * Get provider option info + */ +export function getProviderInfo(provider: LLMProvider): ProviderOption | undefined { + return PROVIDER_REGISTRY[provider]; +} + +/** + * Legacy PROVIDER_OPTIONS for backwards compatibility with init-app + */ +export const PROVIDER_OPTIONS = buildProviderOptions(); + +/** + * Get API key format hint for a provider + */ +export function getApiKeyFormatHint(provider: LLMProvider): string | null { + const info = PROVIDER_REGISTRY[provider]; + if (!info?.apiKeyPrefix) return null; + return `Key should start with "${info.apiKeyPrefix}"`; +} + +/** + * Validates API key format for a provider with detailed error messages + */ +export function validateApiKeyFormat( + apiKey: string, + provider: LLMProvider +): { valid: boolean; error?: string } { + const info = PROVIDER_REGISTRY[provider]; + const trimmed = apiKey.trim(); + + if (!trimmed) { + return { valid: false, error: 'API key cannot be empty' }; + } + + // Check minimum length if specified + if (info?.apiKeyMinLength && trimmed.length < info.apiKeyMinLength) { + return { + valid: false, + error: `API key seems too short (expected ${info.apiKeyMinLength}+ characters, got ${trimmed.length})`, + }; + } + + // Check prefix if specified + if (info?.apiKeyPrefix && !trimmed.startsWith(info.apiKeyPrefix)) { + const prefixLen = info.apiKeyPrefix.length; + return { + valid: false, + error: `Invalid format: ${getProviderDisplayName(provider)} keys start with "${info.apiKeyPrefix}" (got "${trimmed.slice(0, prefixLen)}...")`, + }; + } + + return { valid: true }; +} + +/** + * Legacy validation function for backwards compatibility + */ +export function isValidApiKeyFormat(apiKey: string, provider: LLMProvider): boolean { + return validateApiKeyFormat(apiKey, provider).valid; +} + +/** + * Gets provider-specific instructions for API key setup + */ +export function getProviderInstructions( + provider: LLMProvider +): { title: string; content: string; url?: string | undefined } | null { + const info = PROVIDER_REGISTRY[provider]; + if (!info) return null; + + const freeTag = info.free ? chalk.green(' (Free)') : ''; + const title = `${getProviderDisplayName(provider)} API Key${freeTag}`; + + let content = ''; + + if (info.apiKeyUrl) { + content += `1. Visit: ${chalk.cyan(info.apiKeyUrl)}\n`; + content += `2. Sign in to your account\n`; + content += `3. Create a new API key\n`; + content += `4. Copy and paste it below\n`; + } else if (info.requiresBaseURL) { + content += `This provider requires a custom endpoint URL.\n`; + content += `You'll configure both the URL and API key in the next steps.\n`; + } + + if (info.apiKeyPrefix) { + content += `\n${chalk.gray(`Key format: ${info.apiKeyPrefix}...`)}`; + } + + return { title, content, url: info.apiKeyUrl }; +} + +/** + * Open the API key URL in the browser + */ +export async function openApiKeyUrl(provider: LLMProvider): Promise<boolean> { + const info = PROVIDER_REGISTRY[provider]; + if (!info?.apiKeyUrl) return false; + + try { + await open(info.apiKeyUrl); + return true; + } catch { + return false; + } +} + +/** + * Check if provider requires a base URL + */ +export function providerRequiresBaseURL(provider: LLMProvider): boolean { + return PROVIDER_REGISTRY[provider]?.requiresBaseURL === true; +} + +/** + * Get default model for provider, with fallback for custom providers + */ +export function getDefaultModel(provider: LLMProvider): string { + const defaultModel = getDefaultModelForProvider(provider); + if (defaultModel) return defaultModel; + + // Fallback for providers without a default (custom providers) + const providerInfo = LLM_REGISTRY[provider]; + if (providerInfo?.models && providerInfo.models.length > 0) { + return providerInfo.models[0]!.name; + } + + // For providers that accept any model, return empty to prompt user + return ''; +} + +/** + * Get environment variable name for provider's API key. + * Uses the canonical env var from the core api-key-resolver. + */ +export function getProviderEnvVar(provider: LLMProvider): string { + return getPrimaryApiKeyEnvVar(provider); +} diff --git a/dexto/packages/cli/src/cli/utils/scaffolding-utils.test.ts b/dexto/packages/cli/src/cli/utils/scaffolding-utils.test.ts new file mode 100644 index 00000000..66a74e01 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/scaffolding-utils.test.ts @@ -0,0 +1,430 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { + validateProjectName, + promptForProjectName, + createProjectDirectory, + setupGitRepo, + createGitignore, + initPackageJson, + createTsconfigForApp, + createTsconfigForImage, + createTsconfigForProject, + installDependencies, + createEnvExample, + ensureDirectory, +} from './scaffolding-utils.js'; + +// Mock dependencies +vi.mock('fs-extra', () => { + const mock = { + mkdir: vi.fn(), + writeFile: vi.fn(), + readFile: vi.fn(), + writeJSON: vi.fn(), + ensureDir: vi.fn(), + }; + return { + default: mock, + ...mock, + }; +}); + +vi.mock('@clack/prompts', () => ({ + text: vi.fn(), + isCancel: vi.fn(), + cancel: vi.fn(), +})); + +vi.mock('./execute.js', () => ({ + executeWithTimeout: vi.fn(), +})); + +vi.mock('./package-mgmt.js', () => ({ + getPackageManager: vi.fn(), + getPackageManagerInstallCommand: vi.fn(), +})); + +const fs = await import('fs-extra'); +const p = await import('@clack/prompts'); +const { executeWithTimeout } = await import('./execute.js'); +const { getPackageManager, getPackageManagerInstallCommand } = await import('./package-mgmt.js'); + +describe('scaffolding-utils', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('validateProjectName', () => { + it('should accept valid project names', () => { + expect(validateProjectName('my-project')).toBeUndefined(); + expect(validateProjectName('myProject')).toBeUndefined(); + expect(validateProjectName('my_project')).toBeUndefined(); + expect(validateProjectName('Project123')).toBeUndefined(); + expect(validateProjectName('a')).toBeUndefined(); + }); + + it('should reject names starting with numbers', () => { + const error = validateProjectName('123-project'); + expect(error).toBeDefined(); + expect(error).toContain('Must start with a letter'); + }); + + it('should reject names starting with special characters', () => { + const error = validateProjectName('-my-project'); + expect(error).toBeDefined(); + expect(error).toContain('Must start with a letter'); + }); + + it('should reject names with invalid characters', () => { + const error = validateProjectName('my@project'); + expect(error).toBeDefined(); + expect(error).toContain('Must start with a letter'); + }); + + it('should reject empty names', () => { + const error = validateProjectName(''); + expect(error).toBeDefined(); + }); + }); + + describe('promptForProjectName', () => { + it('should return valid project name on first attempt', async () => { + vi.mocked(p.text).mockResolvedValue('valid-project'); + + const result = await promptForProjectName('default-name', 'Enter name'); + + expect(result).toBe('valid-project'); + expect(p.text).toHaveBeenCalledTimes(1); + }); + + it('should exit on cancel', async () => { + const mockExit = vi.spyOn(process, 'exit').mockImplementation(() => { + throw new Error('process.exit called'); + }); + vi.mocked(p.text).mockResolvedValue(Symbol('cancel') as any); + vi.mocked(p.isCancel).mockReturnValue(true); + + await expect(promptForProjectName()).rejects.toThrow('process.exit called'); + + expect(p.cancel).toHaveBeenCalledWith('Project creation cancelled'); + expect(mockExit).toHaveBeenCalledWith(0); + + mockExit.mockRestore(); + }); + + it('should re-prompt on invalid name', async () => { + vi.mocked(p.text) + .mockResolvedValueOnce('123-invalid') + .mockResolvedValueOnce('valid-project'); + vi.mocked(p.isCancel).mockReturnValue(false); + + const result = await promptForProjectName(); + + expect(result).toBe('valid-project'); + expect(p.text).toHaveBeenCalledTimes(2); + }); + }); + + describe('createProjectDirectory', () => { + const mockSpinner = { + stop: vi.fn(), + } as any; + + it('should create directory successfully', async () => { + vi.mocked(fs.mkdir).mockResolvedValue(undefined); + + const result = await createProjectDirectory('my-project', mockSpinner); + + expect(result).toContain('my-project'); + expect(fs.mkdir).toHaveBeenCalledWith(expect.stringContaining('my-project')); + }); + + it('should exit if directory exists', async () => { + const mockExit = vi.spyOn(process, 'exit').mockImplementation(() => { + throw new Error('process.exit called'); + }); + const error = new Error('EEXIST') as any; + error.code = 'EEXIST'; + vi.mocked(fs.mkdir).mockRejectedValue(error); + + await expect(createProjectDirectory('existing-project', mockSpinner)).rejects.toThrow( + 'process.exit called' + ); + + expect(mockSpinner.stop).toHaveBeenCalledWith( + expect.stringContaining('already exists') + ); + expect(mockExit).toHaveBeenCalledWith(1); + + mockExit.mockRestore(); + }); + + it('should throw on other errors', async () => { + const error = new Error('Permission denied'); + vi.mocked(fs.mkdir).mockRejectedValue(error); + + await expect(createProjectDirectory('my-project', mockSpinner)).rejects.toThrow( + 'Permission denied' + ); + }); + }); + + describe('setupGitRepo', () => { + it('should initialize git repository', async () => { + vi.mocked(executeWithTimeout).mockResolvedValue(undefined as any); + + await setupGitRepo('/path/to/project'); + + expect(executeWithTimeout).toHaveBeenCalledWith('git', ['init'], { + cwd: '/path/to/project', + }); + }); + }); + + describe('createGitignore', () => { + it('should create .gitignore with base entries', async () => { + vi.mocked(fs.writeFile).mockResolvedValue(undefined); + + await createGitignore('/path/to/project'); + + expect(fs.writeFile).toHaveBeenCalledWith( + '/path/to/project/.gitignore', + expect.stringContaining('node_modules') + ); + expect(fs.writeFile).toHaveBeenCalledWith( + '/path/to/project/.gitignore', + expect.stringContaining('.env') + ); + }); + + it('should include additional entries', async () => { + vi.mocked(fs.writeFile).mockResolvedValue(undefined); + + await createGitignore('/path/to/project', ['*.tmp', 'cache/']); + + expect(fs.writeFile).toHaveBeenCalledWith( + '/path/to/project/.gitignore', + expect.stringMatching(/\*.tmp/) + ); + expect(fs.writeFile).toHaveBeenCalledWith( + '/path/to/project/.gitignore', + expect.stringMatching(/cache\//) + ); + }); + }); + + describe('initPackageJson', () => { + beforeEach(() => { + vi.mocked(executeWithTimeout).mockResolvedValue(undefined as any); + (vi.mocked(fs.readFile) as any).mockResolvedValue( + JSON.stringify({ + name: 'temp-name', + version: '0.0.0', + }) + ); + vi.mocked(fs.writeFile).mockResolvedValue(undefined); + }); + + it('should initialize package.json for app', async () => { + await initPackageJson('/path/to/project', 'my-app', 'app'); + + expect(executeWithTimeout).toHaveBeenCalledWith('npm', ['init', '-y'], { + cwd: '/path/to/project', + }); + + const writeCall = vi.mocked(fs.writeFile).mock.calls[0]; + if (!writeCall) { + throw new Error('writeFile was not called'); + } + const packageJson = JSON.parse(writeCall[1] as string); + + expect(packageJson.name).toBe('my-app'); + expect(packageJson.version).toBe('1.0.0'); + expect(packageJson.type).toBe('module'); + expect(packageJson.description).toBe('Dexto application'); + }); + + it('should initialize package.json for image', async () => { + await initPackageJson('/path/to/project', 'my-image', 'image'); + + const writeCall = vi.mocked(fs.writeFile).mock.calls[0]; + if (!writeCall) { + throw new Error('writeFile was not called'); + } + const packageJson = JSON.parse(writeCall[1] as string); + + expect(packageJson.description).toBe('Dexto image providing agent harness'); + expect(packageJson.main).toBe('./dist/index.js'); + expect(packageJson.types).toBe('./dist/index.d.ts'); + expect(packageJson.exports).toBeDefined(); + }); + + it('should initialize package.json for project', async () => { + await initPackageJson('/path/to/project', 'my-project', 'project'); + + const writeCall = vi.mocked(fs.writeFile).mock.calls[0]; + if (!writeCall) { + throw new Error('writeFile was not called'); + } + const packageJson = JSON.parse(writeCall[1] as string); + + expect(packageJson.description).toBe('Custom Dexto project'); + expect(packageJson.bin).toBeDefined(); + expect(packageJson.bin['my-project']).toBe('./dist/src/index.js'); + }); + }); + + describe('createTsconfigForApp', () => { + it('should create tsconfig.json for app', async () => { + vi.mocked(fs.writeJSON).mockResolvedValue(undefined); + + await createTsconfigForApp('/path/to/project', 'src'); + + expect(fs.writeJSON).toHaveBeenCalledWith( + '/path/to/project/tsconfig.json', + expect.objectContaining({ + compilerOptions: expect.objectContaining({ + target: 'ES2022', + module: 'ESNext', + outDir: 'dist', + rootDir: 'src', + }), + include: ['src/**/*.ts'], + }), + { spaces: 4 } + ); + }); + }); + + describe('createTsconfigForImage', () => { + it('should create tsconfig.json for image', async () => { + vi.mocked(fs.writeJSON).mockResolvedValue(undefined); + + await createTsconfigForImage('/path/to/project'); + + expect(fs.writeJSON).toHaveBeenCalledWith( + '/path/to/project/tsconfig.json', + expect.objectContaining({ + compilerOptions: expect.objectContaining({ + target: 'ES2022', + module: 'ES2022', + moduleResolution: 'bundler', + declaration: true, + }), + include: expect.arrayContaining([ + 'dexto.image.ts', + 'tools/**/*', + 'blob-store/**/*', + ]), + }), + { spaces: 2 } + ); + }); + }); + + describe('createTsconfigForProject', () => { + it('should create tsconfig.json for project', async () => { + vi.mocked(fs.writeJSON).mockResolvedValue(undefined); + + await createTsconfigForProject('/path/to/project'); + + expect(fs.writeJSON).toHaveBeenCalledWith( + '/path/to/project/tsconfig.json', + expect.objectContaining({ + compilerOptions: expect.objectContaining({ + module: 'ES2022', + moduleResolution: 'bundler', + }), + include: expect.arrayContaining([ + 'src/**/*', + 'storage/**/*', + 'dexto.config.ts', + ]), + }), + { spaces: 2 } + ); + }); + }); + + describe('installDependencies', () => { + beforeEach(() => { + vi.mocked(getPackageManager).mockReturnValue('pnpm'); + vi.mocked(getPackageManagerInstallCommand).mockReturnValue('add'); + vi.mocked(executeWithTimeout).mockResolvedValue(undefined as any); + }); + + it('should install production dependencies', async () => { + await installDependencies('/path/to/project', { + dependencies: ['@dexto/core', 'zod'], + }); + + expect(executeWithTimeout).toHaveBeenCalledWith('pnpm', ['add', '@dexto/core', 'zod'], { + cwd: '/path/to/project', + }); + }); + + it('should install dev dependencies', async () => { + await installDependencies('/path/to/project', { + devDependencies: ['typescript', '@types/node'], + }); + + expect(executeWithTimeout).toHaveBeenCalledWith( + 'pnpm', + ['add', 'typescript', '@types/node', '--save-dev'], + { cwd: '/path/to/project' } + ); + }); + + it('should install both dependencies and devDependencies', async () => { + await installDependencies('/path/to/project', { + dependencies: ['@dexto/core'], + devDependencies: ['typescript'], + }); + + expect(executeWithTimeout).toHaveBeenCalledTimes(2); + }); + + it('should handle empty dependency arrays', async () => { + await installDependencies('/path/to/project', { + dependencies: [], + devDependencies: [], + }); + + expect(executeWithTimeout).not.toHaveBeenCalled(); + }); + }); + + describe('createEnvExample', () => { + it('should create .env.example with entries', async () => { + vi.mocked(fs.writeFile).mockResolvedValue(undefined); + + await createEnvExample('/path/to/project', { + OPENAI_API_KEY: 'sk-...', + DATABASE_URL: 'postgresql://...', + }); + + expect(fs.writeFile).toHaveBeenCalledWith( + '/path/to/project/.env.example', + 'OPENAI_API_KEY=sk-...\nDATABASE_URL=postgresql://...' + ); + }); + + it('should handle empty entries', async () => { + vi.mocked(fs.writeFile).mockResolvedValue(undefined); + + await createEnvExample('/path/to/project', {}); + + expect(fs.writeFile).toHaveBeenCalledWith('/path/to/project/.env.example', ''); + }); + }); + + describe('ensureDirectory', () => { + it('should ensure directory exists', async () => { + vi.mocked(fs.ensureDir).mockResolvedValue(undefined); + + await ensureDirectory('/path/to/directory'); + + expect(fs.ensureDir).toHaveBeenCalledWith('/path/to/directory'); + }); + }); +}); diff --git a/dexto/packages/cli/src/cli/utils/scaffolding-utils.ts b/dexto/packages/cli/src/cli/utils/scaffolding-utils.ts new file mode 100644 index 00000000..55fcff78 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/scaffolding-utils.ts @@ -0,0 +1,296 @@ +import fs from 'fs-extra'; +import path from 'path'; +import chalk from 'chalk'; +import * as p from '@clack/prompts'; +import { executeWithTimeout } from './execute.js'; +import { textOrExit } from './prompt-helpers.js'; +import { getPackageManager, getPackageManagerInstallCommand } from './package-mgmt.js'; + +/** + * Validates a project name against the standard regex + * @param name - The project name to validate + * @returns Error message if invalid, undefined if valid + */ +export function validateProjectName(name: string): string | undefined { + const nameRegex = /^[a-zA-Z][a-zA-Z0-9-_]*$/; + if (!nameRegex.test(name)) { + return 'Must start with a letter and contain only letters, numbers, hyphens, or underscores'; + } + return undefined; +} + +/** + * Prompts user for project name with validation + * @param defaultName - Default project name + * @param promptMessage - Custom prompt message + * @returns The validated project name + */ +export async function promptForProjectName( + defaultName: string = 'my-dexto-project', + promptMessage: string = 'What do you want to name your project?' +): Promise<string> { + let input; + do { + input = await textOrExit( + { + message: promptMessage, + placeholder: defaultName, + defaultValue: defaultName, + }, + 'Project creation cancelled' + ); + + const error = validateProjectName(input); + if (error) { + console.log(chalk.red(`Invalid project name: ${error}`)); + } + } while (validateProjectName(input)); + + return input; +} + +/** + * Creates a project directory with error handling + * @param projectName - The name of the project + * @param spinner - Clack spinner instance + * @returns The absolute path to the created project + */ +export async function createProjectDirectory( + projectName: string, + spinner: ReturnType<typeof p.spinner> +): Promise<string> { + const projectPath = path.resolve(process.cwd(), projectName); + + try { + await fs.mkdir(projectPath); + return projectPath; + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'EEXIST') { + spinner.stop( + `Directory "${projectName}" already exists. Please choose a different name or delete the existing directory.` + ); + process.exit(1); + } else { + spinner.stop(`Failed to create project: ${error}`); + throw error; + } + } +} + +/** + * Initializes a git repository in the project + * @param projectPath - The project directory path + */ +export async function setupGitRepo(projectPath: string): Promise<void> { + await executeWithTimeout('git', ['init'], { cwd: projectPath }); +} + +/** + * Creates a .gitignore file with common entries + * @param projectPath - The project directory path + * @param additionalEntries - Additional entries to include + */ +export async function createGitignore( + projectPath: string, + additionalEntries: string[] = [] +): Promise<void> { + const baseEntries = ['node_modules', '.env', 'dist', '.dexto', '*.log']; + const allEntries = [...baseEntries, ...additionalEntries]; + await fs.writeFile(path.join(projectPath, '.gitignore'), allEntries.join('\n')); +} + +/** + * Initializes package.json for a project + * @param projectPath - The project directory path + * @param projectName - The project name + * @param type - Project type for package.json customization + */ +export async function initPackageJson( + projectPath: string, + projectName: string, + type: 'app' | 'image' | 'project' +): Promise<void> { + // Initialize with npm (it creates package.json regardless of package manager) + await executeWithTimeout('npm', ['init', '-y'], { cwd: projectPath }); + + // Read and customize package.json + const packageJsonPath = path.join(projectPath, 'package.json'); + const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8')); + + packageJson.name = projectName; + packageJson.version = '1.0.0'; + packageJson.type = 'module'; + + // Customize based on type + if (type === 'app') { + packageJson.description = 'Dexto application'; + } else if (type === 'image') { + packageJson.description = `Dexto image providing agent harness`; + packageJson.main = './dist/index.js'; + packageJson.types = './dist/index.d.ts'; + packageJson.exports = { + '.': { + types: './dist/index.d.ts', + import: './dist/index.js', + }, + }; + } else if (type === 'project') { + packageJson.description = 'Custom Dexto project'; + packageJson.bin = { + [projectName]: './dist/src/index.js', + }; + } + + await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); +} + +/** + * Creates tsconfig.json for an app project + * @param projectPath - The project directory path + * @param srcDir - Source directory (e.g., 'src') + */ +export async function createTsconfigForApp(projectPath: string, srcDir: string): Promise<void> { + const tsconfig = { + compilerOptions: { + target: 'ES2022', + module: 'ESNext', + moduleResolution: 'node', + strict: true, + esModuleInterop: true, + forceConsistentCasingInFileNames: true, + skipLibCheck: true, + outDir: 'dist', + rootDir: srcDir, + }, + include: [`${srcDir}/**/*.ts`], + exclude: ['node_modules', 'dist', '.dexto'], + }; + + await fs.writeJSON(path.join(projectPath, 'tsconfig.json'), tsconfig, { spaces: 4 }); +} + +/** + * Creates tsconfig.json for an image project + * @param projectPath - The project directory path + */ +export async function createTsconfigForImage(projectPath: string): Promise<void> { + const tsconfig = { + compilerOptions: { + target: 'ES2022', + module: 'ES2022', + lib: ['ES2022'], + moduleResolution: 'bundler', + outDir: './dist', + declaration: true, + declarationMap: true, + sourceMap: true, + strict: true, + esModuleInterop: true, + skipLibCheck: true, + forceConsistentCasingInFileNames: true, + resolveJsonModule: true, + allowSyntheticDefaultImports: true, + types: ['node'], + }, + include: [ + 'dexto.image.ts', + 'tools/**/*', + 'blob-store/**/*', + 'compression/**/*', + 'plugins/**/*', + ], + exclude: ['node_modules', 'dist'], + }; + + await fs.writeJSON(path.join(projectPath, 'tsconfig.json'), tsconfig, { spaces: 2 }); +} + +/** + * Creates tsconfig.json for a project (manual registration) + * @param projectPath - The project directory path + */ +export async function createTsconfigForProject(projectPath: string): Promise<void> { + const tsconfig = { + compilerOptions: { + target: 'ES2022', + module: 'ES2022', + lib: ['ES2022'], + moduleResolution: 'bundler', + outDir: './dist', + declaration: true, + declarationMap: true, + sourceMap: true, + strict: true, + esModuleInterop: true, + skipLibCheck: true, + forceConsistentCasingInFileNames: true, + resolveJsonModule: true, + allowSyntheticDefaultImports: true, + types: ['node'], + }, + include: [ + 'src/**/*', + 'storage/**/*', + 'tools/**/*', + 'plugins/**/*', + 'shared/**/*', + 'dexto.config.ts', + ], + exclude: ['node_modules', 'dist'], + }; + + await fs.writeJSON(path.join(projectPath, 'tsconfig.json'), tsconfig, { spaces: 2 }); +} + +/** + * Installs dependencies for a project + * @param projectPath - The project directory path + * @param deps - Dependencies to install + */ +export async function installDependencies( + projectPath: string, + deps: { + dependencies?: string[]; + devDependencies?: string[]; + }, + packageManager?: string +): Promise<void> { + const pm = packageManager || getPackageManager(); + const installCommand = getPackageManagerInstallCommand(pm); + + if (deps.dependencies && deps.dependencies.length > 0) { + await executeWithTimeout(pm, [installCommand, ...deps.dependencies], { + cwd: projectPath, + }); + } + + if (deps.devDependencies && deps.devDependencies.length > 0) { + await executeWithTimeout(pm, [installCommand, ...deps.devDependencies, '--save-dev'], { + cwd: projectPath, + }); + } +} + +/** + * Creates a .env.example file + * @param projectPath - The project directory path + * @param entries - Environment variables to include (key-value pairs) + */ +export async function createEnvExample( + projectPath: string, + entries: Record<string, string> +): Promise<void> { + const content = Object.entries(entries) + .map(([key, value]) => `${key}=${value}`) + .join('\n'); + + await fs.writeFile(path.join(projectPath, '.env.example'), content); +} + +/** + * Ensures a directory exists, creates if not + * @param dirPath - The directory path to ensure + */ +export async function ensureDirectory(dirPath: string): Promise<void> { + await fs.ensureDir(dirPath); +} diff --git a/dexto/packages/cli/src/cli/utils/setup-utils.test.ts b/dexto/packages/cli/src/cli/utils/setup-utils.test.ts new file mode 100644 index 00000000..446801c7 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/setup-utils.test.ts @@ -0,0 +1,270 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { requiresSetup, isFirstTimeUser } from './setup-utils.js'; + +// Mock dependencies +vi.mock('@dexto/agent-management', async () => { + const actual = await vi.importActual('@dexto/agent-management'); + return { + ...actual, + globalPreferencesExist: vi.fn(), + loadGlobalPreferences: vi.fn(), + }; +}); + +vi.mock('@dexto/core', async () => { + const actual = await vi.importActual('@dexto/core'); + return { + ...actual, + getExecutionContext: vi.fn(), + }; +}); + +const { globalPreferencesExist, loadGlobalPreferences } = await import('@dexto/agent-management'); +const { getExecutionContext } = await import('@dexto/core'); + +describe('requiresSetup', () => { + const originalEnv = process.env.DEXTO_DEV_MODE; + + beforeEach(() => { + vi.clearAllMocks(); + delete process.env.DEXTO_DEV_MODE; + }); + + afterEach(() => { + if (originalEnv === undefined) { + delete process.env.DEXTO_DEV_MODE; + } else { + process.env.DEXTO_DEV_MODE = originalEnv; + } + }); + + describe('Dev mode (DEXTO_DEV_MODE=true)', () => { + beforeEach(() => { + process.env.DEXTO_DEV_MODE = 'true'; + }); + + it('should skip setup in source context', async () => { + vi.mocked(getExecutionContext).mockReturnValue('dexto-source'); + + const result = await requiresSetup(); + + expect(result).toBe(false); + expect(globalPreferencesExist).not.toHaveBeenCalled(); + }); + + it('should not skip setup in global-cli context', async () => { + vi.mocked(getExecutionContext).mockReturnValue('global-cli'); + vi.mocked(globalPreferencesExist).mockReturnValue(false); + + const result = await requiresSetup(); + + expect(result).toBe(true); + }); + + it('should not skip setup in project context', async () => { + vi.mocked(getExecutionContext).mockReturnValue('dexto-project'); + + const result = await requiresSetup(); + + expect(result).toBe(false); + }); + }); + + describe('Project context', () => { + beforeEach(() => { + vi.mocked(getExecutionContext).mockReturnValue('dexto-project'); + }); + + it('should skip setup even for first-time user', async () => { + vi.mocked(globalPreferencesExist).mockReturnValue(false); + + const result = await requiresSetup(); + + expect(result).toBe(false); + }); + + it('should skip setup even with existing preferences', async () => { + vi.mocked(globalPreferencesExist).mockReturnValue(true); + + const result = await requiresSetup(); + + expect(result).toBe(false); + expect(loadGlobalPreferences).not.toHaveBeenCalled(); + }); + }); + + describe('First-time user (no preferences)', () => { + beforeEach(() => { + vi.mocked(globalPreferencesExist).mockReturnValue(false); + }); + + it('should require setup in source context', async () => { + vi.mocked(getExecutionContext).mockReturnValue('dexto-source'); + + const result = await requiresSetup(); + + expect(result).toBe(true); + }); + + it('should require setup in global-cli context', async () => { + vi.mocked(getExecutionContext).mockReturnValue('global-cli'); + + const result = await requiresSetup(); + + expect(result).toBe(true); + }); + }); + + describe('Has preferences', () => { + beforeEach(() => { + vi.mocked(globalPreferencesExist).mockReturnValue(true); + }); + + it('should not require setup with valid preferences in source context', async () => { + vi.mocked(getExecutionContext).mockReturnValue('dexto-source'); + vi.mocked(loadGlobalPreferences).mockResolvedValue({ + llm: { + provider: 'google', + model: 'gemini-2.5-pro', + apiKey: '$GOOGLE_GENERATIVE_AI_API_KEY', + }, + defaults: { + defaultAgent: 'coding-agent', + defaultMode: 'web', + }, + setup: { + completed: true, + apiKeyPending: false, + baseURLPending: false, + }, + sounds: { + enabled: true, + onApprovalRequired: true, + onTaskComplete: true, + }, + }); + + const result = await requiresSetup(); + + expect(result).toBe(false); + }); + + it('should not require setup with valid preferences in global-cli context', async () => { + vi.mocked(getExecutionContext).mockReturnValue('global-cli'); + vi.mocked(loadGlobalPreferences).mockResolvedValue({ + llm: { + provider: 'google', + model: 'gemini-2.5-pro', + apiKey: '$GOOGLE_GENERATIVE_AI_API_KEY', + }, + defaults: { + defaultAgent: 'coding-agent', + defaultMode: 'web', + }, + setup: { + completed: true, + apiKeyPending: false, + baseURLPending: false, + }, + sounds: { + enabled: true, + onApprovalRequired: true, + onTaskComplete: true, + }, + }); + + const result = await requiresSetup(); + + expect(result).toBe(false); + }); + + it('should require setup with incomplete setup flag', async () => { + vi.mocked(getExecutionContext).mockReturnValue('dexto-source'); + vi.mocked(loadGlobalPreferences).mockResolvedValue({ + llm: { + provider: 'google', + model: 'gemini-2.5-pro', + apiKey: '$GOOGLE_GENERATIVE_AI_API_KEY', + }, + defaults: { + defaultAgent: 'coding-agent', + defaultMode: 'web', + }, + setup: { + completed: false, + apiKeyPending: false, + baseURLPending: false, + }, + sounds: { + enabled: true, + onApprovalRequired: true, + onTaskComplete: true, + }, + }); + + const result = await requiresSetup(); + + expect(result).toBe(true); + }); + + it('should require setup with missing defaultAgent', async () => { + vi.mocked(getExecutionContext).mockReturnValue('dexto-source'); + vi.mocked(loadGlobalPreferences).mockResolvedValue({ + llm: { + provider: 'google', + model: 'gemini-2.5-pro', + apiKey: '$GOOGLE_GENERATIVE_AI_API_KEY', + }, + defaults: { + defaultAgent: '', + defaultMode: 'web', + }, + setup: { + completed: true, + apiKeyPending: false, + baseURLPending: false, + }, + sounds: { + enabled: true, + onApprovalRequired: true, + onTaskComplete: true, + }, + }); + + const result = await requiresSetup(); + + expect(result).toBe(true); + }); + + it('should require setup with corrupted preferences', async () => { + vi.mocked(getExecutionContext).mockReturnValue('dexto-source'); + vi.mocked(loadGlobalPreferences).mockRejectedValue(new Error('Corrupted YAML')); + + const result = await requiresSetup(); + + expect(result).toBe(true); + }); + }); +}); + +describe('isFirstTimeUser', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should return true when preferences do not exist', () => { + vi.mocked(globalPreferencesExist).mockReturnValue(false); + + const result = isFirstTimeUser(); + + expect(result).toBe(true); + }); + + it('should return false when preferences exist', () => { + vi.mocked(globalPreferencesExist).mockReturnValue(true); + + const result = isFirstTimeUser(); + + expect(result).toBe(false); + }); +}); diff --git a/dexto/packages/cli/src/cli/utils/setup-utils.ts b/dexto/packages/cli/src/cli/utils/setup-utils.ts new file mode 100644 index 00000000..c62f5aec --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/setup-utils.ts @@ -0,0 +1,153 @@ +// packages/cli/src/cli/utils/setup-utils.ts + +import { + globalPreferencesExist, + loadGlobalPreferences, + type GlobalPreferences, +} from '@dexto/agent-management'; +import { getExecutionContext } from '@dexto/core'; + +/** + * Check if this is a first-time user (no preferences file exists) + * @returns true if user has never run setup + */ +export function isFirstTimeUser(): boolean { + return !globalPreferencesExist(); +} + +/** + * Result of checking setup state + */ +export interface SetupState { + needsSetup: boolean; + isFirstTime: boolean; + apiKeyPending: boolean; + baseURLPending: boolean; + preferences: GlobalPreferences | null; +} + +/** + * Get detailed setup state including pending items + * @returns Setup state with detailed flags + */ +export async function getSetupState(): Promise<SetupState> { + const context = getExecutionContext(); + + // Skip setup checks in dev mode when in source context + if (context === 'dexto-source' && process.env.DEXTO_DEV_MODE === 'true') { + return { + needsSetup: false, + isFirstTime: false, + apiKeyPending: false, + baseURLPending: false, + preferences: null, + }; + } + + // Project context: skip (might have project-local config) + if (context === 'dexto-project') { + return { + needsSetup: false, + isFirstTime: false, + apiKeyPending: false, + baseURLPending: false, + preferences: null, + }; + } + + // First-time user (no preferences) + if (isFirstTimeUser()) { + return { + needsSetup: true, + isFirstTime: true, + apiKeyPending: false, + baseURLPending: false, + preferences: null, + }; + } + + // Has preferences - check state + try { + const preferences = await loadGlobalPreferences(); + + // Check setup completion flag + if (!preferences.setup.completed) { + return { + needsSetup: true, + isFirstTime: false, + apiKeyPending: false, + baseURLPending: false, + preferences, + }; + } + + // Check required fields + if (!preferences.defaults.defaultAgent) { + return { + needsSetup: true, + isFirstTime: false, + apiKeyPending: false, + baseURLPending: false, + preferences, + }; + } + + // Setup is complete but may have pending items + return { + needsSetup: false, + isFirstTime: false, + apiKeyPending: preferences.setup.apiKeyPending ?? false, + baseURLPending: preferences.setup.baseURLPending ?? false, + preferences, + }; + } catch (_error) { + // Corrupted or invalid preferences + return { + needsSetup: true, + isFirstTime: false, + apiKeyPending: false, + baseURLPending: false, + preferences: null, + }; + } +} + +/** + * Check if user requires setup (missing, corrupted, or incomplete preferences) + * Context-aware: + * - Dev mode (source + DEXTO_DEV_MODE): Skip setup, uses repo configs + * - Project context: Skip setup (might have project-local config) + * - First-time user (source/global-cli): Require setup + * - Has preferences (source/global-cli): Validate them + * @returns true if setup is required + */ +export async function requiresSetup(): Promise<boolean> { + const state = await getSetupState(); + return state.needsSetup; +} + +/** + * Get appropriate guidance message based on user state + */ +export async function getSetupGuidanceMessage(): Promise<string> { + if (isFirstTimeUser()) { + return [ + "👋 Welcome to Dexto! Let's get you set up...", + '', + '🚀 Run `dexto setup` to configure your AI preferences', + ' • Choose your AI provider (Google Gemini, OpenAI, etc.)', + ' • Set up your API keys', + ' • Configure your default agent', + '', + '💡 After setup, you can install agents with: `dexto install <agent-name>`', + ].join('\n'); + } + + // Invalid, incomplete, or corrupted preferences + return [ + '⚠️ Your Dexto preferences need attention', + '', + '🔧 Run `dexto setup` to fix your configuration', + ' This will restore your AI provider settings and preferences', + ].join('\n'); +} diff --git a/dexto/packages/cli/src/cli/utils/template-engine.test.ts b/dexto/packages/cli/src/cli/utils/template-engine.test.ts new file mode 100644 index 00000000..09a559bb --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/template-engine.test.ts @@ -0,0 +1,395 @@ +import { describe, it, expect } from 'vitest'; +import { + generateIndexForImage, + generateDextoImageFile, + generateDextoConfigFile, + generateImageReadme, + generateExampleTool, + generateAppReadme, +} from './template-engine.js'; + +describe('template-engine', () => { + describe('generateIndexForImage', () => { + it('should generate index.ts for image-based app', () => { + const result = generateIndexForImage({ + projectName: 'my-app', + packageName: 'my-app', + description: 'Test app', + imageName: '@dexto/image-local', + }); + + expect(result).toContain("import { DextoAgent } from '@dexto/core'"); + expect(result).toContain("import { loadAgentConfig } from '@dexto/agent-management'"); + expect(result).toContain('Starting my-app'); + expect(result).toContain( + "const config = await loadAgentConfig('./agents/default.yml')" + ); + expect(result).toContain('const agent = new DextoAgent(config'); + expect(result).toContain('await agent.start()'); + }); + + it('should use image harness terminology in comments', () => { + const result = generateIndexForImage({ + projectName: 'my-app', + packageName: 'my-app', + description: 'Test app', + imageName: '@dexto/image-local', + }); + + expect(result).toContain( + '// Create agent - providers already registered by image environment' + ); + expect(result).toContain('// This auto-registers providers as a side-effect'); + }); + }); + + describe('generateDextoImageFile', () => { + it('should generate basic dexto.image.ts', () => { + const result = generateDextoImageFile({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + imageName: 'my-image', + }); + + expect(result).toContain("import { defineImage } from '@dexto/core'"); + expect(result).toContain('export default defineImage({'); + expect(result).toContain("name: 'my-image'"); + expect(result).toContain("version: '1.0.0'"); + expect(result).toContain("description: 'Test image'"); + }); + + it('should include convention-based auto-discovery comments', () => { + const result = generateDextoImageFile({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + }); + + expect(result).toContain( + '// Providers are AUTO-DISCOVERED from convention-based folders' + ); + expect(result).toContain('// tools/ - Custom tool providers'); + expect(result).toContain('// blob-store/ - Blob storage providers'); + expect(result).toContain('// compression/ - Compression strategy providers'); + expect(result).toContain('// plugins/ - Plugin providers'); + }); + + it('should include extends field when baseImage provided', () => { + const result = generateDextoImageFile({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + baseImage: '@dexto/image-local', + }); + + expect(result).toContain("extends: '@dexto/image-local'"); + }); + + it('should not include extends field when no baseImage', () => { + const result = generateDextoImageFile({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + }); + + expect(result).not.toContain('extends:'); + }); + + it('should use target from context or default', () => { + const resultWithTarget = generateDextoImageFile({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + target: 'cloud-production', + }); + + expect(resultWithTarget).toContain("target: 'cloud-production'"); + + const resultDefault = generateDextoImageFile({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + }); + + expect(resultDefault).toContain("target: 'local-development'"); + }); + + it('should include default configurations', () => { + const result = generateDextoImageFile({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + }); + + expect(result).toContain('defaults: {'); + expect(result).toContain('storage: {'); + expect(result).toContain("type: 'local'"); + expect(result).toContain("type: 'sqlite'"); + expect(result).toContain('logging: {'); + }); + }); + + describe('generateDextoConfigFile', () => { + it('should generate dexto.config.ts', () => { + const result = generateDextoConfigFile({ + projectName: 'my-project', + packageName: 'my-project', + description: 'Test project', + }); + + expect(result).toContain('import {'); + expect(result).toContain('blobStoreRegistry'); + expect(result).toContain('customToolRegistry'); + expect(result).toContain('compactionRegistry'); + expect(result).toContain('pluginRegistry'); + expect(result).toContain("} from '@dexto/core'"); + }); + + it('should include project metadata', () => { + const result = generateDextoConfigFile({ + projectName: 'my-project', + packageName: 'my-project', + description: 'Test project', + }); + + expect(result).toContain('export const projectConfig = {'); + expect(result).toContain("name: 'my-project'"); + expect(result).toContain("version: '1.0.0'"); + expect(result).toContain("description: 'Test project'"); + }); + + it('should include registerProviders function', () => { + const result = generateDextoConfigFile({ + projectName: 'my-project', + packageName: 'my-project', + description: 'Test project', + }); + + expect(result).toContain('export function registerProviders() {'); + expect(result).toContain('// Example: Register blob storage provider'); + expect(result).toContain('// blobStoreRegistry.register(myBlobProvider)'); + }); + + it('should include initialize and cleanup functions', () => { + const result = generateDextoConfigFile({ + projectName: 'my-project', + packageName: 'my-project', + description: 'Test project', + }); + + expect(result).toContain('export async function initialize() {'); + expect(result).toContain('export async function cleanup() {'); + }); + }); + + describe('generateImageReadme', () => { + it('should generate README for image', () => { + const result = generateImageReadme({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image description', + imageName: 'my-image', + }); + + expect(result).toContain('# my-image'); + expect(result).toContain('Test image description'); + }); + + it('should use image terminology for artifacts', () => { + const result = generateImageReadme({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + imageName: 'my-image', + }); + + expect(result).toContain('A **Dexto image**'); + expect(result).toContain('# Build the image'); + expect(result).toContain('pnpm add my-image'); + }); + + it('should use harness terminology for runtime behavior', () => { + const result = generateImageReadme({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + imageName: 'my-image', + }); + + expect(result).toContain('agent harness packaged as an npm module'); + expect(result).toContain('complete runtime harness'); + expect(result).toContain('The harness provides:'); + }); + + it('should include extends note when baseImage provided', () => { + const result = generateImageReadme({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + imageName: 'my-image', + baseImage: '@dexto/image-local', + }); + + expect(result).toContain('This image extends `@dexto/image-local`'); + }); + + it('should include bundler documentation', () => { + const result = generateImageReadme({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + imageName: 'my-image', + }); + + expect(result).toContain('pnpm run build'); + expect(result).toContain('dexto-bundle build'); + expect(result).toContain('Discovers providers from convention-based folders'); + }); + + it('should include architecture explanation', () => { + const result = generateImageReadme({ + projectName: 'my-image', + packageName: 'my-image', + description: 'Test image', + imageName: 'my-image', + }); + + expect(result).toContain('## Architecture'); + expect(result).toContain('When imported, this image:'); + expect(result).toContain('Auto-registers providers (side effect)'); + }); + }); + + describe('generateExampleTool', () => { + it('should generate example tool with default name', () => { + const result = generateExampleTool(); + + expect(result).toContain("import { z } from 'zod'"); + expect(result).toContain('import type { CustomToolProvider'); + expect(result).toContain('InternalTool'); + expect(result).toContain("type: z.literal('example-tool')"); + expect(result).toContain('export const exampleToolProvider: CustomToolProvider'); + }); + + it('should generate tool with custom name', () => { + const result = generateExampleTool('weather-api'); + + expect(result).toContain("type: z.literal('weather-api')"); + expect(result).toContain('export const weatherApiProvider: CustomToolProvider'); + expect(result).toContain("id: 'weatherApi'"); + }); + + it('should include zod schemas', () => { + const result = generateExampleTool('test-tool'); + + expect(result).toContain('const ConfigSchema = z'); + expect(result).toContain('.object({'); + expect(result).toContain('.strict()'); + expect(result).toContain('configSchema: ConfigSchema'); + }); + + it('should include metadata', () => { + const result = generateExampleTool('test-tool'); + + expect(result).toContain('metadata: {'); + expect(result).toContain('displayName:'); + expect(result).toContain('description:'); + expect(result).toContain('category:'); + }); + + it('should include create function with tool definition', () => { + const result = generateExampleTool('test-tool'); + + expect(result).toContain('create: (config'); + expect(result).toContain('context'); + expect(result).toContain('InternalTool[]'); + expect(result).toContain('const tool: InternalTool = {'); + expect(result).toContain('inputSchema: z.object({'); + expect(result).toContain('execute: async (input'); + }); + }); + + describe('generateAppReadme', () => { + it('should generate README for app', () => { + const result = generateAppReadme({ + projectName: 'my-app', + packageName: 'my-app', + description: 'Test app description', + }); + + expect(result).toContain('# my-app'); + expect(result).toContain('Test app description'); + }); + + it('should include quick start instructions', () => { + const result = generateAppReadme({ + projectName: 'my-app', + packageName: 'my-app', + description: 'Test app', + }); + + expect(result).toContain('## Quick Start'); + expect(result).toContain('pnpm install'); + expect(result).toContain('pnpm start'); + }); + + it('should include image section when using image', () => { + const result = generateAppReadme({ + projectName: 'my-app', + packageName: 'my-app', + description: 'Test app', + imageName: '@dexto/image-local', + }); + + expect(result).toContain('## Image'); + expect(result).toContain('This app uses the `@dexto/image-local` image'); + expect(result).toContain('Pre-configured providers'); + expect(result).toContain('Runtime orchestration'); + }); + + it('should not include image section when not using image', () => { + const result = generateAppReadme({ + projectName: 'my-app', + packageName: 'my-app', + description: 'Test app', + }); + + expect(result).not.toContain('## Image'); + }); + + it('should include project structure', () => { + const result = generateAppReadme({ + projectName: 'my-app', + packageName: 'my-app', + description: 'Test app', + }); + + expect(result).toContain('## Project Structure'); + expect(result).toContain('src/'); + expect(result).toContain('agents/'); + }); + + it('should include configuration section', () => { + const result = generateAppReadme({ + projectName: 'my-app', + packageName: 'my-app', + description: 'Test app', + }); + + expect(result).toContain('## Configuration'); + expect(result).toContain('agents/default.yml'); + }); + + it('should include learn more section', () => { + const result = generateAppReadme({ + projectName: 'my-app', + packageName: 'my-app', + description: 'Test app', + }); + + expect(result).toContain('## Learn More'); + expect(result).toContain('Dexto Documentation'); + }); + }); +}); diff --git a/dexto/packages/cli/src/cli/utils/template-engine.ts b/dexto/packages/cli/src/cli/utils/template-engine.ts new file mode 100644 index 00000000..98817593 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/template-engine.ts @@ -0,0 +1,1115 @@ +/** + * Template Engine for Dexto Project Scaffolding + * + * Provides code generation functions for various project types. + * Uses the image/harness terminology strategy: + * - "Image" for distributable artifacts, packages, composition + * - "Harness" for runtime behavior, what it provides + */ + +interface TemplateContext { + projectName: string; + packageName: string; + description: string; + imageName?: string; + baseImage?: string; + target?: string; + llmProvider?: string; + llmModel?: string; + [key: string]: any; +} + +/** + * Generates src/index.ts for an app using an image + */ +export function generateIndexForImage(context: TemplateContext): string { + return `// Load image environment (Pattern 1: Static Import) +// This auto-registers providers as a side-effect +import '${context.imageName}'; + +// Import from core packages +import { DextoAgent } from '@dexto/core'; +import { loadAgentConfig } from '@dexto/agent-management'; + +async function main() { + console.log('🚀 Starting ${context.projectName}\\n'); + + // Load agent configuration + const config = await loadAgentConfig('./agents/default.yml'); + + // Create agent - providers already registered by image environment + const agent = new DextoAgent(config, './agents/default.yml'); + + await agent.start(); + console.log('✅ Agent started\\n'); + + // Create a session + const session = await agent.createSession(); + + // Example interaction + const response = await agent.run( + 'Hello! What can you help me with?', + undefined, // imageDataInput + undefined, // fileDataInput + session.id // sessionId + ); + + console.log('Agent response:', response); + + // Cleanup + await agent.stop(); +} + +main().catch((error) => { + console.error('Error:', error); + process.exit(1); +}); +`; +} + +/** + * Generates src/index.ts for a web server application using an image + */ +export function generateWebServerIndex(context: TemplateContext): string { + return `// Load image environment (Pattern 1: Static Import) +// This auto-registers providers as a side-effect +import '${context.imageName}'; + +// Import from core packages +import { DextoAgent } from '@dexto/core'; +import { loadAgentConfig } from '@dexto/agent-management'; +import { startDextoServer } from '@dexto/server'; +import { resolve } from 'node:path'; +import { existsSync } from 'node:fs'; + +async function main() { + console.log('🚀 Starting ${context.projectName}\\n'); + + // Load agent configuration + console.log('📝 Loading configuration...'); + const config = await loadAgentConfig('./agents/default.yml'); + console.log('✅ Config loaded\\n'); + + // Create agent + console.log('🤖 Creating agent...'); + const agent = new DextoAgent(config, './agents/default.yml'); + console.log('✅ Agent created\\n'); + + // Start the server + console.log('🌐 Starting Dexto server...'); + + const webRoot = resolve(process.cwd(), 'app'); + + if (!existsSync(webRoot)) { + console.error(\`❌ Error: Web root not found at \${webRoot}\`); + console.error(' Make sure the app/ directory exists'); + process.exit(1); + } + + console.log(\`📁 Serving static files from: \${webRoot}\`); + + const { stop } = await startDextoServer(agent, { + port: 3000, + webRoot, + agentCard: { + name: '${context.projectName}', + description: '${context.description}', + }, + }); + + console.log('\\n✅ Server is running!\\n'); + console.log('🌐 Open your browser:'); + console.log(' http://localhost:3000\\n'); + console.log('📚 Available endpoints:'); + console.log(' • Web UI: http://localhost:3000'); + console.log(' • REST API: http://localhost:3000/api/*'); + console.log(' • Health Check: http://localhost:3000/health'); + console.log(' • OpenAPI Spec: http://localhost:3000/openapi.json'); + console.log(' • Agent Card: http://localhost:3000/.well-known/agent-card.json\\n'); + + console.log('Press Ctrl+C to stop the server...\\n'); + + // Handle graceful shutdown + const shutdown = async () => { + console.log('\\n🛑 Shutting down...'); + await stop(); + console.log('✅ Server stopped\\n'); + process.exit(0); + }; + + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); +} + +main().catch((error) => { + console.error('Error:', error); + process.exit(1); +}); +`; +} + +/** + * Generates HTML for web app + */ +export function generateWebAppHTML(projectName: string): string { + return `<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>${projectName} + + + +
+
+

🤖 ${projectName}

+

AI-Powered Assistant

+
+ Initializing... +
+
+ +
+
+ +
+ + +
+
+
+ + + + +`; +} + +/** + * Generates JavaScript for web app + */ +export function generateWebAppJS(): string { + return `// Dexto Chat - Frontend +// Use relative URL so it works regardless of hostname/port +const API_BASE = '/api'; + +let sessionId = null; +let isProcessing = false; + +// DOM elements +const messagesContainer = document.getElementById('messages'); +const messageInput = document.getElementById('message-input'); +const sendButton = document.getElementById('send-button'); +const sessionStatus = document.getElementById('session-status'); + +// Initialize the app +async function init() { + try { + sessionStatus.textContent = 'Creating session...'; + + // Create a new session + const response = await fetch(\`\${API_BASE}/sessions\`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }); + + if (!response.ok) { + throw new Error(\`Failed to create session: \${response.statusText}\`); + } + + const data = await response.json(); + sessionId = data.session.id; + + sessionStatus.textContent = \`Session: \${sessionId.substring(0, 12)}...\`; + + // Enable input + messageInput.disabled = false; + sendButton.disabled = false; + messageInput.focus(); + + // Add welcome message + addMessage('assistant', "Hello! I'm your Dexto assistant. How can I help you today?"); + } catch (error) { + console.error('Initialization error:', error); + const errorMsg = error.message || String(error); + showError(\`Failed to initialize: \${errorMsg}\`); + sessionStatus.textContent = \`Error: \${errorMsg}\`; + + // Log more details for debugging + console.error('Full error details:', { + error, + apiBase: API_BASE, + url: \`\${API_BASE}/sessions\`, + }); + } +} + +// Send a message to the agent +async function sendMessage() { + const text = messageInput.value.trim(); + if (!text || isProcessing) return; + + // Add user message to UI + addMessage('user', text); + messageInput.value = ''; + messageInput.disabled = true; + sendButton.disabled = true; + isProcessing = true; + + // Add loading indicator + const loadingId = addMessage('assistant', 'Thinking...', true); + + try { + const response = await fetch(\`\${API_BASE}/message-sync\`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + content: text, + sessionId: sessionId, + }), + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error?.message || response.statusText); + } + + const data = await response.json(); + + // Remove loading indicator + removeMessage(loadingId); + + // Add agent response + addMessage('assistant', data.response); + + // Show token usage in console + if (data.tokenUsage) { + console.log('Token usage:', data.tokenUsage); + } + } catch (error) { + console.error('Send message error:', error); + removeMessage(loadingId); + showError(\`Failed to send message: \${error.message}\`); + } finally { + isProcessing = false; + messageInput.disabled = false; + sendButton.disabled = false; + messageInput.focus(); + } +} + +// Add a message to the chat UI +function addMessage(role, content, isLoading = false) { + const messageId = \`msg-\${Date.now()}-\${Math.random()}\`; + const messageEl = document.createElement('div'); + messageEl.className = \`message \${role}\`; + messageEl.id = messageId; + + const avatar = document.createElement('div'); + avatar.className = 'message-avatar'; + avatar.textContent = role === 'user' ? '👤' : '🤖'; + + const contentEl = document.createElement('div'); + contentEl.className = \`message-content \${isLoading ? 'loading' : ''}\`; + contentEl.textContent = content; + + messageEl.appendChild(avatar); + messageEl.appendChild(contentEl); + + messagesContainer.appendChild(messageEl); + messagesContainer.scrollTop = messagesContainer.scrollHeight; + + return messageId; +} + +// Remove a message from the UI +function removeMessage(messageId) { + const messageEl = document.getElementById(messageId); + if (messageEl) { + messageEl.remove(); + } +} + +// Show an error message +function showError(message) { + const errorEl = document.createElement('div'); + errorEl.className = 'error-message'; + errorEl.textContent = \`Error: \${message}\`; + messagesContainer.appendChild(errorEl); + messagesContainer.scrollTop = messagesContainer.scrollHeight; + + // Auto-remove after 5 seconds + setTimeout(() => errorEl.remove(), 5000); +} + +// Event listeners +sendButton.addEventListener('click', sendMessage); + +messageInput.addEventListener('keypress', (e) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + sendMessage(); + } +}); + +// Initialize when page loads +document.addEventListener('DOMContentLoaded', init); +`; +} + +/** + * Generates CSS for web app + */ +export function generateWebAppCSS(): string { + return `* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + min-height: 100vh; + display: flex; + align-items: center; + justify-content: center; + padding: 20px; +} + +.container { + width: 100%; + max-width: 800px; + background: white; + border-radius: 16px; + box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3); + overflow: hidden; + display: flex; + flex-direction: column; + height: 90vh; + max-height: 700px; +} + +header { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; + padding: 24px; + text-align: center; +} + +header h1 { + font-size: 28px; + margin-bottom: 8px; +} + +.subtitle { + font-size: 14px; + opacity: 0.9; + margin-bottom: 12px; +} + +.session-info { + font-size: 12px; + opacity: 0.8; + font-family: 'Monaco', 'Courier New', monospace; +} + +.chat-container { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.messages { + flex: 1; + overflow-y: auto; + padding: 24px; + display: flex; + flex-direction: column; + gap: 16px; +} + +.message { + display: flex; + gap: 12px; + animation: slideIn 0.3s ease-out; +} + +@keyframes slideIn { + from { + opacity: 0; + transform: translateY(10px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +.message.user { + flex-direction: row-reverse; +} + +.message-avatar { + width: 40px; + height: 40px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + font-size: 20px; + flex-shrink: 0; +} + +.message.user .message-avatar { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); +} + +.message.assistant .message-avatar { + background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%); +} + +.message-content { + max-width: 70%; + padding: 12px 16px; + border-radius: 12px; + line-height: 1.5; +} + +.message.user .message-content { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; + border-bottom-right-radius: 4px; +} + +.message.assistant .message-content { + background: #f5f5f5; + color: #333; + border-bottom-left-radius: 4px; +} + +.message-content.loading { + font-style: italic; + opacity: 0.7; +} + +.input-container { + display: flex; + gap: 12px; + padding: 20px 24px; + border-top: 1px solid #e5e5e5; + background: white; +} + +#message-input { + flex: 1; + padding: 12px 16px; + border: 2px solid #e5e5e5; + border-radius: 24px; + font-size: 15px; + outline: none; + transition: border-color 0.2s; +} + +#message-input:focus { + border-color: #667eea; +} + +#message-input:disabled { + background: #f5f5f5; + cursor: not-allowed; +} + +#send-button { + padding: 12px 28px; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; + border: none; + border-radius: 24px; + font-size: 15px; + font-weight: 600; + cursor: pointer; + transition: transform 0.2s, opacity 0.2s; +} + +#send-button:hover:not(:disabled) { + transform: translateY(-2px); +} + +#send-button:active:not(:disabled) { + transform: translateY(0); +} + +#send-button:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.error-message { + background: #fee; + color: #c33; + padding: 12px 16px; + border-radius: 8px; + margin: 16px 24px; + border-left: 4px solid #c33; +} + +/* Scrollbar styling */ +.messages::-webkit-scrollbar { + width: 8px; +} + +.messages::-webkit-scrollbar-track { + background: #f1f1f1; +} + +.messages::-webkit-scrollbar-thumb { + background: #888; + border-radius: 4px; +} + +.messages::-webkit-scrollbar-thumb:hover { + background: #555; +} +`; +} + +/** + * Generates dexto.image.ts file for an image project + */ +export function generateDextoImageFile(context: TemplateContext): string { + const extendsField = context.baseImage ? ` extends: '${context.baseImage}',\n` : ''; + + return `import { defineImage } from '@dexto/core'; + +export default defineImage({ + name: '${context.imageName || context.projectName}', + version: '1.0.0', + description: '${context.description}', + target: '${context.target || 'local-development'}', +${extendsField} + // Providers are AUTO-DISCOVERED from convention-based folders: + // tools/ - Custom tool providers + // blob-store/ - Blob storage providers + // compression/ - Compression strategy providers + // plugins/ - Plugin providers + // + // Each provider must export from an index.ts file in its folder. + // The bundler will automatically register them when the image is imported. + + providers: { + // Manual registration for built-in core providers + // (These come from core, not from our providers/ folder) + // TODO: This is a hack to get the local blob store provider to work. Should be auto-registered or dealt with in a better way. + blobStore: { + register: async () => { + const { localBlobStoreProvider, inMemoryBlobStoreProvider } = await import( + '@dexto/core' + ); + const { blobStoreRegistry } = await import('@dexto/core'); + + blobStoreRegistry.register(localBlobStoreProvider); + blobStoreRegistry.register(inMemoryBlobStoreProvider); + + console.log('✓ Registered core blob storage providers: local, in-memory'); + }, + }, + }, + + defaults: { + storage: { + blob: { + type: 'local', + storePath: './data/blobs', + }, + database: { + type: 'sqlite', + path: './data/agent.db', + }, + cache: { + type: 'in-memory', + }, + }, + logging: { + level: 'info', + fileLogging: true, + }, + }, + + constraints: ['filesystem-required', 'offline-capable'], +}); +`; +} + +/** + * Generates dexto.config.ts file for manual registration projects + */ +export function generateDextoConfigFile(context: TemplateContext): string { + return `/** + * ${context.projectName} - Provider Registration + * + * This file registers all custom providers before agent initialization. + * This is the manual registration approach - for most use cases, consider + * using Dexto images instead (see: dexto create-image). + */ + +import { + blobStoreRegistry, + customToolRegistry, + compactionRegistry, + pluginRegistry, +} from '@dexto/core'; + +/** + * Project metadata + */ +export const projectConfig = { + name: '${context.projectName}', + version: '1.0.0', + description: '${context.description}', +}; + +/** + * Register all custom providers + * + * This function is called at application startup before loading agent configs. + * Register your providers here by importing them and calling the appropriate + * registry.register() method. + */ +export function registerProviders() { + // Example: Register blob storage provider + // import { myBlobProvider } from './storage/my-blob.js'; + // blobStoreRegistry.register(myBlobProvider); + + // Example: Register custom tool + // import { myToolProvider } from './tools/my-tool.js'; + // customToolRegistry.register(myToolProvider); + + // Example: Register plugin + // import { myPluginProvider } from './plugins/my-plugin.js'; + // pluginRegistry.register(myPluginProvider); + + console.log(\`✓ Registered providers for \${projectConfig.name}\`); +} + +/** + * Optional: Project-wide initialization logic + * Use this for setting up monitoring, analytics, error tracking, etc. + */ +export async function initialize() { + console.log(\`✓ Initialized \${projectConfig.name} v\${projectConfig.version}\`); +} + +/** + * Optional: Cleanup logic + * Called when the application shuts down + */ +export async function cleanup() { + console.log(\`✓ Cleaned up \${projectConfig.name}\`); +} +`; +} + +/** + * Generates README.md for an image project + */ +export function generateImageReadme(context: TemplateContext): string { + const imageName = context.imageName || context.projectName; + const extendsNote = context.baseImage + ? `\n\nThis image extends \`${context.baseImage}\`, inheriting its providers and adding custom ones.\n` + : ''; + + return `# ${imageName} + +${context.description}${extendsNote} + +## What is this? + +A **Dexto image** - a pre-configured agent harness packaged as an npm module. +Install it, import it, and you have a complete runtime harness ready to use. + +## What's Included + +The harness provides: +- ✅ Pre-registered providers (auto-discovered from convention-based folders) +- ✅ Runtime orchestration +- ✅ Context management +- ✅ Default configurations + +## Quick Start + +\`\`\`bash +# Build the image +pnpm run build + +# Use in your app +pnpm add ${imageName} +\`\`\` + +## Usage + +\`\`\`typescript +import { createAgent } from '${imageName}'; +import { loadAgentConfig } from '@dexto/agent-management'; + +const config = await loadAgentConfig('./agents/default.yml'); + +// Import creates the harness (providers auto-registered) +const agent = createAgent(config); + +// The harness handles everything +await agent.start(); +\`\`\` + +## Adding Providers + +Add your custom providers to convention-based folders: +- \`tools/\` - Custom tool providers +- \`blob-store/\` - Blob storage providers +- \`compression/\` - Compression strategies +- \`plugins/\` - Plugin providers + +**Convention:** Each provider lives in its own folder with an \`index.ts\` file. + +Example: +\`\`\` +tools/ + my-tool/ + index.ts # Provider implementation (auto-discovered) + helpers.ts # Optional helper functions + types.ts # Optional type definitions +\`\`\` + +## Building + +\`\`\`bash +pnpm run build +\`\`\` + +This runs \`dexto-bundle build\` which: +1. Discovers providers from convention-based folders +2. Generates \`dist/index.js\` with side-effect registration +3. Exports \`createAgent()\` factory function + +## Architecture + +When imported, this image: +1. Auto-registers providers (side effect) +2. Exposes harness factory (\`createAgent\`) +3. Re-exports registries for runtime customization + +The resulting harness manages your agent's runtime, including provider lifecycle, +context management, and tool orchestration. + +## Publishing + +\`\`\`bash +npm publish +\`\`\` + +Users can then: +\`\`\`bash +pnpm add ${imageName} +\`\`\` + +## Learn More + +- [Dexto Images Guide](https://docs.dexto.ai/docs/guides/images) +- [Provider Development](https://docs.dexto.ai/docs/guides/providers) +- [Bundler Documentation](https://docs.dexto.ai/docs/tools/bundler) +`; +} + +/** + * Generates an example custom tool provider + */ +export function generateExampleTool(toolName: string = 'example-tool'): string { + // Convert kebab-case to camelCase for provider name + const providerName = toolName.replace(/-([a-z])/g, (_, char) => char.toUpperCase()); + return `import { z } from 'zod'; +import type { CustomToolProvider, InternalTool, ToolCreationContext } from '@dexto/core'; + +const ConfigSchema = z + .object({ + type: z.literal('${toolName}'), + // Add your configuration options here + }) + .strict(); + +type ${providerName.charAt(0).toUpperCase() + providerName.slice(1)}Config = z.output; + +/** + * Example custom tool provider + * + * This demonstrates how to create a custom tool that can be used by the agent. + * The tool is auto-discovered by the bundler when placed in the tools/ folder. + */ +export const ${providerName}Provider: CustomToolProvider<'${toolName}', ${providerName.charAt(0).toUpperCase() + providerName.slice(1)}Config> = { + type: '${toolName}', + configSchema: ConfigSchema, + + create: (config: ${providerName.charAt(0).toUpperCase() + providerName.slice(1)}Config, context: ToolCreationContext): InternalTool[] => { + // Create and return tools + const tool: InternalTool = { + id: '${providerName}', + description: 'An example custom tool that demonstrates the tool provider pattern', + inputSchema: z.object({ + input: z.string().describe('Input text to process'), + }), + + execute: async (input: unknown) => { + const { input: inputText } = input as { input: string }; + context.logger.info(\`Example tool called with: \${inputText}\`); + + // Your tool logic here + return { + result: \`Processed: \${inputText}\`, + }; + }, + }; + + return [tool]; + }, + + metadata: { + displayName: 'Example Tool', + description: 'Example custom tool provider', + category: 'utilities', + }, +}; +`; +} + +/** + * Generates README for an app project + */ +export function generateAppReadme(context: TemplateContext): string { + const usingImage = context.imageName; + const imageSection = usingImage + ? `\n## Image + +This app uses the \`${context.imageName}\` image, which provides a complete agent harness with: +- Pre-configured providers +- Runtime orchestration +- Context management + +The harness is automatically initialized when you import the image.\n` + : ''; + + return `# ${context.projectName} + +${context.description}${imageSection} + +## Quick Start + +\`\`\`bash +# Install dependencies +pnpm install + +# Set up environment +cp .env.example .env +# Edit .env with your API keys + +# Run +pnpm start +\`\`\` + +## Project Structure + +\`\`\` +${context.projectName}/ +├── src/ +│ └── index.ts # Entry point +├── agents/ +│ └── default.yml # Agent configuration +├── .env # Environment variables (gitignored) +├── package.json +└── tsconfig.json +\`\`\` + +## Configuration + +Edit \`agents/default.yml\` to configure: +- System prompts +- LLM provider and model +- MCP servers +- Internal tools +- Custom tools + +## Learn More + +- [Dexto Documentation](https://docs.dexto.ai) +- [Agent Configuration Guide](https://docs.dexto.ai/docs/guides/configuration) +- [Using Images](https://docs.dexto.ai/docs/guides/images) +`; +} + +/** + * Generates auto-discovery script for from-core mode + */ +export function generateDiscoveryScript(): string { + return `#!/usr/bin/env tsx +/** + * Provider Auto-Discovery Script + * + * Scans conventional folders (tools/, blob-store/, compression/, plugins/) + * and generates src/providers.ts with import + registration statements. + */ + +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const projectRoot = path.resolve(__dirname, '..'); + +interface ProviderInfo { + category: 'customTools' | 'blobStore' | 'compression' | 'plugins'; + folderName: string; + path: string; + registryName: string; +} + +const PROVIDER_CATEGORIES = [ + { folder: 'tools', category: 'customTools' as const, registry: 'customToolRegistry' }, + { folder: 'blob-store', category: 'blobStore' as const, registry: 'blobStoreRegistry' }, + { folder: 'compaction', category: 'compaction' as const, registry: 'compactionRegistry' }, + { folder: 'plugins', category: 'plugins' as const, registry: 'pluginRegistry' }, +]; + +async function discoverProviders(): Promise { + const providers: ProviderInfo[] = []; + + for (const { folder, category, registry } of PROVIDER_CATEGORIES) { + const folderPath = path.join(projectRoot, folder); + + try { + const entries = await fs.readdir(folderPath, { withFileTypes: true }); + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (entry.name.startsWith('.')) continue; + + // Check if provider has index.ts + const indexPath = path.join(folderPath, entry.name, 'index.ts'); + try { + await fs.access(indexPath); + providers.push({ + category, + folderName: entry.name, + path: \`../\${folder}/\${entry.name}/index.js\`, + registryName: registry, + }); + } catch { + // No index.ts found, skip + } + } + } catch { + // Folder doesn't exist or can't be read, skip + } + } + + return providers; +} + +function generateProvidersFile(providers: ProviderInfo[]): string { + // Helper to convert kebab-case to camelCase for valid JS identifiers + const toCamelCase = (str: string): string => { + return str.replace(/-([a-z])/g, (_, char) => char.toUpperCase()); + }; + + const imports: string[] = []; + const registrations: string[] = []; + const registries = new Set(); + + providers.forEach((provider, index) => { + const varName = \`provider\${index}\`; + const providerName = \`\${toCamelCase(provider.folderName)}Provider\`; + imports.push(\`import { \${providerName} as \${varName} } from '\${provider.path}';\`); + registrations.push(\` \${provider.registryName}.register(\${varName});\`); + registries.add(provider.registryName); + }); + + const registryImports = Array.from(registries).join(', '); + + return \`// AUTO-GENERATED - DO NOT EDIT +// This file is generated by scripts/discover-providers.ts +// Run 'pnpm run discover' to regenerate + +import { \${registryImports} } from '@dexto/core'; +\${imports.join('\\n')} + +/** + * Register all discovered providers + * Called automatically when this module is imported + */ +export function registerProviders(): void { +\${registrations.join('\\n')} +} + +// Auto-register on import +registerProviders(); + +console.log('✓ Registered \${providers.length} provider(s)'); +\`; +} + +function generateEntryPoint(): string { + return \`// AUTO-GENERATED - DO NOT EDIT +// This file is the build entry point that wires everything together +// Run 'pnpm run discover' to regenerate + +// Register providers first +import './_providers.js'; + +// Then run the user's app +import './index.js'; +\`; +} + +async function main() { + console.log('🔍 Discovering providers...\\n'); + + const providers = await discoverProviders(); + + if (providers.length === 0) { + console.log('⚠️ No providers found'); + console.log(' Add providers to: tools/, blob-store/, compression/, or plugins/\\n'); + } else { + console.log(\`✅ Found \${providers.length} provider(s):\`); + providers.forEach(p => { + console.log(\` • \${p.category}/\${p.folderName}\`); + }); + console.log(); + } + + // Generate provider registrations + const providersPath = path.join(projectRoot, 'src', '_providers.ts'); + const providersContent = generateProvidersFile(providers); + await fs.writeFile(providersPath, providersContent, 'utf-8'); + console.log(\`📝 Generated: src/_providers.ts\`); + + // Generate build entry point + const entryPath = path.join(projectRoot, 'src', '_entry.ts'); + const entryContent = generateEntryPoint(); + await fs.writeFile(entryPath, entryContent, 'utf-8'); + console.log(\`📝 Generated: src/_entry.ts\`); + + console.log(); +} + +main().catch(error => { + console.error('❌ Discovery failed:', error); + process.exit(1); +}); +`; +} diff --git a/dexto/packages/cli/src/cli/utils/version-check.test.ts b/dexto/packages/cli/src/cli/utils/version-check.test.ts new file mode 100644 index 00000000..176ebd71 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/version-check.test.ts @@ -0,0 +1,228 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { promises as fs } from 'fs'; +import { checkForUpdates, displayUpdateNotification } from './version-check.js'; + +// Mock fs module +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + promises: { + readFile: vi.fn(), + writeFile: vi.fn(), + mkdir: vi.fn(), + }, + }; +}); + +// Mock fetch +const mockFetch = vi.fn(); +global.fetch = mockFetch; + +// Mock getDextoGlobalPath +vi.mock('@dexto/agent-management', () => ({ + getDextoGlobalPath: vi.fn((_type: string, filename?: string) => + filename ? `/mock/.dexto/cache/${filename}` : '/mock/.dexto/cache' + ), +})); + +describe('version-check', () => { + beforeEach(() => { + vi.clearAllMocks(); + delete process.env.DEXTO_NO_UPDATE_CHECK; + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe('checkForUpdates', () => { + it('returns null when DEXTO_NO_UPDATE_CHECK is set', async () => { + process.env.DEXTO_NO_UPDATE_CHECK = 'true'; + + const result = await checkForUpdates('1.0.0'); + + expect(result).toBeNull(); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('returns update info when newer version available from npm', async () => { + // No cache - force fetch + vi.mocked(fs.readFile).mockRejectedValue(new Error('ENOENT')); + vi.mocked(fs.writeFile).mockResolvedValue(); + vi.mocked(fs.mkdir).mockResolvedValue(undefined); + + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ version: '2.0.0' }), + }); + + const result = await checkForUpdates('1.0.0'); + + expect(result).toEqual({ + current: '1.0.0', + latest: '2.0.0', + updateCommand: 'npm i -g dexto', + }); + }); + + it('returns null when current version matches latest', async () => { + vi.mocked(fs.readFile).mockRejectedValue(new Error('ENOENT')); + vi.mocked(fs.writeFile).mockResolvedValue(); + vi.mocked(fs.mkdir).mockResolvedValue(undefined); + + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ version: '1.0.0' }), + }); + + const result = await checkForUpdates('1.0.0'); + + expect(result).toBeNull(); + }); + + it('returns null when current version is newer than npm', async () => { + vi.mocked(fs.readFile).mockRejectedValue(new Error('ENOENT')); + vi.mocked(fs.writeFile).mockResolvedValue(); + vi.mocked(fs.mkdir).mockResolvedValue(undefined); + + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ version: '1.0.0' }), + }); + + const result = await checkForUpdates('2.0.0'); + + expect(result).toBeNull(); + }); + + it('uses cached result when cache is fresh', async () => { + const freshCache = { + lastCheck: Date.now() - 1000, // 1 second ago + latestVersion: '2.0.0', + currentVersion: '1.0.0', + }; + + vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(freshCache)); + + const result = await checkForUpdates('1.0.0'); + + expect(result).toEqual({ + current: '1.0.0', + latest: '2.0.0', + updateCommand: 'npm i -g dexto', + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('fetches new data when cache is expired', async () => { + const expiredCache = { + lastCheck: Date.now() - 25 * 60 * 60 * 1000, // 25 hours ago + latestVersion: '1.5.0', + currentVersion: '1.0.0', + }; + + vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(expiredCache)); + vi.mocked(fs.writeFile).mockResolvedValue(); + vi.mocked(fs.mkdir).mockResolvedValue(undefined); + + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ version: '2.0.0' }), + }); + + const result = await checkForUpdates('1.0.0'); + + expect(result).toEqual({ + current: '1.0.0', + latest: '2.0.0', + updateCommand: 'npm i -g dexto', + }); + expect(mockFetch).toHaveBeenCalled(); + }); + + it('returns null on network error', async () => { + vi.mocked(fs.readFile).mockRejectedValue(new Error('ENOENT')); + + mockFetch.mockRejectedValue(new Error('Network error')); + + const result = await checkForUpdates('1.0.0'); + + expect(result).toBeNull(); + }); + + it('returns null on non-ok response', async () => { + vi.mocked(fs.readFile).mockRejectedValue(new Error('ENOENT')); + + mockFetch.mockResolvedValue({ + ok: false, + status: 404, + }); + + const result = await checkForUpdates('1.0.0'); + + expect(result).toBeNull(); + }); + }); + + describe('semver comparison (via checkForUpdates)', () => { + beforeEach(() => { + vi.mocked(fs.readFile).mockRejectedValue(new Error('ENOENT')); + vi.mocked(fs.writeFile).mockResolvedValue(); + vi.mocked(fs.mkdir).mockResolvedValue(undefined); + }); + + it('correctly identifies major version updates', async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ version: '2.0.0' }), + }); + + const result = await checkForUpdates('1.9.9'); + expect(result?.latest).toBe('2.0.0'); + }); + + it('correctly identifies minor version updates', async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ version: '1.2.0' }), + }); + + const result = await checkForUpdates('1.1.9'); + expect(result?.latest).toBe('1.2.0'); + }); + + it('correctly identifies patch version updates', async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ version: '1.0.2' }), + }); + + const result = await checkForUpdates('1.0.1'); + expect(result?.latest).toBe('1.0.2'); + }); + + it('handles versions with v prefix', async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ version: 'v2.0.0' }), + }); + + const result = await checkForUpdates('v1.0.0'); + expect(result?.latest).toBe('v2.0.0'); + }); + }); + + describe('displayUpdateNotification', () => { + it('does not throw', () => { + // Just verify it doesn't throw - output goes to console + expect(() => + displayUpdateNotification({ + current: '1.0.0', + latest: '2.0.0', + updateCommand: 'npm i -g dexto', + }) + ).not.toThrow(); + }); + }); +}); diff --git a/dexto/packages/cli/src/cli/utils/version-check.ts b/dexto/packages/cli/src/cli/utils/version-check.ts new file mode 100644 index 00000000..81fd83f3 --- /dev/null +++ b/dexto/packages/cli/src/cli/utils/version-check.ts @@ -0,0 +1,239 @@ +// packages/cli/src/cli/utils/version-check.ts + +import { promises as fs } from 'fs'; +import path from 'path'; +import chalk from 'chalk'; +import boxen from 'boxen'; +import { logger } from '@dexto/core'; +import { getDextoGlobalPath } from '@dexto/agent-management'; + +/** + * Version cache stored in ~/.dexto/version-check.json + */ +interface VersionCache { + lastCheck: number; // timestamp in ms + latestVersion: string; + currentVersion: string; +} + +/** + * Update info returned when a newer version is available + */ +export interface UpdateInfo { + current: string; + latest: string; + updateCommand: string; +} + +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours +const NPM_REGISTRY_URL = 'https://registry.npmjs.org/dexto/latest'; +const CACHE_FILE_PATH = getDextoGlobalPath('cache', 'version-check.json'); + +/** + * Compare two semver versions. + * Returns: + * - negative if v1 < v2 + * - 0 if v1 === v2 + * - positive if v1 > v2 + * + * Note: Pre-release versions (e.g., 1.0.0-beta.1) are not fully supported. + * The comparison strips pre-release suffixes, so 1.0.0 and 1.0.0-beta.1 + * would be considered equal. This is acceptable for update notifications + * since we don't publish pre-release versions to npm's latest tag. + */ +function compareSemver(v1: string, v2: string): number { + const parse = (v: string) => { + // Strip leading 'v' if present and split on '.' + const cleaned = v.replace(/^v/, ''); + const parts = cleaned.split('.').map((p) => parseInt(p, 10) || 0); + // Pad to 3 parts + while (parts.length < 3) parts.push(0); + return parts; + }; + + const p1 = parse(v1); + const p2 = parse(v2); + + for (let i = 0; i < 3; i++) { + const v1Part = p1[i] ?? 0; + const v2Part = p2[i] ?? 0; + if (v1Part !== v2Part) { + return v1Part - v2Part; + } + } + return 0; +} + +/** + * Load cached version info from disk + */ +async function loadCache(): Promise { + try { + const content = await fs.readFile(CACHE_FILE_PATH, 'utf-8'); + return JSON.parse(content) as VersionCache; + } catch { + return null; + } +} + +/** + * Save version cache to disk + */ +async function saveCache(cache: VersionCache): Promise { + try { + await fs.mkdir(path.dirname(CACHE_FILE_PATH), { recursive: true }); + await fs.writeFile(CACHE_FILE_PATH, JSON.stringify(cache, null, 2)); + } catch (error) { + // Non-critical - just log and continue + logger.debug( + `Failed to save version cache: ${error instanceof Error ? error.message : String(error)}` + ); + } +} + +/** + * Fetch latest version from npm registry + */ +async function fetchLatestVersion(): Promise { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 5000); // 5 second timeout + + try { + const response = await fetch(NPM_REGISTRY_URL, { + signal: controller.signal, + headers: { + Accept: 'application/json', + }, + }); + + if (!response.ok) { + logger.debug(`npm registry returned status ${response.status}`); + return null; + } + + const data = (await response.json()) as { version?: string }; + return data.version || null; + } catch (error) { + // Network errors, timeouts, etc. - silent fail + logger.debug( + `Failed to fetch latest version: ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } finally { + clearTimeout(timeoutId); + } +} + +/** + * Check for updates (non-blocking, cached) + * + * @param currentVersion The current installed version + * @returns UpdateInfo if a newer version is available, null otherwise + * + * This function is designed to be called at CLI startup. It: + * - Respects DEXTO_NO_UPDATE_CHECK=true to disable checks + * - Uses a 24-hour cache to avoid hammering npm + * - Fails silently on network errors + * - Never blocks startup for more than 5 seconds + * + * @example + * ```typescript + * const updateInfo = await checkForUpdates('1.5.4'); + * if (updateInfo) { + * displayUpdateNotification(updateInfo); + * } + * ``` + */ +export async function checkForUpdates(currentVersion: string): Promise { + // Check if update checks are disabled + if (process.env.DEXTO_NO_UPDATE_CHECK === 'true') { + logger.debug('Version check disabled via DEXTO_NO_UPDATE_CHECK'); + return null; + } + + try { + const now = Date.now(); + const cache = await loadCache(); + + // Check if cache is valid + if (cache && cache.currentVersion === currentVersion) { + const cacheAge = now - cache.lastCheck; + if (cacheAge < CACHE_TTL_MS) { + logger.debug( + `Using cached version info (age: ${Math.round(cacheAge / 1000 / 60)} minutes)` + ); + // Return cached result if newer version exists + if (compareSemver(cache.latestVersion, currentVersion) > 0) { + return { + current: currentVersion, + latest: cache.latestVersion, + updateCommand: 'npm i -g dexto', + }; + } + return null; + } + } + + // Cache expired or invalid - fetch from npm + logger.debug('Fetching latest version from npm registry'); + const latestVersion = await fetchLatestVersion(); + + if (!latestVersion) { + return null; + } + + // Update cache + const newCache: VersionCache = { + lastCheck: now, + latestVersion, + currentVersion, + }; + await saveCache(newCache); + + // Check if update is available + if (compareSemver(latestVersion, currentVersion) > 0) { + return { + current: currentVersion, + latest: latestVersion, + updateCommand: 'npm i -g dexto', + }; + } + + return null; + } catch (error) { + // Never fail the CLI startup due to version check errors + logger.debug( + `Version check error: ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } +} + +/** + * Display update notification in a styled box + * + * @param updateInfo Update information to display + * + * @example + * ```typescript + * displayUpdateNotification({ + * current: '1.5.4', + * latest: '1.6.0', + * updateCommand: 'npm i -g dexto' + * }); + * ``` + */ +export function displayUpdateNotification(updateInfo: UpdateInfo): void { + const message = + `Update available: ${chalk.gray(updateInfo.current)} ${chalk.gray('→')} ${chalk.green(updateInfo.latest)}\n` + + `Run: ${chalk.cyan(updateInfo.updateCommand)}`; + + console.log( + boxen(message, { + padding: 1, + margin: { top: 1, bottom: 1, left: 0, right: 0 }, + borderColor: 'yellow', + borderStyle: 'round', + }) + ); +} diff --git a/dexto/packages/cli/src/config/cli-overrides.test.ts b/dexto/packages/cli/src/config/cli-overrides.test.ts new file mode 100644 index 00000000..15fc8484 --- /dev/null +++ b/dexto/packages/cli/src/config/cli-overrides.test.ts @@ -0,0 +1,234 @@ +import { describe, test, expect } from 'vitest'; +import { + applyCLIOverrides, + applyUserPreferences, + type CLIConfigOverrides, +} from './cli-overrides.js'; +import type { AgentConfig } from '@dexto/core'; +// Note: applyUserPreferences accepts Partial since it only uses the llm field + +function clone(obj: T): T { + return JSON.parse(JSON.stringify(obj)); +} + +describe('CLI Overrides', () => { + const baseConfig: AgentConfig = { + systemPrompt: 'hi', + mcpServers: { + test: { + type: 'stdio', + command: 'node', + args: ['agent-server.js'], + }, + }, + llm: { + provider: 'openai', + model: 'gpt-5', + apiKey: 'file-api-key', + }, + toolConfirmation: { + mode: 'manual', + timeout: 120000, + allowedToolsStorage: 'storage', + }, + }; + + test('applies CLI overrides correctly', () => { + const cliOverrides: CLIConfigOverrides = { + model: 'claude-sonnet-4-5-20250929', + provider: 'anthropic', + apiKey: 'cli-api-key', + }; + + const result = applyCLIOverrides(clone(baseConfig), cliOverrides); + + expect(result.llm.model).toBe('claude-sonnet-4-5-20250929'); + expect(result.llm.provider).toBe('anthropic'); + expect(result.llm.apiKey).toBe('cli-api-key'); + }); + + test('applies partial CLI overrides', () => { + const cliOverrides: CLIConfigOverrides = { + model: 'gpt-5-mini', + // Only override model, leave others unchanged + }; + + const result = applyCLIOverrides(clone(baseConfig), cliOverrides); + + expect(result.llm.model).toBe('gpt-5-mini'); // Overridden + expect(result.llm.provider).toBe('openai'); // Original + expect(result.llm.apiKey).toBe('file-api-key'); // Original + }); + + test('returns original config when no overrides provided', () => { + const result = applyCLIOverrides(clone(baseConfig), undefined); + + expect(result).toEqual(baseConfig); // Should return baseConfig as-is + expect(result).not.toBe(baseConfig); // Should be a copy + }); + + test('returns original config when empty overrides provided', () => { + const result = applyCLIOverrides(clone(baseConfig), {}); + + expect(result).toEqual(baseConfig); // Should return baseConfig as-is + expect(result).not.toBe(baseConfig); // Should be a copy + }); + + test('does not mutate original config', () => { + const originalConfig = clone(baseConfig); + const cliOverrides: CLIConfigOverrides = { + model: 'gpt-5-mini', + provider: 'openai', + }; + + applyCLIOverrides(originalConfig, cliOverrides); + + // Original should be unchanged + expect(originalConfig.llm.model).toBe('gpt-5'); + expect(originalConfig.llm.provider).toBe('openai'); + }); + + test('preserves all non-LLM config fields', () => { + const cliOverrides: CLIConfigOverrides = { + model: 'gpt-5-mini', + }; + + const result = applyCLIOverrides(clone(baseConfig), cliOverrides); + + // Non-LLM fields should be preserved as-is (no schema transformation yet) + expect(result.systemPrompt).toBe('hi'); // Raw value preserved + expect(result.mcpServers?.test?.type).toBe('stdio'); + if (result.mcpServers?.test?.type === 'stdio') { + expect(result.mcpServers.test.command).toBe('node'); + expect(result.mcpServers.test.args).toEqual(['agent-server.js']); + } + expect(result.toolConfirmation?.timeout).toBe(120000); + expect(result.toolConfirmation?.allowedToolsStorage).toBe('storage'); + }); + + test('handles undefined values in overrides gracefully', () => { + const cliOverrides: CLIConfigOverrides = { + model: 'gpt-5-mini', + // provider, apiKey intentionally omitted to test undefined handling + }; + + const result = applyCLIOverrides(clone(baseConfig), cliOverrides); + + expect(result.llm.model).toBe('gpt-5-mini'); // Applied + expect(result.llm.provider).toBe('openai'); // Original (undefined ignored) + expect(result.llm.apiKey).toBe('file-api-key'); // Original (undefined ignored) + }); + + test('sets tool confirmation mode to auto-approve when override enabled', () => { + const cliOverrides: CLIConfigOverrides = { + autoApprove: true, + }; + + const result = applyCLIOverrides(clone(baseConfig), cliOverrides); + + expect(result.toolConfirmation?.mode).toBe('auto-approve'); + expect(result.toolConfirmation?.timeout).toBe(120000); // Existing fields preserved + }); +}); + +describe('applyUserPreferences', () => { + const baseAgentConfig: AgentConfig = { + systemPrompt: 'test agent', + llm: { + provider: 'anthropic', + model: 'claude-haiku-4-5-20251001', + apiKey: '$ANTHROPIC_API_KEY', + }, + }; + + test('applies user preferences fully (provider, model, apiKey)', () => { + const preferences = { + llm: { + provider: 'openai' as const, + model: 'gpt-5-mini', + apiKey: '$OPENAI_API_KEY', + }, + }; + + const result = applyUserPreferences(clone(baseAgentConfig), preferences); + + expect(result.llm.provider).toBe('openai'); + expect(result.llm.model).toBe('gpt-5-mini'); + expect(result.llm.apiKey).toBe('$OPENAI_API_KEY'); + }); + + test('applies dexto provider preferences', () => { + const preferences = { + llm: { + provider: 'dexto' as const, + model: 'anthropic/claude-sonnet-4', + apiKey: '$DEXTO_API_KEY', + }, + }; + + const result = applyUserPreferences(clone(baseAgentConfig), preferences); + + expect(result.llm.provider).toBe('dexto'); + expect(result.llm.model).toBe('anthropic/claude-sonnet-4'); + expect(result.llm.apiKey).toBe('$DEXTO_API_KEY'); + }); + + test('without llm preferences -> returns config unchanged', () => { + const preferences = {}; + + const result = applyUserPreferences(clone(baseAgentConfig), preferences); + + expect(result.llm.provider).toBe('anthropic'); + expect(result.llm.model).toBe('claude-haiku-4-5-20251001'); + expect(result.llm.apiKey).toBe('$ANTHROPIC_API_KEY'); + }); + + test('preserves agent apiKey if user has no apiKey configured', () => { + const preferences = { + llm: { + provider: 'anthropic' as const, + model: 'claude-sonnet-4-5-20250929', + // No apiKey specified + }, + }; + + const result = applyUserPreferences(clone(baseAgentConfig), preferences); + + expect(result.llm.provider).toBe('anthropic'); + expect(result.llm.model).toBe('claude-sonnet-4-5-20250929'); + expect(result.llm.apiKey).toBe('$ANTHROPIC_API_KEY'); // Original preserved + }); + + test('applies baseURL if provided in preferences', () => { + const preferences = { + llm: { + provider: 'openai-compatible' as const, + model: 'local-model', + apiKey: 'test-key', + baseURL: 'http://localhost:8080/v1', + }, + }; + + const result = applyUserPreferences(clone(baseAgentConfig), preferences); + + expect(result.llm.provider).toBe('openai-compatible'); + expect(result.llm.baseURL).toBe('http://localhost:8080/v1'); + }); + + test('does not mutate original config', () => { + const originalConfig = clone(baseAgentConfig); + const preferences = { + llm: { + provider: 'openai' as const, + model: 'gpt-5-mini', + apiKey: '$OPENAI_API_KEY', + }, + }; + + applyUserPreferences(originalConfig, preferences); + + // Original should be unchanged + expect(originalConfig.llm.provider).toBe('anthropic'); + expect(originalConfig.llm.model).toBe('claude-haiku-4-5-20251001'); + }); +}); diff --git a/dexto/packages/cli/src/config/cli-overrides.ts b/dexto/packages/cli/src/config/cli-overrides.ts new file mode 100644 index 00000000..741f377e --- /dev/null +++ b/dexto/packages/cli/src/config/cli-overrides.ts @@ -0,0 +1,223 @@ +/** + * CLI-specific configuration types and utilities + * This file handles CLI argument processing and config merging logic + * + * Current behavior (Three-Layer LLM Resolution): + * Global preferences from preferences.yml are applied to ALL agents at runtime. + * See feature-plans/auto-update.md section 8.11 for the resolution order: + * 1. agent.local.yml llm section → Agent-specific override (NOT YET IMPLEMENTED) + * 2. preferences.yml llm section → User's global default (CURRENT) + * 3. agent.yml llm section → Bundled fallback + * + * Note: Sub-agents spawned via RuntimeService have separate LLM resolution logic + * that tries to preserve the sub-agent's intended model when possible. + * See packages/agent-management/src/tool-provider/llm-resolution.ts + * + * TODO: Future enhancements + * - Per-agent local overrides (~/.dexto/agents/{id}/{id}.local.yml) + * - Agent capability requirements (requires: { vision: true, toolUse: true }) + * - Merge strategy configuration for non-LLM fields + */ + +import type { AgentConfig, LLMConfig, LLMProvider } from '@dexto/core'; +import type { GlobalPreferences } from '@dexto/agent-management'; + +/** + * CLI config override type for fields that can be overridden via CLI + * Uses input type (LLMConfig) since these represent user-provided CLI arguments + */ +export interface CLIConfigOverrides + extends Partial> { + autoApprove?: boolean; + /** When false (via --no-elicitation), disables elicitation */ + elicitation?: boolean; +} + +/** + * Applies CLI overrides to an agent configuration + * This merges CLI arguments into the base config without validation. + * Validation should be performed separately after this merge step. + * + * @param baseConfig The configuration loaded from file + * @param cliOverrides CLI arguments to override specific fields + * @returns Merged configuration (unvalidated) + */ +export function applyCLIOverrides( + baseConfig: AgentConfig, + cliOverrides?: CLIConfigOverrides +): AgentConfig { + if (!cliOverrides || Object.keys(cliOverrides).length === 0) { + // No overrides, return base config as-is (no validation yet) + return baseConfig; + } + + // Create a deep copy of the base config for modification + const mergedConfig = JSON.parse(JSON.stringify(baseConfig)) as AgentConfig; + + // Apply CLI overrides to LLM config (llm is required in AgentConfig) + if (cliOverrides.provider) { + mergedConfig.llm.provider = cliOverrides.provider; + } + if (cliOverrides.model) { + mergedConfig.llm.model = cliOverrides.model; + } + if (cliOverrides.apiKey) { + mergedConfig.llm.apiKey = cliOverrides.apiKey; + } + + if (cliOverrides.autoApprove) { + // Ensure toolConfirmation section exists before overriding + if (!mergedConfig.toolConfirmation) { + mergedConfig.toolConfirmation = { mode: 'auto-approve' }; + } else { + mergedConfig.toolConfirmation.mode = 'auto-approve'; + } + } + + if (cliOverrides.elicitation === false) { + // Ensure elicitation section exists before overriding + if (!mergedConfig.elicitation) { + mergedConfig.elicitation = { enabled: false }; + } else { + mergedConfig.elicitation.enabled = false; + } + } + + // Return merged config without validation - validation happens later + return mergedConfig; +} + +/** + * Applies global user preferences to an agent configuration at runtime. + * This is used to ensure user's LLM preferences are applied to all agents. + * + * Unlike writeLLMPreferences() which modifies files, this performs an in-memory merge. + * User preferences fully override agent defaults for provider, model, and apiKey. + * + * @param baseConfig The configuration loaded from agent file + * @param preferences Global user preferences + * @returns Merged configuration with user preferences applied + */ +export function applyUserPreferences( + baseConfig: AgentConfig, + preferences: Partial +): AgentConfig { + // Create a deep copy to avoid mutating the original + const mergedConfig = JSON.parse(JSON.stringify(baseConfig)) as AgentConfig; + + // No LLM preferences to apply + if (!preferences.llm) { + return mergedConfig; + } + + // Apply user preferences - only override if defined (preferences is Partial) + if (preferences.llm.provider) { + mergedConfig.llm.provider = preferences.llm.provider; + } + if (preferences.llm.model) { + mergedConfig.llm.model = preferences.llm.model; + } + if (preferences.llm.apiKey) { + mergedConfig.llm.apiKey = preferences.llm.apiKey; + } + if (preferences.llm.baseURL) { + mergedConfig.llm.baseURL = preferences.llm.baseURL; + } + + return mergedConfig; +} + +/** + * Result of agent compatibility check + */ +export interface AgentCompatibilityResult { + compatible: boolean; + warnings: string[]; + instructions: string[]; + agentProvider: LLMProvider; + agentModel: string; + userProvider: LLMProvider | undefined; + userModel: string | undefined; + userHasApiKey: boolean; +} + +/** + * Check if user's current setup is compatible with an agent's requirements. + * Used when switching to non-default agents to warn users about potential issues. + * + * @param agentConfig The agent's configuration + * @param preferences User's global preferences (if available) + * @param resolvedApiKey Whether user has a valid API key for the agent's provider + * @returns Compatibility result with warnings and instructions + */ +export function checkAgentCompatibility( + agentConfig: AgentConfig, + preferences: GlobalPreferences | null, + resolvedApiKey: string | undefined +): AgentCompatibilityResult { + const warnings: string[] = []; + const instructions: string[] = []; + + const agentProvider = agentConfig.llm.provider; + const agentModel = agentConfig.llm.model; + const userProvider = preferences?.llm?.provider; + const userModel = preferences?.llm?.model; + const userHasApiKey = Boolean(resolvedApiKey); + + // Check if user has API key for this agent's provider + if (!userHasApiKey) { + warnings.push( + `This agent uses ${agentProvider} but you don't have an API key configured for it.` + ); + instructions.push(`Run: dexto setup --provider ${agentProvider}`); + } + + // Check if agent uses a different provider than user's default + // Only show this as a warning if API key is missing; otherwise just informational + if (userProvider && agentProvider !== userProvider && !userHasApiKey) { + const userDefault = userModel ? `${userProvider}/${userModel}` : userProvider; + warnings.push( + `This agent uses ${agentProvider}/${agentModel} (your default is ${userDefault}).` + ); + instructions.push( + `Make sure you have ${getEnvVarForProvider(agentProvider)} set in your environment.` + ); + } + + return { + compatible: warnings.length === 0, + warnings, + instructions, + agentProvider, + agentModel, + userProvider, + userModel, + userHasApiKey, + }; +} + +/** + * Get the environment variable name for a provider's API key + */ +function getEnvVarForProvider(provider: LLMProvider): string { + const envVarMap: Record = { + openai: 'OPENAI_API_KEY', + 'openai-compatible': 'OPENAI_API_KEY', + anthropic: 'ANTHROPIC_API_KEY', + google: 'GOOGLE_GENERATIVE_AI_API_KEY', + groq: 'GROQ_API_KEY', + xai: 'XAI_API_KEY', + cohere: 'COHERE_API_KEY', + openrouter: 'OPENROUTER_API_KEY', + litellm: 'LITELLM_API_KEY', + glama: 'GLAMA_API_KEY', + vertex: 'GOOGLE_APPLICATION_CREDENTIALS', + bedrock: 'AWS_ACCESS_KEY_ID', + // Local providers don't require API keys (empty string signals no key needed) + local: '', + ollama: '', + // Dexto gateway uses DEXTO_API_KEY from `dexto login` + dexto: 'DEXTO_API_KEY', + }; + return envVarMap[provider]; +} diff --git a/dexto/packages/cli/src/config/effective-llm.ts b/dexto/packages/cli/src/config/effective-llm.ts new file mode 100644 index 00000000..e6682607 --- /dev/null +++ b/dexto/packages/cli/src/config/effective-llm.ts @@ -0,0 +1,232 @@ +/** + * Effective LLM Configuration Resolution + * + * This module provides utilities to determine the effective LLM configuration + * at runtime, considering the layered config approach. + * + * ## Configuration Layers (Priority Order) + * + * 1. **agent.local.yml** (highest priority) - Agent-specific user overrides + * - Path: `~/.dexto/agents/{agent-id}/{agent-id}.local.yml` + * - Use case: User wants a specific agent to use a different LLM + * - NOT YET IMPLEMENTED - see feature-plans/auto-update.md section 8.9-8.11 + * + * 2. **preferences.yml** - User's global default LLM + * - Path: `~/.dexto/preferences.yml` + * - Use case: User's default choice from setup wizard or `/model` command + * - This is where most users' LLM config comes from + * + * 3. **agent.yml** (lowest priority) - Bundled agent defaults + * - Path: `~/.dexto/agents/{agent-id}/{agent-id}.yml` + * - Use case: Fallback for users who skip setup or power users with BYOK + * - This file is managed by Dexto and replaced on CLI updates + * + * ## Usage + * + * ```typescript + * import { getEffectiveLLMConfig } from './config/effective-llm.js'; + * + * const llm = await getEffectiveLLMConfig(); + * if (llm?.provider === 'dexto') { + * // User is configured to use Dexto credits + * } + * + * console.log(`Using ${llm.model} via ${llm.provider} (from ${llm.source})`); + * ``` + * + * ## Related Documentation + * + * - feature-plans/auto-update.md - Layered config and .local.yml design + * - feature-plans/holistic-dexto-auth-analysis/ - Explicit provider routing + * + * @module effective-llm + */ + +import type { LLMProvider } from '@dexto/core'; +import { + loadGlobalPreferences, + globalPreferencesExist, + loadAgentConfig, + resolveAgentPath, +} from '@dexto/agent-management'; +import { logger } from '@dexto/core'; + +/** + * Source of the effective LLM configuration + */ +export type LLMConfigSource = + | 'local' // From agent.local.yml (not yet implemented) + | 'preferences' // From preferences.yml (most common) + | 'bundled'; // From bundled agent.yml (fallback) + +/** + * The resolved effective LLM configuration with source tracking + */ +export interface EffectiveLLMConfig { + /** LLM provider (e.g., 'dexto', 'anthropic', 'openai') */ + provider: LLMProvider; + /** Model identifier (format depends on provider) */ + model: string; + /** API key or environment variable reference (e.g., '$DEXTO_API_KEY') */ + apiKey?: string; + /** Base URL for custom endpoints */ + baseURL?: string; + /** Where this config came from */ + source: LLMConfigSource; +} + +/** + * Options for getEffectiveLLMConfig + */ +export interface GetEffectiveLLMConfigOptions { + /** + * Agent ID to resolve config for. + * @default 'coding-agent' + */ + agentId?: string; + + /** + * Whether to include the bundled agent config as fallback. + * Set to false if you only want user-configured LLM. + * @default true + */ + includeBundledFallback?: boolean; +} + +/** + * Get the effective LLM configuration considering all config layers. + * + * This function resolves which LLM config will actually be used at runtime + * by checking each layer in priority order: + * + * 1. agent.local.yml (NOT YET IMPLEMENTED) + * 2. preferences.yml + * 3. bundled agent.yml (if includeBundledFallback is true) + * + * @param options - Configuration options + * @returns The effective LLM config with source, or null if none found + * + * @example + * ```typescript + * // Get effective LLM for default agent + * const llm = await getEffectiveLLMConfig(); + * + * // Get effective LLM for a specific agent + * const llm = await getEffectiveLLMConfig({ agentId: 'explore-agent' }); + * + * // Only get user-configured LLM (no bundled fallback) + * const llm = await getEffectiveLLMConfig({ includeBundledFallback: false }); + * ``` + */ +export async function getEffectiveLLMConfig( + options: GetEffectiveLLMConfigOptions = {} +): Promise { + const { agentId = 'coding-agent', includeBundledFallback = true } = options; + + // ------------------------------------------------------------------------- + // Layer 1: agent.local.yml (NOT YET IMPLEMENTED) + // ------------------------------------------------------------------------- + // TODO: Implement .local.yml loading when the feature is built + // See feature-plans/auto-update.md section 8.9-8.11 for the design + // + // The implementation would look something like: + // + // const localConfig = await loadLocalAgentConfig(agentId); + // if (localConfig?.llm?.provider && localConfig?.llm?.model) { + // logger.debug(`Using LLM config from ${agentId}.local.yml`); + // return { + // provider: localConfig.llm.provider, + // model: localConfig.llm.model, + // apiKey: localConfig.llm.apiKey, + // baseURL: localConfig.llm.baseURL, + // source: 'local', + // }; + // } + // ------------------------------------------------------------------------- + + // ------------------------------------------------------------------------- + // Layer 2: preferences.yml (user's global default) + // ------------------------------------------------------------------------- + if (globalPreferencesExist()) { + try { + const preferences = await loadGlobalPreferences(); + if (preferences?.llm?.provider && preferences?.llm?.model) { + logger.debug('Using LLM config from preferences.yml'); + const result: EffectiveLLMConfig = { + provider: preferences.llm.provider, + model: preferences.llm.model, + source: 'preferences', + }; + // Only set optional fields if they have values + if (preferences.llm.apiKey) { + result.apiKey = preferences.llm.apiKey; + } + if (preferences.llm.baseURL) { + result.baseURL = preferences.llm.baseURL; + } + return result; + } + } catch (error) { + logger.debug( + `Could not load preferences: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + // ------------------------------------------------------------------------- + // Layer 3: Bundled agent.yml (fallback) + // ------------------------------------------------------------------------- + if (includeBundledFallback) { + try { + const agentPath = await resolveAgentPath(agentId); + if (agentPath) { + const agentConfig = await loadAgentConfig(agentPath); + if (agentConfig?.llm?.provider && agentConfig?.llm?.model) { + logger.debug(`Using LLM config from bundled ${agentId}.yml`); + const result: EffectiveLLMConfig = { + provider: agentConfig.llm.provider, + model: agentConfig.llm.model, + source: 'bundled', + }; + // Only set optional fields if they have values + if (agentConfig.llm.apiKey) { + result.apiKey = agentConfig.llm.apiKey; + } + if (agentConfig.llm.baseURL) { + result.baseURL = agentConfig.llm.baseURL; + } + return result; + } + } + } catch (error) { + logger.debug( + `Could not load agent config: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + return null; +} + +/** + * Check if the effective LLM config uses Dexto credits. + * + * Convenience function that checks if the user is configured to use + * the Dexto provider (which requires authentication). + * + * @param options - Same options as getEffectiveLLMConfig + * @returns true if using provider: dexto, false otherwise + * + * @example + * ```typescript + * if (await isUsingDextoCredits()) { + * // Check authentication, show billing info, etc. + * } + * ``` + */ +export async function isUsingDextoCredits( + options: GetEffectiveLLMConfigOptions = {} +): Promise { + const config = await getEffectiveLLMConfig(options); + return config?.provider === 'dexto'; +} diff --git a/dexto/packages/cli/src/index.ts b/dexto/packages/cli/src/index.ts new file mode 100644 index 00000000..98717c55 --- /dev/null +++ b/dexto/packages/cli/src/index.ts @@ -0,0 +1,2008 @@ +#!/usr/bin/env node +// Load environment variables FIRST with layered loading +import { applyLayeredEnvironmentLoading } from './utils/env.js'; + +// Apply layered environment loading before any other imports +await applyLayeredEnvironmentLoading(); + +import { existsSync, readFileSync } from 'fs'; +import { createRequire } from 'module'; +import path from 'path'; +import { Command } from 'commander'; +import * as p from '@clack/prompts'; +import chalk from 'chalk'; +import { initAnalytics, capture, getWebUIAnalyticsConfig } from './analytics/index.js'; +import { withAnalytics, safeExit, ExitSignal } from './analytics/wrapper.js'; + +// Use createRequire to import package.json without experimental warning +const require = createRequire(import.meta.url); +const pkg = require('../package.json'); + +// Set CLI version for Dexto Gateway usage tracking +process.env.DEXTO_CLI_VERSION = pkg.version; + +// Populate DEXTO_API_KEY for Dexto gateway routing +// Resolution order in getDextoApiKey(): +// 1. Explicit env var (CI, testing, account override) +// 2. auth.json from `dexto login` +import { isDextoAuthEnabled } from '@dexto/agent-management'; +if (isDextoAuthEnabled()) { + const { getDextoApiKey } = await import('./cli/auth/index.js'); + const dextoApiKey = await getDextoApiKey(); + if (dextoApiKey) { + process.env.DEXTO_API_KEY = dextoApiKey; + } +} + +import { + logger, + getProviderFromModel, + getAllSupportedModels, + DextoAgent, + type LLMProvider, + isPath, + resolveApiKeyForProvider, + getPrimaryApiKeyEnvVar, +} from '@dexto/core'; +import { + resolveAgentPath, + loadAgentConfig, + globalPreferencesExist, + loadGlobalPreferences, + resolveBundledScript, +} from '@dexto/agent-management'; +import type { ValidatedAgentConfig } from '@dexto/core'; +import { startHonoApiServer } from './api/server-hono.js'; +import { validateCliOptions, handleCliOptionsError } from './cli/utils/options.js'; +import { validateAgentConfig } from './cli/utils/config-validation.js'; +import { applyCLIOverrides, applyUserPreferences } from './config/cli-overrides.js'; +import { enrichAgentConfig } from '@dexto/agent-management'; +import { getPort } from './utils/port-utils.js'; +import { + createDextoProject, + type CreateAppOptions, + createImage, + getUserInputToInitDextoApp, + initDexto, + postInitDexto, +} from './cli/commands/index.js'; +import { + handleSetupCommand, + type CLISetupOptionsInput, + handleInstallCommand, + type InstallCommandOptions, + handleUninstallCommand, + type UninstallCommandOptions, + handleListAgentsCommand, + type ListAgentsCommandOptionsInput, + handleWhichCommand, + handleSyncAgentsCommand, + shouldPromptForSync, + markSyncDismissed, + clearSyncDismissed, + type SyncAgentsCommandOptions, + handleLoginCommand, + handleLogoutCommand, + handleStatusCommand, + handleBillingStatusCommand, + handlePluginListCommand, + handlePluginInstallCommand, + handlePluginUninstallCommand, + handlePluginValidateCommand, + // Marketplace handlers + handleMarketplaceAddCommand, + handleMarketplaceRemoveCommand, + handleMarketplaceUpdateCommand, + handleMarketplaceListCommand, + handleMarketplacePluginsCommand, + handleMarketplaceInstallCommand, + type PluginListCommandOptionsInput, + type PluginInstallCommandOptionsInput, + type MarketplaceListCommandOptionsInput, + type MarketplaceInstallCommandOptionsInput, +} from './cli/commands/index.js'; +import { + handleSessionListCommand, + handleSessionHistoryCommand, + handleSessionDeleteCommand, + handleSessionSearchCommand, +} from './cli/commands/session-commands.js'; +import { requiresSetup } from './cli/utils/setup-utils.js'; +import { checkForFileInCurrentDirectory, FileNotFoundError } from './cli/utils/package-mgmt.js'; +import { checkForUpdates, displayUpdateNotification } from './cli/utils/version-check.js'; +import { resolveWebRoot } from './web.js'; +import { initializeMcpServer, createMcpTransport } from '@dexto/server'; +import { createAgentCard } from '@dexto/core'; +import { initializeMcpToolAggregationServer } from './api/mcp/tool-aggregation-handler.js'; +import { CLIConfigOverrides } from './config/cli-overrides.js'; + +const program = new Command(); + +// Initialize analytics early (no-op if disabled) +await initAnalytics({ appVersion: pkg.version }); + +// Start version check early (non-blocking) +// We'll check the result later and display notification for interactive modes +const versionCheckPromise = checkForUpdates(pkg.version); + +/** + * Recursively removes null values from an object. + * This handles YAML files that have explicit `apiKey: null` entries + * which would otherwise cause "Expected string, received null" validation errors. + */ +function cleanNullValues>(obj: T): T { + if (obj === null || obj === undefined) return obj; + if (typeof obj !== 'object') return obj; + if (Array.isArray(obj)) { + return obj.map((item) => + typeof item === 'object' && item !== null + ? cleanNullValues(item as Record) + : item + ) as unknown as T; + } + + const cleaned: Record = {}; + for (const [key, value] of Object.entries(obj)) { + if (value === null) { + // Skip null values - they become undefined (missing) + continue; + } + if (typeof value === 'object' && !Array.isArray(value)) { + cleaned[key] = cleanNullValues(value as Record); + } else if (Array.isArray(value)) { + cleaned[key] = value.map((item) => + typeof item === 'object' && item !== null + ? cleanNullValues(item as Record) + : item + ); + } else { + cleaned[key] = value; + } + } + return cleaned as T; +} + +// 1) GLOBAL OPTIONS +program + .name('dexto') + .description('AI-powered CLI and WebUI for interacting with MCP servers.') + .version(pkg.version, '-v, --version', 'output the current version') + .option('-a, --agent ', 'Agent ID or path to agent config file') + .option( + '-p, --prompt ', + 'Run prompt and exit. Alternatively provide a single quoted string as positional argument.' + ) + .option('-s, --strict', 'Require all server connections to succeed') + .option('--no-verbose', 'Disable verbose output') + .option('--no-interactive', 'Disable interactive prompts and API key setup') + .option('--skip-setup', 'Skip global setup validation (useful for MCP mode, automation)') + .option('-m, --model ', 'Specify the LLM model to use') + .option('--auto-approve', 'Always approve tool executions without confirmation prompts') + .option('--no-elicitation', 'Disable elicitation (agent cannot prompt user for input)') + .option('-c, --continue', 'Continue most recent session (requires -p/prompt)') + .option('-r, --resume ', 'Resume specific session (requires -p/prompt)') + .option( + '--mode ', + 'The application in which dexto should talk to you - web | cli | server | mcp', + 'web' + ) + .option('--port ', 'port for the server (default: 3000 for web, 3001 for server mode)') + .option('--no-auto-install', 'Disable automatic installation of missing agents from registry') + .option( + '--image ', + 'Image package to load (e.g., @dexto/image-local). Overrides config image field.' + ) + .option( + '--dev', + '[maintainers] Use local ./agents instead of ~/.dexto (for dexto repo development)' + ) + .enablePositionalOptions(); + +// 2) `create-app` SUB-COMMAND +program + .command('create-app [name]') + .description('Create a Dexto application (CLI, web, bot, etc.)') + .option('--from-image ', 'Use existing image (e.g., @dexto/image-local)') + .option('--extend-image ', 'Extend image with custom providers') + .option('--from-core', 'Build from @dexto/core (advanced)') + .option('--type ', 'App type: script, webapp (default: script)') + .action( + withAnalytics('create-app', async (name?: string, options?: CreateAppOptions) => { + try { + p.intro(chalk.inverse('Create Dexto App')); + + // Create the app project structure (fully self-contained) + await createDextoProject(name, options); + + p.outro(chalk.greenBright('Dexto app created successfully!')); + safeExit('create-app', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto create-app command failed: ${err}`); + safeExit('create-app', 1, 'error'); + } + }) + ); + +// 3) `create-image` SUB-COMMAND +program + .command('create-image [name]') + .description('Create a Dexto image - a distributable agent harness package') + .action( + withAnalytics('create-image', async (name?: string) => { + try { + p.intro(chalk.inverse('Create Dexto Image')); + + // Create the image project structure + const projectPath = await createImage(name); + + p.outro(chalk.greenBright(`Dexto image created successfully at ${projectPath}!`)); + safeExit('create-image', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto create-image command failed: ${err}`); + safeExit('create-image', 1, 'error'); + } + }) + ); + +// 4) `init-app` SUB-COMMAND +program + .command('init-app') + .description('Initialize an existing Typescript app with Dexto') + .action( + withAnalytics('init-app', async () => { + try { + // pre-condition: check that package.json and tsconfig.json exist in current directory to know that project is valid + await checkForFileInCurrentDirectory('package.json'); + await checkForFileInCurrentDirectory('tsconfig.json'); + + // start intro + p.intro(chalk.inverse('Dexto Init App')); + const userInput = await getUserInputToInitDextoApp(); + try { + capture('dexto_init', { + provider: userInput.llmProvider, + providedKey: Boolean(userInput.llmApiKey), + }); + } catch { + // Analytics failures should not block CLI execution. + } + await initDexto( + userInput.directory, + userInput.createExampleFile, + userInput.llmProvider, + userInput.llmApiKey + ); + p.outro(chalk.greenBright('Dexto app initialized successfully!')); + + // add notes for users to get started with their new initialized Dexto project + await postInitDexto(userInput.directory); + safeExit('init-app', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + // if the package.json or tsconfig.json is not found, we give instructions to create a new project + if (err instanceof FileNotFoundError) { + console.error(`❌ ${err.message} Run "dexto create-app" to create a new app`); + safeExit('init-app', 1, 'file-not-found'); + } + console.error(`❌ Initialization failed: ${err}`); + safeExit('init-app', 1, 'error'); + } + }) + ); + +// 5) `setup` SUB-COMMAND +program + .command('setup') + .description('Configure global Dexto preferences') + .option('--provider ', 'LLM provider (openai, anthropic, google, groq)') + .option('--model ', 'Model name (uses provider default if not specified)') + .option('--default-agent ', 'Default agent name (default: coding-agent)') + .option('--no-interactive', 'Skip interactive prompts and API key setup') + .option('--force', 'Overwrite existing setup without confirmation') + .action( + withAnalytics('setup', async (options: CLISetupOptionsInput) => { + try { + await handleSetupCommand(options); + safeExit('setup', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error( + `❌ dexto setup command failed: ${err}. Check logs in ~/.dexto/logs/dexto.log for more information` + ); + safeExit('setup', 1, 'error'); + } + }) + ); + +// 6) `install` SUB-COMMAND +program + .command('install [agents...]') + .description('Install agents from registry or custom YAML files/directories') + .option('--all', 'Install all available agents from registry') + .option('--no-inject-preferences', 'Skip injecting global preferences into installed agents') + .option('--force', 'Force reinstall even if agent is already installed') + .addHelpText( + 'after', + ` +Examples: + $ dexto install coding-agent Install agent from registry + $ dexto install agent1 agent2 Install multiple registry agents + $ dexto install --all Install all available registry agents + $ dexto install ./my-agent.yml Install custom agent from YAML file + $ dexto install ./my-agent-dir/ Install custom agent from directory (interactive)` + ) + .action( + withAnalytics( + 'install', + async (agents: string[] = [], options: Partial) => { + try { + await handleInstallCommand(agents, options); + safeExit('install', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto install command failed: ${err}`); + safeExit('install', 1, 'error'); + } + } + ) + ); + +// 7) `uninstall` SUB-COMMAND +program + .command('uninstall [agents...]') + .description('Uninstall agents from the local installation') + .option('--all', 'Uninstall all installed agents') + .option('--force', 'Force uninstall even if agent is protected (e.g., coding-agent)') + .action( + withAnalytics( + 'uninstall', + async (agents: string[], options: Partial) => { + try { + await handleUninstallCommand(agents, options); + safeExit('uninstall', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto uninstall command failed: ${err}`); + safeExit('uninstall', 1, 'error'); + } + } + ) + ); + +// 8) `list-agents` SUB-COMMAND +program + .command('list-agents') + .description('List available and installed agents') + .option('--verbose', 'Show detailed agent information') + .option('--installed', 'Show only installed agents') + .option('--available', 'Show only available agents') + .action( + withAnalytics('list-agents', async (options: ListAgentsCommandOptionsInput) => { + try { + await handleListAgentsCommand(options); + safeExit('list-agents', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto list-agents command failed: ${err}`); + safeExit('list-agents', 1, 'error'); + } + }) + ); + +// 9) `which` SUB-COMMAND +program + .command('which ') + .description('Show the path to an agent') + .action( + withAnalytics('which', async (agent: string) => { + try { + await handleWhichCommand(agent); + safeExit('which', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto which command failed: ${err}`); + safeExit('which', 1, 'error'); + } + }) + ); + +// 10) `sync-agents` SUB-COMMAND +program + .command('sync-agents') + .description('Sync installed agents with bundled versions') + .option('--list', 'List agent status without updating') + .option('--force', 'Update all agents without prompting') + .action( + withAnalytics('sync-agents', async (options: Partial) => { + try { + await handleSyncAgentsCommand(options); + safeExit('sync-agents', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto sync-agents command failed: ${err}`); + safeExit('sync-agents', 1, 'error'); + } + }) + ); + +// 11) `plugin` SUB-COMMAND +const pluginCommand = program.command('plugin').description('Manage plugins'); + +pluginCommand + .command('list') + .description('List installed plugins') + .option('--verbose', 'Show detailed plugin information') + .action( + withAnalytics('plugin list', async (options: PluginListCommandOptionsInput) => { + try { + await handlePluginListCommand(options); + safeExit('plugin list', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin list command failed: ${err}`); + safeExit('plugin list', 1, 'error'); + } + }) + ); + +pluginCommand + .command('install') + .description('Install a plugin from a local directory') + .requiredOption('--path ', 'Path to the plugin directory') + .option('--scope ', 'Installation scope: user, project, or local', 'user') + .option('--force', 'Force overwrite if already installed') + .action( + withAnalytics('plugin install', async (options: PluginInstallCommandOptionsInput) => { + try { + await handlePluginInstallCommand(options); + safeExit('plugin install', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin install command failed: ${err}`); + safeExit('plugin install', 1, 'error'); + } + }) + ); + +pluginCommand + .command('uninstall ') + .description('Uninstall a plugin by name') + .action( + withAnalytics('plugin uninstall', async (name: string) => { + try { + await handlePluginUninstallCommand({ name }); + safeExit('plugin uninstall', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin uninstall command failed: ${err}`); + safeExit('plugin uninstall', 1, 'error'); + } + }) + ); + +pluginCommand + .command('validate [path]') + .description('Validate a plugin directory structure') + .action( + withAnalytics('plugin validate', async (path?: string) => { + try { + await handlePluginValidateCommand({ path: path || '.' }); + safeExit('plugin validate', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin validate command failed: ${err}`); + safeExit('plugin validate', 1, 'error'); + } + }) + ); + +// 12) `plugin marketplace` SUB-COMMANDS +const marketplaceCommand = pluginCommand + .command('marketplace') + .alias('market') + .description('Manage plugin marketplaces'); + +marketplaceCommand + .command('add ') + .description('Add a marketplace (GitHub: owner/repo, git URL, or local path)') + .option('--name ', 'Custom name for the marketplace') + .action( + withAnalytics( + 'plugin marketplace add', + async (source: string, options: { name?: string }) => { + try { + await handleMarketplaceAddCommand({ source, name: options.name }); + safeExit('plugin marketplace add', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin marketplace add command failed: ${err}`); + safeExit('plugin marketplace add', 1, 'error'); + } + } + ) + ); + +marketplaceCommand + .command('list') + .description('List registered marketplaces') + .option('--verbose', 'Show detailed marketplace information') + .action( + withAnalytics( + 'plugin marketplace list', + async (options: MarketplaceListCommandOptionsInput) => { + try { + await handleMarketplaceListCommand(options); + safeExit('plugin marketplace list', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin marketplace list command failed: ${err}`); + safeExit('plugin marketplace list', 1, 'error'); + } + } + ) + ); + +marketplaceCommand + .command('remove ') + .alias('rm') + .description('Remove a registered marketplace') + .action( + withAnalytics('plugin marketplace remove', async (name: string) => { + try { + await handleMarketplaceRemoveCommand({ name }); + safeExit('plugin marketplace remove', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin marketplace remove command failed: ${err}`); + safeExit('plugin marketplace remove', 1, 'error'); + } + }) + ); + +marketplaceCommand + .command('update [name]') + .description('Update marketplace(s) from remote (git pull)') + .action( + withAnalytics('plugin marketplace update', async (name?: string) => { + try { + await handleMarketplaceUpdateCommand({ name }); + safeExit('plugin marketplace update', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin marketplace update command failed: ${err}`); + safeExit('plugin marketplace update', 1, 'error'); + } + }) + ); + +marketplaceCommand + .command('plugins [marketplace]') + .description('List plugins available in marketplaces') + .option('--verbose', 'Show plugin descriptions') + .action( + withAnalytics( + 'plugin marketplace plugins', + async (marketplace?: string, options?: { verbose?: boolean }) => { + try { + await handleMarketplacePluginsCommand({ + marketplace, + verbose: options?.verbose, + }); + safeExit('plugin marketplace plugins', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin marketplace plugins command failed: ${err}`); + safeExit('plugin marketplace plugins', 1, 'error'); + } + } + ) + ); + +marketplaceCommand + .command('install ') + .description('Install a plugin from marketplace (plugin or plugin@marketplace)') + .option('--scope ', 'Installation scope: user, project, or local', 'user') + .option('--force', 'Force reinstall if already exists') + .action( + withAnalytics( + 'plugin marketplace install', + async (plugin: string, options: MarketplaceInstallCommandOptionsInput) => { + try { + await handleMarketplaceInstallCommand({ ...options, plugin }); + safeExit('plugin marketplace install', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto plugin marketplace install command failed: ${err}`); + safeExit('plugin marketplace install', 1, 'error'); + } + } + ) + ); + +// Helper to bootstrap a minimal agent for non-interactive session/search ops +async function bootstrapAgentFromGlobalOpts() { + const globalOpts = program.opts(); + const resolvedPath = await resolveAgentPath(globalOpts.agent, globalOpts.autoInstall !== false); + const rawConfig = await loadAgentConfig(resolvedPath); + const mergedConfig = applyCLIOverrides(rawConfig, globalOpts); + + // Load image first to get bundled plugins + // Priority: CLI flag > Agent config > Environment variable > Default + const imageName = + globalOpts.image || // --image flag + mergedConfig.image || // image field in agent config + process.env.DEXTO_IMAGE || // DEXTO_IMAGE env var + '@dexto/image-local'; // Default for convenience + + let imageMetadata: { bundledPlugins?: string[] } | null = null; + try { + const imageModule = await import(imageName); + imageMetadata = imageModule.imageMetadata || null; + } catch (_err) { + console.error(`❌ Failed to load image '${imageName}'`); + console.error( + `💡 Install it with: ${ + existsSync('package.json') ? 'npm install' : 'npm install -g' + } ${imageName}` + ); + safeExit('bootstrap', 1, 'image-load-failed'); + } + + // Enrich config with bundled plugins from image + const enrichedConfig = enrichAgentConfig(mergedConfig, resolvedPath, { + logLevel: 'info', // CLI uses info-level logging for visibility + bundledPlugins: imageMetadata?.bundledPlugins || [], + }); + + // Override approval config for read-only commands (never run conversations) + // This avoids needing to set up unused approval handlers + enrichedConfig.toolConfirmation = { + mode: 'auto-approve', + ...(enrichedConfig.toolConfirmation?.timeout !== undefined && { + timeout: enrichedConfig.toolConfirmation.timeout, + }), + }; + enrichedConfig.elicitation = { + enabled: false, + ...(enrichedConfig.elicitation?.timeout !== undefined && { + timeout: enrichedConfig.elicitation.timeout, + }), + }; + + // Use relaxed validation for session commands - they don't need LLM calls + const agent = new DextoAgent(enrichedConfig, resolvedPath, { strict: false }); + await agent.start(); + + // Register graceful shutdown + const shutdown = async () => { + try { + await agent.stop(); + } catch (_err) { + // Ignore shutdown errors + } + }; + + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); + + return agent; +} + +// Helper to find the most recent session +// @param includeHeadless - If false, skip ephemeral headless sessions (for interactive mode) +// If true, include all sessions (for headless mode continuation) +async function getMostRecentSessionId( + agent: DextoAgent, + includeHeadless: boolean = false +): Promise { + const sessionIds = await agent.listSessions(); + if (sessionIds.length === 0) { + return null; + } + + // Get metadata for all sessions to find most recent + let mostRecentId: string | null = null; + let mostRecentActivity = 0; + + for (const sessionId of sessionIds) { + // Skip ephemeral headless sessions unless includeHeadless is true + if (!includeHeadless && sessionId.startsWith('headless-')) { + continue; + } + + const metadata = await agent.getSessionMetadata(sessionId); + if (metadata && metadata.lastActivity > mostRecentActivity) { + mostRecentActivity = metadata.lastActivity; + mostRecentId = sessionId; + } + } + + return mostRecentId; +} + +// 11) `session` SUB-COMMAND +const sessionCommand = program.command('session').description('Manage chat sessions'); + +sessionCommand + .command('list') + .description('List all sessions') + .action( + withAnalytics('session list', async () => { + try { + const agent = await bootstrapAgentFromGlobalOpts(); + + await handleSessionListCommand(agent); + await agent.stop(); + safeExit('session list', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto session list command failed: ${err}`); + safeExit('session list', 1, 'error'); + } + }) + ); + +sessionCommand + .command('history') + .description('Show session history') + .argument('[sessionId]', 'Session ID (defaults to current session)') + .action( + withAnalytics('session history', async (sessionId: string) => { + try { + const agent = await bootstrapAgentFromGlobalOpts(); + + await handleSessionHistoryCommand(agent, sessionId); + await agent.stop(); + safeExit('session history', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto session history command failed: ${err}`); + safeExit('session history', 1, 'error'); + } + }) + ); + +sessionCommand + .command('delete') + .description('Delete a session') + .argument('', 'Session ID to delete') + .action( + withAnalytics('session delete', async (sessionId: string) => { + try { + const agent = await bootstrapAgentFromGlobalOpts(); + + await handleSessionDeleteCommand(agent, sessionId); + await agent.stop(); + safeExit('session delete', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto session delete command failed: ${err}`); + safeExit('session delete', 1, 'error'); + } + }) + ); + +// 12) `search` SUB-COMMAND +program + .command('search') + .description('Search session history') + .argument('', 'Search query') + .option('--session ', 'Search in specific session') + .option('--role ', 'Filter by role (user, assistant, system, tool)') + .option('--limit ', 'Limit number of results', '10') + .action( + withAnalytics( + 'search', + async (query: string, options: { session?: string; role?: string; limit?: string }) => { + try { + const agent = await bootstrapAgentFromGlobalOpts(); + + const searchOptions: { + sessionId?: string; + role?: 'user' | 'assistant' | 'system' | 'tool'; + limit?: number; + } = {}; + + if (options.session) { + searchOptions.sessionId = options.session; + } + if (options.role) { + const allowed = new Set(['user', 'assistant', 'system', 'tool']); + if (!allowed.has(options.role)) { + console.error( + `❌ Invalid role: ${options.role}. Use one of: user, assistant, system, tool` + ); + safeExit('search', 1, 'invalid-role'); + } + searchOptions.role = options.role as + | 'user' + | 'assistant' + | 'system' + | 'tool'; + } + if (options.limit) { + const parsed = parseInt(options.limit, 10); + if (Number.isNaN(parsed) || parsed <= 0) { + console.error( + `❌ Invalid --limit: ${options.limit}. Use a positive integer (e.g., 10).` + ); + safeExit('search', 1, 'invalid-limit'); + } + searchOptions.limit = parsed; + } + + await handleSessionSearchCommand(agent, query, searchOptions); + await agent.stop(); + safeExit('search', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto search command failed: ${err}`); + safeExit('search', 1, 'error'); + } + } + ) + ); + +// 13) `auth` SUB-COMMAND GROUP +const authCommand = program.command('auth').description('Manage authentication'); + +authCommand + .command('login') + .description('Login to Dexto') + .option('--api-key ', 'Use Dexto API key instead of browser login') + .option('--no-interactive', 'Disable interactive prompts') + .action( + withAnalytics('auth login', async (options: { apiKey?: string; interactive?: boolean }) => { + try { + await handleLoginCommand(options); + safeExit('auth login', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto auth login command failed: ${err}`); + safeExit('auth login', 1, 'error'); + } + }) + ); + +authCommand + .command('logout') + .description('Logout from Dexto') + .option('--force', 'Skip confirmation prompt') + .option('--no-interactive', 'Disable interactive prompts') + .action( + withAnalytics( + 'auth logout', + async (options: { force?: boolean; interactive?: boolean }) => { + try { + await handleLogoutCommand(options); + safeExit('auth logout', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto auth logout command failed: ${err}`); + safeExit('auth logout', 1, 'error'); + } + } + ) + ); + +authCommand + .command('status') + .description('Show authentication status') + .action( + withAnalytics('auth status', async () => { + try { + await handleStatusCommand(); + safeExit('auth status', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto auth status command failed: ${err}`); + safeExit('auth status', 1, 'error'); + } + }) + ); + +// Also add convenience aliases at root level +program + .command('login') + .description('Login to Dexto (alias for `dexto auth login`)') + .option('--api-key ', 'Use Dexto API key instead of browser login') + .option('--no-interactive', 'Disable interactive prompts') + .action( + withAnalytics('login', async (options: { apiKey?: string; interactive?: boolean }) => { + try { + await handleLoginCommand(options); + safeExit('login', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto login command failed: ${err}`); + safeExit('login', 1, 'error'); + } + }) + ); + +program + .command('logout') + .description('Logout from Dexto (alias for `dexto auth logout`)') + .option('--force', 'Skip confirmation prompt') + .option('--no-interactive', 'Disable interactive prompts') + .action( + withAnalytics('logout', async (options: { force?: boolean; interactive?: boolean }) => { + try { + await handleLogoutCommand(options); + safeExit('logout', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto logout command failed: ${err}`); + safeExit('logout', 1, 'error'); + } + }) + ); + +// 14) `billing` COMMAND +program + .command('billing') + .description('Show billing status and credit balance') + .action( + withAnalytics('billing', async () => { + try { + await handleBillingStatusCommand(); + safeExit('billing', 0); + } catch (err) { + if (err instanceof ExitSignal) throw err; + console.error(`❌ dexto billing command failed: ${err}`); + safeExit('billing', 1, 'error'); + } + }) + ); + +// 15) `mcp` SUB-COMMAND +// For now, this mode simply aggregates and re-expose tools from configured MCP servers (no agent) +// dexto --mode mcp will be moved to this sub-command in the future +program + .command('mcp') + .description( + 'Start Dexto as an MCP server. Use --group-servers to aggregate and re-expose tools from configured MCP servers. \ + In the future, this command will expose the agent as an MCP server by default.' + ) + .option('-s, --strict', 'Require all MCP server connections to succeed') + .option( + '--group-servers', + 'Aggregate and re-expose tools from configured MCP servers (required for now)' + ) + .option('--name ', 'Name for the MCP server', 'dexto-tools') + .option('--version ', 'Version for the MCP server', '1.0.0') + .action( + withAnalytics( + 'mcp', + async (options) => { + try { + // Validate that --group-servers flag is provided (mandatory for now) + if (!options.groupServers) { + console.error( + '❌ The --group-servers flag is required. This command currently only supports aggregating and re-exposing tools from configured MCP servers.' + ); + console.error('Usage: dexto mcp --group-servers'); + safeExit('mcp', 1, 'missing-group-servers'); + } + + // Load and resolve config + // Get the global agent option from the main program + const globalOpts = program.opts(); + const nameOrPath = globalOpts.agent; + + const configPath = await resolveAgentPath( + nameOrPath, + globalOpts.autoInstall !== false + ); + console.log(`📄 Loading Dexto config from: ${configPath}`); + const config = await loadAgentConfig(configPath); + + logger.info(`Validating MCP servers...`); + // Validate that MCP servers are configured + if (!config.mcpServers || Object.keys(config.mcpServers).length === 0) { + console.error( + '❌ No MCP servers configured. Please configure mcpServers in your config file.' + ); + safeExit('mcp', 1, 'no-mcp-servers'); + } + + const { ServerConfigsSchema } = await import('@dexto/core'); + const validatedServers = ServerConfigsSchema.parse(config.mcpServers); + logger.info( + `Validated MCP servers. Configured servers: ${Object.keys(validatedServers).join(', ')}` + ); + + // Logs are already redirected to file by default to prevent interference with stdio transport + const currentLogPath = logger.getLogFilePath(); + logger.info( + `MCP mode using log file: ${currentLogPath || 'default .dexto location'}` + ); + + logger.info( + `Starting MCP tool aggregation server: ${options.name} v${options.version}` + ); + + // Create stdio transport for MCP tool aggregation + const mcpTransport = await createMcpTransport('stdio'); + // Initialize tool aggregation server + await initializeMcpToolAggregationServer( + validatedServers, + mcpTransport, + options.name, + options.version, + options.strict + ); + + logger.info('MCP tool aggregation server started successfully'); + } catch (err) { + if (err instanceof ExitSignal) throw err; + // Write to stderr to avoid interfering with MCP protocol + process.stderr.write(`MCP tool aggregation server startup failed: ${err}\n`); + safeExit('mcp', 1, 'mcp-agg-failed'); + } + }, + { timeoutMs: 0 } + ) + ); + +// 16) Main dexto CLI - Interactive/One shot (CLI/HEADLESS) or run in other modes (--mode web/server/mcp) +program + .argument( + '[prompt...]', + 'Natural-language prompt to run once. If not passed, dexto will start as an interactive CLI' + ) + // Main customer facing description + .description( + 'Dexto CLI - AI-powered assistant with session management.\n\n' + + 'Basic Usage:\n' + + ' dexto Start web UI (default)\n' + + ' dexto "query" Run one-shot query (auto-uses CLI mode)\n' + + ' dexto -p "query" Run one-shot query, then exit\n' + + ' cat file | dexto -p "query" Process piped content\n\n' + + 'CLI Mode:\n' + + ' dexto --mode cli Start interactive CLI\n\n' + + 'Headless Session Continuation:\n' + + ' dexto -c -p "message" Continue most recent session\n' + + ' dexto -r -p "msg" Resume specific session by ID\n' + + ' (Interactive mode: use /resume command instead)\n\n' + + 'Session Management Commands:\n' + + ' dexto session list List all sessions\n' + + ' dexto session history [id] Show session history\n' + + ' dexto session delete Delete a session\n' + + ' dexto search Search across sessions\n' + + ' Options: --session , --role , --limit \n\n' + + 'Agent Selection:\n' + + ' dexto --agent coding-agent Use installed agent by name\n' + + ' dexto --agent ./my-agent.yml Use agent from file path\n' + + ' dexto -a agents/custom.yml Short form with relative path\n\n' + + 'Tool Confirmation:\n' + + ' dexto --auto-approve Auto-approve all tool executions\n\n' + + 'Advanced Modes:\n' + + ' dexto --mode server Run as API server\n' + + ' dexto --mode mcp Run as MCP server\n\n' + + 'See https://docs.dexto.ai for documentation and examples' + ) + .action( + withAnalytics( + 'main', + async (prompt: string[] = []) => { + // ——— ENV CHECK (optional) ——— + if (!existsSync('.env')) { + logger.debug( + 'WARNING: .env file not found; copy .env.example and set your API keys.' + ); + } + + const opts = program.opts(); + + // Set dev mode early to use local repo agents instead of ~/.dexto + if (opts.dev) { + process.env.DEXTO_DEV_MODE = 'true'; + } + + // ——— LOAD DEFAULT MODE FROM PREFERENCES ——— + // If --mode was not explicitly provided on CLI, use defaultMode from preferences + const modeSource = program.getOptionValueSource('mode'); + const explicitModeProvided = modeSource === 'cli'; + if (!explicitModeProvided) { + try { + if (globalPreferencesExist()) { + const preferences = await loadGlobalPreferences(); + if (preferences.defaults?.defaultMode) { + opts.mode = preferences.defaults.defaultMode; + logger.debug(`Using default mode from preferences: ${opts.mode}`); + } + } + } catch (error) { + // Silently fall back to hardcoded default if preferences loading fails + logger.debug( + `Failed to load default mode from preferences: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + let headlessInput: string | undefined = undefined; + + // Prefer explicit -p/--prompt for headless one-shot + if (opts.prompt !== undefined && String(opts.prompt).trim() !== '') { + headlessInput = String(opts.prompt); + } else if (opts.prompt !== undefined) { + // Explicit empty -p "" was provided + console.error( + '❌ For headless one-shot mode, prompt cannot be empty. Provide a non-empty prompt with -p/--prompt or use positional argument.' + ); + safeExit('main', 1, 'empty-prompt'); + } else if (prompt.length > 0) { + // Enforce quoted single positional argument for headless mode + if (prompt.length === 1) { + headlessInput = prompt[0]; + } else { + console.error( + '❌ For headless one-shot mode, pass the prompt in double quotes as a single argument (e.g., "say hello") or use -p/--prompt.' + ); + safeExit('main', 1, 'too-many-positional'); + } + } + + // Note: Agent selection must be passed via -a/--agent. We no longer interpret + // the first positional argument as an agent name to avoid ambiguity with prompts. + + // ——— VALIDATE SESSION FLAGS ——— + // -c and -r are for headless mode only (require a prompt) + if ((opts.continue || opts.resume) && !headlessInput) { + console.error( + '❌ Session continuation flags (-c/--continue or -r/--resume) require a prompt for headless mode.' + ); + console.error( + ' Provide a prompt: dexto -c -p "your message" or dexto -r -p "your message"' + ); + console.error( + ' For interactive mode with session management, use: dexto (starts new) or use /resume command' + ); + safeExit('main', 1, 'session-flag-without-prompt'); + } + + // ——— FORCE CLI MODE FOR HEADLESS PROMPTS ——— + // If a prompt was provided via -p or positional args, force CLI mode + if (headlessInput && opts.mode !== 'cli') { + console.error( + `ℹ️ Prompt detected via -p or positional argument. Forcing CLI mode for one-shot execution.` + ); + console.error(` Original mode: ${opts.mode} → Overridden to: cli`); + opts.mode = 'cli'; + } + + // ——— Infer provider & API key from model ——— + if (opts.model) { + let provider: LLMProvider; + try { + provider = getProviderFromModel(opts.model); + } catch (err) { + console.error(`❌ ${(err as Error).message}`); + console.error(`Supported models: ${getAllSupportedModels().join(', ')}`); + safeExit('main', 1, 'invalid-model'); + } + + const apiKey = resolveApiKeyForProvider(provider); + if (!apiKey) { + const envVar = getPrimaryApiKeyEnvVar(provider); + console.error( + `❌ Missing API key for provider '${provider}' - please set $${envVar}` + ); + safeExit('main', 1, 'missing-api-key'); + } + opts.provider = provider; + opts.apiKey = apiKey; + } + + try { + validateCliOptions(opts); + } catch (err) { + handleCliOptionsError(err); + } + + // ——— ENHANCED PREFERENCE-AWARE CONFIG LOADING ——— + let validatedConfig: ValidatedAgentConfig; + let resolvedPath: string; + + // Determine validation mode early - used throughout config loading and agent creation + // Use relaxed validation for interactive modes (web/cli) where users can configure later + // Use strict validation for headless modes (server/mcp) that need full config upfront + const isInteractiveMode = opts.mode === 'web' || opts.mode === 'cli'; + + try { + // Case 1: File path - skip all validation and setup + if (opts.agent && isPath(opts.agent)) { + resolvedPath = await resolveAgentPath( + opts.agent, + opts.autoInstall !== false + ); + } + // Cases 2 & 3: Default agent or registry agent + else { + // Early registry validation for named agents + if (opts.agent) { + // Load bundled registry to check if agent exists + try { + const bundledRegistryPath = resolveBundledScript( + 'agents/agent-registry.json' + ); + const registryContent = readFileSync(bundledRegistryPath, 'utf-8'); + const bundledRegistry = JSON.parse(registryContent); + + // Check if agent exists in bundled registry + if (!(opts.agent in bundledRegistry.agents)) { + console.error(`❌ Agent '${opts.agent}' not found in registry`); + + // Show available agents + const available = Object.keys(bundledRegistry.agents); + if (available.length > 0) { + console.log(`📋 Available agents: ${available.join(', ')}`); + } else { + console.log('📋 No agents available in registry'); + } + safeExit('main', 1, 'agent-not-in-registry'); + return; + } + } catch (error) { + logger.warn( + `Could not validate agent against registry: ${error instanceof Error ? error.message : String(error)}` + ); + // Continue anyway - resolver will handle it + } + } + + // Check setup state and auto-trigger if needed + // Skip if --skip-setup flag is set (for MCP mode, automation, etc.) + if (!opts.skipSetup && (await requiresSetup())) { + if (opts.interactive === false) { + console.error( + '❌ Setup required but --no-interactive flag is set.' + ); + console.error( + '💡 Run `dexto setup` first, or use --skip-setup to bypass global setup.' + ); + safeExit('main', 1, 'setup-required-non-interactive'); + } + + await handleSetupCommand({ interactive: true }); + + // Reload preferences after setup to get the newly selected default mode + // (setup may have just saved a different mode than the default 'web') + try { + const newPreferences = await loadGlobalPreferences(); + if (newPreferences.defaults?.defaultMode) { + opts.mode = newPreferences.defaults.defaultMode; + logger.debug( + `Updated mode from setup preferences: ${opts.mode}` + ); + } + } catch { + // Ignore errors - will use default mode + } + } + + // Now resolve agent (will auto-install since setup is complete) + resolvedPath = await resolveAgentPath( + opts.agent, + opts.autoInstall !== false + ); + } + + // Load raw config and apply CLI overrides + const rawConfig = await loadAgentConfig(resolvedPath); + let mergedConfig = applyCLIOverrides(rawConfig, opts as CLIConfigOverrides); + + // ——— PREFERENCE-AWARE CONFIG HANDLING ——— + // User's LLM preferences from preferences.yml apply to ALL agents + // See feature-plans/auto-update.md section 8.11 - Three-Layer LLM Resolution + const agentId = opts.agent ?? 'coding-agent'; + let preferences: Awaited> | null = + null; + + if (globalPreferencesExist()) { + try { + preferences = await loadGlobalPreferences(); + } catch { + // Preferences exist but couldn't load - continue without them + logger.debug('Could not load preferences, continuing without them'); + } + } + + // Check if user is configured for Dexto credits but not authenticated + // This can happen if user logged out after setting up with Dexto + // Now that preferences apply to ALL agents, we check for any agent + // Only run this check when Dexto auth feature is enabled + if (isDextoAuthEnabled()) { + const { checkDextoAuthState } = await import( + './cli/utils/dexto-auth-check.js' + ); + const authCheck = await checkDextoAuthState( + opts.interactive !== false, + agentId + ); + + if (!authCheck.shouldContinue) { + if (authCheck.action === 'login') { + // User wants to log in - run login flow then restart + const { handleLoginCommand } = await import( + './cli/commands/auth/login.js' + ); + await handleLoginCommand({ interactive: true }); + + // Verify key was actually provisioned (provisionKeys silently catches errors) + const { canUseDextoProvider } = await import( + './cli/utils/dexto-setup.js' + ); + if (!(await canUseDextoProvider())) { + console.error( + '\n❌ API key provisioning failed. Please try again or run `dexto setup` to use a different provider.\n' + ); + safeExit('main', 1, 'dexto-key-provisioning-failed'); + } + // After login, continue with startup (preferences unchanged, now authenticated) + } else if (authCheck.action === 'setup') { + // User wants to configure different provider - run setup + const { handleSetupCommand } = await import( + './cli/commands/setup.js' + ); + await handleSetupCommand({ interactive: true, force: true }); + // Reload preferences after setup + preferences = await loadGlobalPreferences(); + } else { + // User cancelled + safeExit('main', 0, 'dexto-auth-check-cancelled'); + } + } + } + + // Check for pending API key setup (user skipped during initial setup) + // Since preferences now apply to ALL agents, this check runs for any agent + if (preferences?.setup?.apiKeyPending && opts.interactive !== false) { + // Check if API key is still missing (user may have set it manually) + const configuredApiKey = resolveApiKeyForProvider(preferences.llm.provider); + if (!configuredApiKey) { + const { promptForPendingApiKey } = await import( + './cli/utils/api-key-setup.js' + ); + const { updateGlobalPreferences } = await import( + '@dexto/agent-management' + ); + + const result = await promptForPendingApiKey( + preferences.llm.provider, + preferences.llm.model + ); + + if (result.action === 'cancel') { + safeExit('main', 0, 'pending-api-key-cancelled'); + } + + if (result.action === 'setup' && result.apiKey) { + // API key was configured - update preferences to clear pending flag + await updateGlobalPreferences({ + setup: { apiKeyPending: false }, + }); + // Update the merged config with the new API key + mergedConfig.llm.apiKey = result.apiKey; + logger.debug('API key configured, pending flag cleared'); + } + // If 'skip', continue without API key (user chose to proceed) + } else { + // API key exists (user set it manually) - clear the pending flag + const { updateGlobalPreferences } = await import( + '@dexto/agent-management' + ); + await updateGlobalPreferences({ + setup: { apiKeyPending: false }, + }); + logger.debug('API key found in environment, cleared pending flag'); + } + } + + // Apply user's LLM preferences to ALL agents (not just the default) + // See feature-plans/auto-update.md section 8.11 - Three-Layer LLM Resolution: + // local.llm ?? preferences.llm ?? bundled.llm + // The preferences.llm acts as a "global .local.yml" for LLM settings + if (preferences?.llm?.provider && preferences?.llm?.model) { + mergedConfig = applyUserPreferences(mergedConfig, preferences); + logger.debug(`Applied user preferences to ${agentId}`, { + provider: preferences.llm.provider, + model: preferences.llm.model, + }); + } + + // Clean up null values from config (can happen from YAML files with explicit nulls) + // This prevents "Expected string, received null" errors for optional fields + const cleanedConfig = cleanNullValues(mergedConfig); + + // Load image first to get bundled plugins + // Priority: CLI flag > Agent config > Environment variable > Default + const imageNameForEnrichment = + opts.image || // --image flag + cleanedConfig.image || // image field in agent config + process.env.DEXTO_IMAGE || // DEXTO_IMAGE env var + '@dexto/image-local'; // Default for convenience + + let imageMetadataForEnrichment: { bundledPlugins?: string[] } | null = null; + try { + const imageModule = await import(imageNameForEnrichment); + imageMetadataForEnrichment = imageModule.imageMetadata || null; + logger.debug(`Loaded image for enrichment: ${imageNameForEnrichment}`); + } catch (err) { + console.error(`❌ Failed to load image '${imageNameForEnrichment}'`); + if (err instanceof Error) { + logger.debug(`Image load error: ${err.message}`); + } + safeExit('main', 1, 'image-load-failed'); + } + + // Enrich config with per-agent paths and bundled plugins BEFORE validation + // Enrichment adds filesystem paths to storage (schema has in-memory defaults) + // Interactive CLI mode: only log to file (console would interfere with chat UI) + const isInteractiveCli = opts.mode === 'cli' && !headlessInput; + const enrichedConfig = enrichAgentConfig(cleanedConfig, resolvedPath, { + isInteractiveCli, + logLevel: 'info', // CLI uses info-level logging for visibility + bundledPlugins: imageMetadataForEnrichment?.bundledPlugins || [], + }); + + // Validate enriched config with interactive setup if needed (for API key issues) + // isInteractiveMode is defined above the try block + const validationResult = await validateAgentConfig( + enrichedConfig, + opts.interactive !== false, + { strict: !isInteractiveMode } + ); + + if (validationResult.success && validationResult.config) { + validatedConfig = validationResult.config; + } else if (validationResult.skipped) { + // User chose to continue despite validation errors + // SAFETY: This cast is intentionally unsafe - it's an escape hatch for users + // when validation is overly strict or incorrect. Runtime errors will surface + // if the config truly doesn't work. Future: explicit `allowUnvalidated` mode. + logger.warn( + 'Starting with validation warnings - some features may not work' + ); + validatedConfig = enrichedConfig as ValidatedAgentConfig; + } else { + // Validation failed and user didn't skip - show next steps and exit + safeExit('main', 1, 'config-validation-failed'); + } + + // Validate that if config specifies an image, it matches what was loaded + // Skip this check if user explicitly provided --image flag (intentional override) + // Note: Image was already loaded earlier before enrichment + if ( + !opts.image && + validatedConfig.image && + validatedConfig.image !== imageNameForEnrichment + ) { + console.error( + `❌ Config specifies image '${validatedConfig.image}' but '${imageNameForEnrichment}' was loaded instead` + ); + console.error( + `💡 Either remove 'image' from config or ensure it matches the loaded image` + ); + safeExit('main', 1, 'image-mismatch'); + } + } catch (err) { + if (err instanceof ExitSignal) throw err; + // Config loading failed completely + console.error(`❌ Failed to load configuration: ${err}`); + safeExit('main', 1, 'config-load-failed'); + } + + // ——— VALIDATE APPROVAL MODE COMPATIBILITY ——— + // Check if approval handler is needed (manual mode OR elicitation enabled) + const needsHandler = + validatedConfig.toolConfirmation?.mode === 'manual' || + validatedConfig.elicitation.enabled; + + if (needsHandler) { + // Headless CLI cannot do interactive approval + if (opts.mode === 'cli' && headlessInput) { + console.error( + '❌ Manual approval and elicitation are not supported in headless CLI mode (pipes/scripts).' + ); + console.error( + '💡 Use interactive CLI mode, or skip approvals by running `dexto --auto-approve` and disabling elicitation in your config.' + ); + console.error( + ' - toolConfirmation.mode: auto-approve (or auto-deny for strict denial policies)' + ); + console.error(' - elicitation.enabled: false'); + safeExit('main', 1, 'approval-unsupported-headless'); + } + + // Only web, server, and interactive CLI support approval handlers + // TODO: Add approval support for other modes: + // - Discord: Could use Discord message buttons/reactions for approval UI + // - Telegram: Could use Telegram inline keyboards for approval prompts + // - MCP: Could implement callback-based approval mechanism in MCP protocol + const supportedModes = ['web', 'server', 'cli']; + if (!supportedModes.includes(opts.mode)) { + console.error( + `❌ Manual approval and elicitation are not supported in "${opts.mode}" mode.` + ); + console.error( + `💡 These features require interactive UI and are only supported in: ${supportedModes.join( + ', ' + )}` + ); + console.error( + '💡 Run `dexto --auto-approve` or configure your agent to skip approvals when running headlessly.' + ); + console.error( + ' toolConfirmation.mode: auto-approve (or auto-deny if you want to deny certain tools)' + ); + console.error(' elicitation.enabled: false'); + safeExit('main', 1, 'approval-unsupported-mode'); + } + } + + // ——— CREATE AGENT ——— + let agent: DextoAgent; + let derivedAgentId: string; + try { + // Set run mode for tool confirmation provider + process.env.DEXTO_RUN_MODE = opts.mode; + + // Apply --strict flag to all server configs + if (opts.strict && validatedConfig.mcpServers) { + for (const [_serverName, serverConfig] of Object.entries( + validatedConfig.mcpServers + )) { + // All server config types have connectionMode field + serverConfig.connectionMode = 'strict'; + } + } + + // Config is already enriched and validated - ready for agent creation + // DextoAgent will parse/validate again (parse-twice pattern) + // isInteractiveMode is already defined above for validateAgentConfig + agent = new DextoAgent(validatedConfig, resolvedPath, { + strict: !isInteractiveMode, + }); + + // Start the agent (initialize async services) + // - web/server modes: initializeHonoApi will set approval handler and start the agent + // - cli mode: handles its own approval setup in the case block + // - other modes: start immediately (no approval support) + if (opts.mode !== 'web' && opts.mode !== 'server' && opts.mode !== 'cli') { + await agent.start(); + } + + // Derive a concise agent ID for display purposes (used by API/UI) + // Prefer agentCard.name, otherwise extract from filename + derivedAgentId = + validatedConfig.agentCard?.name || + path.basename(resolvedPath, path.extname(resolvedPath)); + } catch (err) { + if (err instanceof ExitSignal) throw err; + // Ensure config errors are shown to user, not hidden in logs + console.error(`❌ Configuration Error: ${(err as Error).message}`); + safeExit('main', 1, 'config-error'); + } + + // ——— Dispatch based on --mode ——— + // TODO: Refactor mode-specific logic into separate handler files + // This switch statement has grown large with nested if-else chains for each mode. + // Consider breaking down into mode-specific handlers (e.g., cli/modes/cli.ts, cli/modes/web.ts) + // to improve maintainability and reduce complexity in this entry point file. + // See PR 450 comment: https://github.com/truffle-ai/dexto/pull/450#discussion_r2546242983 + switch (opts.mode) { + case 'cli': { + // Set up approval handler for interactive CLI if manual mode OR elicitation enabled + // Note: Headless CLI with manual mode is blocked by validation above + const needsHandler = + !headlessInput && + (validatedConfig.toolConfirmation?.mode === 'manual' || + validatedConfig.elicitation.enabled); + + if (needsHandler) { + // CLI uses its own approval handler that works directly with AgentEventBus + // This avoids the indirection of ApprovalCoordinator (designed for HTTP flows) + const { createCLIApprovalHandler } = await import( + './cli/approval/index.js' + ); + const handler = createCLIApprovalHandler(agent.agentEventBus); + agent.setApprovalHandler(handler); + + logger.debug('CLI approval handler configured for Ink CLI'); + } + + // Start the agent now that approval handler is configured + await agent.start(); + + // Session management - CLI uses explicit sessionId like WebUI + // NOTE: Migrated from defaultSession pattern which will be deprecated in core + // We now pass sessionId explicitly to all agent methods (agent.run, agent.switchLLM, etc.) + + // Note: CLI uses different implementations for interactive vs headless modes + // - Interactive: Ink CLI with full TUI + // - Headless: CLISubscriber (no TUI, works in pipes/scripts) + + if (headlessInput) { + // Headless mode - isolated execution by default + // Each run gets a unique ephemeral session to avoid context bleeding between runs + // Use persistent session if explicitly resuming with -r or -c + let headlessSessionId: string; + + if (opts.resume) { + // Resume specific session by ID + try { + const session = await agent.getSession(opts.resume); + if (!session) { + console.error(`❌ Session '${opts.resume}' not found`); + console.error( + '💡 Use `dexto session list` to see available sessions' + ); + safeExit('main', 1, 'resume-failed'); + } + headlessSessionId = opts.resume; + logger.info( + `Resumed session: ${headlessSessionId}`, + null, + 'cyan' + ); + } catch (err) { + console.error( + `❌ Failed to resume session '${opts.resume}': ${err instanceof Error ? err.message : String(err)}` + ); + console.error( + '💡 Use `dexto session list` to see available sessions' + ); + safeExit('main', 1, 'resume-failed'); + } + } else if (opts.continue) { + // Continue most recent conversation (include headless sessions in headless mode) + const mostRecentSessionId = await getMostRecentSessionId( + agent, + true + ); + if (!mostRecentSessionId) { + console.error(`❌ No previous sessions found`); + console.error( + '💡 Start a new conversation or use `dexto session list` to see available sessions' + ); + safeExit('main', 1, 'no-sessions-found'); + } + headlessSessionId = mostRecentSessionId; + logger.info( + `Continuing most recent session: ${headlessSessionId}`, + null, + 'cyan' + ); + } else { + // TODO: Remove this workaround once defaultSession is deprecated in core + // Currently passing null/undefined to agent.run() falls back to defaultSession, + // causing context to bleed between headless runs. We create a unique ephemeral + // session ID to ensure each headless run is isolated. + // When defaultSession is removed, we can pass null here for truly stateless execution. + headlessSessionId = `headless-${Date.now()}-${Math.random().toString(36).substring(7)}`; + logger.debug( + `Created ephemeral session for headless run: ${headlessSessionId}` + ); + } + // Headless mode - use CLISubscriber for simple stdout output + const llm = agent.getCurrentLLMConfig(); + capture('dexto_prompt', { + mode: 'headless', + provider: llm.provider, + model: llm.model, + }); + + const { CLISubscriber } = await import('./cli/cli-subscriber.js'); + const cliSubscriber = new CLISubscriber(); + cliSubscriber.subscribe(agent.agentEventBus); + + try { + await cliSubscriber.runAndWait( + agent, + headlessInput, + headlessSessionId + ); + // Clean up before exit + cliSubscriber.cleanup(); + try { + await agent.stop(); + } catch (stopError) { + logger.debug(`Agent stop error (ignoring): ${stopError}`); + } + safeExit('main', 0); + } catch (error) { + // Rethrow ExitSignal - it's not an error, it's how safeExit works + if (error instanceof ExitSignal) throw error; + + // Write to stderr for headless users/scripts + const errorMessage = + error instanceof Error ? error.message : String(error); + console.error(`❌ Error in headless mode: ${errorMessage}`); + if (error instanceof Error && error.stack) { + console.error(error.stack); + } + + // Also log for diagnostics + logger.error(`Error in headless mode: ${errorMessage}`); + + cliSubscriber.cleanup(); + await agent.stop().catch(() => {}); // Best effort cleanup + safeExit('main', 1, 'headless-error'); + } + } else { + // Interactive mode - session management handled via /resume command + // Note: -c and -r flags are validated to require a prompt (headless mode only) + + // Check if API key is configured before trying to create session + // Session creation triggers LLM service init which requires API key + const llmConfig = agent.getCurrentLLMConfig(); + const { requiresApiKey } = await import('@dexto/core'); + if (requiresApiKey(llmConfig.provider) && !llmConfig.apiKey?.trim()) { + // Offer interactive API key setup instead of just exiting + const { interactiveApiKeySetup } = await import( + './cli/utils/api-key-setup.js' + ); + + console.log( + chalk.yellow( + `\n⚠️ API key required for provider '${llmConfig.provider}'\n` + ) + ); + + const setupResult = await interactiveApiKeySetup( + llmConfig.provider, + { + exitOnCancel: false, + model: llmConfig.model, + } + ); + + if (setupResult.cancelled) { + await agent.stop().catch(() => {}); + safeExit('main', 0, 'api-key-setup-cancelled'); + } + + if (setupResult.skipped) { + // User chose to skip - exit with instructions + await agent.stop().catch(() => {}); + safeExit('main', 0, 'api-key-pending'); + } + + if (setupResult.success && setupResult.apiKey) { + // API key was entered and saved - reload config and continue + // Update the agent's LLM config with the new API key + await agent.switchLLM({ + provider: llmConfig.provider, + model: llmConfig.model, + apiKey: setupResult.apiKey, + }); + logger.info('API key configured successfully, continuing...'); + } + } + + // Create session eagerly so slash commands work immediately + const session = await agent.createSession(); + const cliSessionId: string = session.id; + + // Check for updates (will be shown in Ink header) + const cliUpdateInfo = await versionCheckPromise; + + // Check if installed agents differ from bundled and prompt to sync + const needsSync = await shouldPromptForSync(pkg.version); + if (needsSync) { + const shouldSync = await p.confirm({ + message: 'Agent config updates available. Sync now?', + initialValue: true, + }); + + if (p.isCancel(shouldSync) || !shouldSync) { + await markSyncDismissed(pkg.version); + } else { + await handleSyncAgentsCommand({ force: true, quiet: true }); + await clearSyncDismissed(); + } + } + + // Interactive mode - use Ink CLI with session support + // Suppress console output before starting Ink UI + const originalConsole = { + log: console.log, + error: console.error, + warn: console.warn, + info: console.info, + }; + const noOp = () => {}; + console.log = noOp; + console.error = noOp; + console.warn = noOp; + console.info = noOp; + + let inkError: unknown = undefined; + try { + const { startInkCliRefactored } = await import( + './cli/ink-cli/InkCLIRefactored.js' + ); + await startInkCliRefactored(agent, cliSessionId, { + updateInfo: cliUpdateInfo ?? undefined, + }); + } catch (error) { + inkError = error; + } finally { + // Restore console methods so any errors are visible + console.log = originalConsole.log; + console.error = originalConsole.error; + console.warn = originalConsole.warn; + console.info = originalConsole.info; + } + + // Stop the agent after Ink CLI exits + try { + await agent.stop(); + } catch { + // Ignore shutdown errors + } + + // Handle any errors from Ink CLI + if (inkError) { + if (inkError instanceof ExitSignal) throw inkError; + const errorMessage = + inkError instanceof Error ? inkError.message : String(inkError); + console.error(`❌ Ink CLI failed: ${errorMessage}`); + if (inkError instanceof Error && inkError.stack) { + console.error(inkError.stack); + } + safeExit('main', 1, 'ink-cli-error'); + } + + safeExit('main', 0); + } + } + // falls through - safeExit returns never, but eslint doesn't know that + + case 'web': { + // Default to 3000 for web mode + const defaultPort = opts.port ? parseInt(opts.port, 10) : 3000; + const port = getPort(process.env.PORT, defaultPort, 'PORT'); + const serverUrl = process.env.DEXTO_URL ?? `http://localhost:${port}`; + + // Resolve webRoot path (embedded WebUI dist folder) + const webRoot = resolveWebRoot(); + if (!webRoot) { + console.warn(chalk.yellow('⚠️ WebUI not found in this build.')); + console.info('For production: Run "pnpm build:all" to embed the WebUI'); + console.info('For development: Run "pnpm dev" for hot reload'); + } + + // Build WebUI runtime config (analytics, etc.) for injection into index.html + const webUIConfig = webRoot + ? { analytics: await getWebUIAnalyticsConfig() } + : undefined; + + // Start single Hono server serving both API and WebUI + await startHonoApiServer( + agent, + port, + agent.config.agentCard || {}, + derivedAgentId, + webRoot, + webUIConfig + ); + + console.log(chalk.green(`✅ Server running at ${serverUrl}`)); + + // Show update notification if available + const webUpdateInfo = await versionCheckPromise; + if (webUpdateInfo) { + displayUpdateNotification(webUpdateInfo); + } + + // Open WebUI in browser if webRoot is available + if (webRoot) { + try { + const { default: open } = await import('open'); + await open(serverUrl, { wait: false }); + console.log( + chalk.green(`🌐 Opened WebUI in browser: ${serverUrl}`) + ); + } catch (_error) { + console.log(chalk.yellow(`💡 WebUI is available at: ${serverUrl}`)); + } + } + + break; + } + + // Start server with REST APIs and SSE on port 3001 + // This also enables dexto to be used as a remote mcp server at localhost:3001/mcp + case 'server': { + // Start server with REST APIs and SSE only + const agentCard = agent.config.agentCard ?? {}; + // Default to 3001 for server mode + const defaultPort = opts.port ? parseInt(opts.port, 10) : 3001; + const apiPort = getPort(process.env.PORT, defaultPort, 'PORT'); + const apiUrl = process.env.DEXTO_URL ?? `http://localhost:${apiPort}`; + + console.log('🌐 Starting server (REST APIs + SSE)...'); + await startHonoApiServer(agent, apiPort, agentCard, derivedAgentId); + console.log(`✅ Server running at ${apiUrl}`); + console.log('Available endpoints:'); + console.log(' POST /api/message - Send async message'); + console.log(' POST /api/message-sync - Send sync message'); + console.log(' POST /api/reset - Reset conversation'); + console.log(' GET /api/mcp/servers - List MCP servers'); + console.log(' SSE support available for real-time events'); + + // Show update notification if available + const serverUpdateInfo = await versionCheckPromise; + if (serverUpdateInfo) { + displayUpdateNotification(serverUpdateInfo); + } + break; + } + + // TODO: Remove if server mode is stable and supports mcp + // Starts dexto as a local mcp server + // Use `dexto --mode mcp` to start dexto as a local mcp server + // Use `dexto --mode server` to start dexto as a remote server + case 'mcp': { + // Start stdio mcp server only + const agentCardConfig = agent.config.agentCard || { + name: 'dexto', + version: '1.0.0', + }; + + try { + // Logs are already redirected to file by default to prevent interference with stdio transport + const agentCardData = createAgentCard( + { + defaultName: agentCardConfig.name ?? 'dexto', + defaultVersion: agentCardConfig.version ?? '1.0.0', + defaultBaseUrl: 'stdio://local-dexto', + }, + agentCardConfig // preserve overrides from agent file + ); + // Use stdio transport in mcp mode + const mcpTransport = await createMcpTransport('stdio'); + await initializeMcpServer(agent, agentCardData, mcpTransport); + } catch (err) { + // Write to stderr instead of stdout to avoid interfering with MCP protocol + process.stderr.write(`MCP server startup failed: ${err}\n`); + safeExit('main', 1, 'mcp-startup-failed'); + } + break; + } + + default: + if (opts.mode === 'discord' || opts.mode === 'telegram') { + console.error( + `❌ Error: '${opts.mode}' mode has been moved to examples` + ); + console.error(''); + console.error( + `The ${opts.mode} bot is now a standalone example that you can customize.` + ); + console.error(''); + console.error(`📖 See: examples/${opts.mode}-bot/README.md`); + console.error(''); + console.error(`To run it:`); + console.error(` cd examples/${opts.mode}-bot`); + console.error(` pnpm install`); + console.error(` pnpm start`); + } else { + console.error( + `❌ Unknown mode '${opts.mode}'. Use web, cli, server, or mcp.` + ); + } + safeExit('main', 1, 'unknown-mode'); + } + }, + { timeoutMs: 0 } + ) + ); + +// 17) PARSE & EXECUTE +program.parseAsync(process.argv); diff --git a/dexto/packages/cli/src/utils/agent-helpers.ts b/dexto/packages/cli/src/utils/agent-helpers.ts new file mode 100644 index 00000000..056dc6a0 --- /dev/null +++ b/dexto/packages/cli/src/utils/agent-helpers.ts @@ -0,0 +1,142 @@ +// packages/cli/src/utils/agent-helpers.ts + +import path from 'path'; +import { + AgentManager, + getDextoGlobalPath, + installBundledAgent as installBundledAgentCore, + installCustomAgent as installCustomAgentCore, + uninstallAgent as uninstallAgentCore, + listInstalledAgents as listInstalledAgentsCore, + type InstallOptions, + type AgentMetadata, +} from '@dexto/agent-management'; + +/** + * Singleton AgentManager instance for CLI commands + * Points to ~/.dexto/agents/registry.json + */ +let cliAgentManager: AgentManager | null = null; + +/** + * Get or create the CLI AgentManager instance + * + * This manager operates on the global agents directory (~/.dexto/agents) + * and uses the user registry file (~/.dexto/agents/registry.json). + * + * @returns AgentManager instance for CLI operations + * + * @example + * ```typescript + * const manager = getCLIAgentManager(); + * await manager.loadRegistry(); + * const agents = manager.listAgents(); + * ``` + */ +export function getCLIAgentManager(): AgentManager { + if (cliAgentManager === null) { + const registryPath = path.join(getDextoGlobalPath('agents'), 'registry.json'); + cliAgentManager = new AgentManager(registryPath); + } + return cliAgentManager; +} + +/** + * Reset the CLI AgentManager singleton (primarily for testing) + * + * @example + * ```typescript + * // In tests + * resetCLIAgentManager(); + * const manager = getCLIAgentManager(); // Creates fresh instance + * ``` + */ +export function resetCLIAgentManager(): void { + cliAgentManager = null; +} + +/** + * Install bundled agent from registry to ~/.dexto/agents + * + * @param agentId ID of the agent to install from bundled registry + * @param options Installation options (agentsDir defaults to ~/.dexto/agents) + * @returns Path to the installed agent's main config file + * + * @throws {Error} If agent not found in bundled registry or installation fails + * + * @example + * ```typescript + * await installBundledAgent('coding-agent'); + * console.log('Agent installed to ~/.dexto/agents/coding-agent'); + * ``` + */ +export async function installBundledAgent( + agentId: string, + options?: InstallOptions +): Promise { + return installBundledAgentCore(agentId, options); +} + +/** + * Install custom agent from local path to ~/.dexto/agents + * + * @param agentId Unique ID for the custom agent + * @param sourcePath Absolute path to agent YAML file or directory + * @param metadata Agent metadata (name, description, author, tags) + * @param options Installation options (agentsDir defaults to ~/.dexto/agents) + * @returns Path to the installed agent's main config file + * + * @throws {Error} If agent ID already exists or installation fails + * + * @example + * ```typescript + * await installCustomAgent('my-agent', '/path/to/agent.yml', { + * name: 'My Agent', + * description: 'Custom agent for my use case', + * author: 'John Doe', + * tags: ['custom'] + * }); + * ``` + */ +export async function installCustomAgent( + agentId: string, + sourcePath: string, + metadata: Pick, + options?: InstallOptions +): Promise { + return installCustomAgentCore(agentId, sourcePath, metadata, options); +} + +/** + * Uninstall agent by removing it from disk and user registry + * + * @param agentId ID of the agent to uninstall + * @param options Installation options (agentsDir defaults to ~/.dexto/agents) + * + * @throws {Error} If agent not found or uninstallation fails + * + * @example + * ```typescript + * await uninstallAgent('my-custom-agent'); + * console.log('Agent uninstalled'); + * ``` + */ +export async function uninstallAgent(agentId: string, options?: InstallOptions): Promise { + return uninstallAgentCore(agentId, options); +} + +/** + * List installed agents from ~/.dexto/agents + * + * @param options Installation options (agentsDir defaults to ~/.dexto/agents) + * @returns Array of installed agent IDs + * + * @example + * ```typescript + * const installed = await listInstalledAgents(); + * console.log(installed); // ['coding-agent', 'my-custom-agent'] + * ``` + */ +export async function listInstalledAgents(options?: InstallOptions): Promise { + return listInstalledAgentsCore(options); +} diff --git a/dexto/packages/cli/src/utils/env.test.ts b/dexto/packages/cli/src/utils/env.test.ts new file mode 100644 index 00000000..23007b69 --- /dev/null +++ b/dexto/packages/cli/src/utils/env.test.ts @@ -0,0 +1,434 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { tmpdir } from 'os'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; + +// Mock logger to prevent initialization issues +vi.mock('@core/logger/index.js', () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +// Mock agent-management to control execution context and env path behavior in tests +vi.mock('@dexto/agent-management', async () => { + const actual = + await vi.importActual('@dexto/agent-management'); + return { + ...actual, + getDextoEnvPath: vi.fn((startPath: string = process.cwd()) => { + return path.join(startPath, '.env'); + }), + getExecutionContext: vi.fn((startPath: string = process.cwd()) => { + const pkgPath = path.join(startPath, 'package.json'); + if (fs.existsSync(pkgPath)) { + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')); + if (pkg.name === 'dexto-monorepo' || pkg.name === 'dexto') { + return 'dexto-source'; + } + if (pkg.dependencies?.dexto || pkg.devDependencies?.dexto) { + return 'dexto-project'; + } + } + return 'global-cli'; + }), + ensureDextoGlobalDirectory: vi.fn(async () => { + // No-op in tests to avoid creating real directories + }), + }; +}); + +import { loadEnvironmentVariables, applyLayeredEnvironmentLoading } from './env.js'; +import { updateEnvFile } from '@dexto/agent-management'; + +function createTempDir() { + return fs.mkdtempSync(path.join(tmpdir(), 'dexto-env-test-')); +} + +function createTempDirStructure(structure: Record, baseDir?: string): string { + const tempDir = baseDir || createTempDir(); + + for (const [filePath, content] of Object.entries(structure)) { + const fullPath = path.join(tempDir, filePath); + const dir = path.dirname(fullPath); + + // Create directory if it doesn't exist + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + // Write file content + if (typeof content === 'string') { + fs.writeFileSync(fullPath, content, 'utf-8'); + } else { + fs.writeFileSync(fullPath, JSON.stringify(content, null, 2), 'utf-8'); + } + } + + return tempDir; +} + +function cleanupTempDir(dir: string) { + try { + if (fs.existsSync(dir)) { + fs.rmSync(dir, { recursive: true, force: true }); + } + } catch (error) { + // Ignore cleanup errors in tests to avoid masking real test failures + console.warn(`Failed to cleanup temp dir ${dir}:`, error); + } +} + +describe('Core Environment Loading', () => { + let originalEnv: Record; + let originalCwd: string; + + beforeEach(() => { + // Save original state + originalEnv = { ...process.env }; + originalCwd = process.cwd(); + + // Clean up environment variables that might interfere with tests + delete process.env.OPENAI_API_KEY; + delete process.env.ANTHROPIC_API_KEY; + delete process.env.GOOGLE_GENERATIVE_AI_API_KEY; + delete process.env.GROQ_API_KEY; + delete process.env.DEXTO_LOG_LEVEL; + delete process.env.TEST_VAR; + delete process.env.SHELL_ONLY; + delete process.env.CWD_ONLY; + delete process.env.PROJECT_ONLY; + delete process.env.SHELL_KEY; + delete process.env.NEW_VAR; + delete process.env.EXISTING_VAR; + delete process.env.DEFINED_VAR; + delete process.env.EMPTY_VAR; + delete process.env.UNDEFINED_VAR; + }); + + afterEach(() => { + // Restore original state + process.env = originalEnv; + + // Ensure we're back in the original directory + try { + process.chdir(originalCwd); + } catch { + // If original CWD was deleted, go to tmpdir + process.chdir(tmpdir()); + } + }); + + describe('loadEnvironmentVariables', () => { + it('loads from shell environment (highest priority)', async () => { + const tempDir = createTempDir(); + try { + // Set shell environment + process.env.OPENAI_API_KEY = 'shell-key'; + process.env.DEXTO_LOG_LEVEL = 'debug'; + + const env = await loadEnvironmentVariables(tempDir); + + expect(env.OPENAI_API_KEY).toBe('shell-key'); + expect(env.DEXTO_LOG_LEVEL).toBe('debug'); + } finally { + cleanupTempDir(tempDir); + } + }); + + it('loads from CWD .env', async () => { + // Create a CWD with .env + const cwdDir = createTempDir(); + createTempDirStructure( + { + '.env': 'OPENAI_API_KEY=cwd-key\nDEXTO_LOG_LEVEL=info', + }, + cwdDir + ); + + try { + process.chdir(cwdDir); + const env = await loadEnvironmentVariables(cwdDir); + + expect(env.OPENAI_API_KEY).toBe('cwd-key'); + expect(env.DEXTO_LOG_LEVEL).toBe('info'); + } finally { + process.chdir(originalCwd); + cleanupTempDir(cwdDir); + } + }); + + it('loads from project .env when in dexto project', async () => { + // Create dexto project structure + const projectDir = createTempDir(); + createTempDirStructure( + { + 'package.json': JSON.stringify({ dependencies: { dexto: '1.0.0' } }), + '.env': 'OPENAI_API_KEY=project-key\nDEXTO_LOG_LEVEL=warn', + }, + projectDir + ); + + try { + // Change to project directory so context detection works + process.chdir(projectDir); + const env = await loadEnvironmentVariables(projectDir); + + expect(env.OPENAI_API_KEY).toBe('project-key'); + expect(env.DEXTO_LOG_LEVEL).toBe('warn'); + } finally { + process.chdir(originalCwd); + cleanupTempDir(projectDir); + } + }); + + it('loads from source .env when in dexto-source', async () => { + // Create dexto source structure + const sourceDir = createTempDir(); + createTempDirStructure( + { + 'package.json': JSON.stringify({ name: 'dexto-monorepo', version: '1.0.0' }), + '.env': 'OPENAI_API_KEY=source-key\nDEXTO_LOG_LEVEL=error', + 'agents/default-agent.yml': 'mcpServers: {}', + }, + sourceDir + ); + + try { + // Change to source directory so context detection works + process.chdir(sourceDir); + const env = await loadEnvironmentVariables(sourceDir); + + expect(env.OPENAI_API_KEY).toBe('source-key'); + expect(env.DEXTO_LOG_LEVEL).toBe('error'); + } finally { + process.chdir(originalCwd); + cleanupTempDir(sourceDir); + } + }); + + it('handles priority system: Shell > CWD > Project', async () => { + // Create project directory + const projectDir = createTempDir(); + createTempDirStructure( + { + 'package.json': JSON.stringify({ dependencies: { dexto: '1.0.0' } }), + '.env': 'OPENAI_API_KEY=project-key\nANTHROPIC_API_KEY=project-anthropic\nPROJECT_ONLY=project-value', + }, + projectDir + ); + + // Create CWD directory + const cwdDir = createTempDir(); + createTempDirStructure( + { + '.env': 'OPENAI_API_KEY=cwd-key\nCWD_ONLY=cwd-value', + }, + cwdDir + ); + + try { + process.chdir(cwdDir); + + // Shell environment (highest) + process.env.OPENAI_API_KEY = 'shell-key'; + process.env.SHELL_ONLY = 'shell-value'; + + const env = await loadEnvironmentVariables(projectDir); + + // Shell wins + expect(env.OPENAI_API_KEY).toBe('shell-key'); + expect(env.SHELL_ONLY).toBe('shell-value'); + + // CWD wins over project + expect(env.CWD_ONLY).toBe('cwd-value'); + + // Project used when no override + expect(env.ANTHROPIC_API_KEY).toBe('project-anthropic'); + expect(env.PROJECT_ONLY).toBe('project-value'); + } finally { + process.chdir(originalCwd); + cleanupTempDir(projectDir); + cleanupTempDir(cwdDir); + } + }); + + it('handles missing .env files gracefully', async () => { + const projectDir = createTempDir(); + createTempDirStructure( + { + 'package.json': JSON.stringify({ dependencies: { dexto: '1.0.0' } }), + // No .env file + }, + projectDir + ); + + try { + process.chdir(projectDir); + process.env.SHELL_KEY = 'from-shell'; + const env = await loadEnvironmentVariables(projectDir); + + // Should still get shell variables + expect(env.SHELL_KEY).toBe('from-shell'); + } finally { + process.chdir(originalCwd); + cleanupTempDir(projectDir); + } + }); + + it('handles empty .env files', async () => { + const projectDir = createTempDir(); + createTempDirStructure( + { + 'package.json': JSON.stringify({ dependencies: { dexto: '1.0.0' } }), + '.env': '', + }, + projectDir + ); + + try { + process.chdir(projectDir); + // Shell environment should still work + process.env.OPENAI_API_KEY = 'shell-key'; + + const env = await loadEnvironmentVariables(projectDir); + + expect(env.OPENAI_API_KEY).toBe('shell-key'); + } finally { + process.chdir(originalCwd); + cleanupTempDir(projectDir); + } + }); + + it('filters out undefined and empty environment variables', async () => { + const tempDir = createTempDir(); + try { + process.env.DEFINED_VAR = 'value'; + process.env.EMPTY_VAR = ''; + process.env.UNDEFINED_VAR = undefined; + + const env = await loadEnvironmentVariables(tempDir); + + expect(env.DEFINED_VAR).toBe('value'); + expect('EMPTY_VAR' in env).toBe(false); // Empty string filtered out + expect('UNDEFINED_VAR' in env).toBe(false); + } finally { + cleanupTempDir(tempDir); + } + }); + }); + + describe('applyLayeredEnvironmentLoading', () => { + it('applies loaded environment to process.env', async () => { + const tempDir = createTempDir(); + createTempDirStructure( + { + '.env': 'NEW_VAR=new-value\nOPENAI_API_KEY=file-key', + }, + tempDir + ); + + try { + // Shell value should be preserved + process.env.OPENAI_API_KEY = 'shell-key'; + process.env.EXISTING_VAR = 'existing-value'; + + process.chdir(tempDir); + await applyLayeredEnvironmentLoading(tempDir); + + expect(process.env.OPENAI_API_KEY).toBe('shell-key'); // Shell wins + expect(process.env.EXISTING_VAR).toBe('existing-value'); // Shell preserved + expect(process.env.NEW_VAR).toBe('new-value'); // File vars added + } finally { + process.chdir(originalCwd); + cleanupTempDir(tempDir); + } + }); + + it('creates global .dexto directory if it does not exist', async () => { + const tempDir = createTempDir(); + try { + // This should not throw even if ~/.dexto doesn't exist + await applyLayeredEnvironmentLoading(tempDir); + } finally { + cleanupTempDir(tempDir); + } + }); + }); + + describe('updateEnvFile', () => { + it('creates .env file with variables', async () => { + const tempDir = createTempDir(); + const envPath = path.join(tempDir, '.env'); + + try { + await updateEnvFile(envPath, { + OPENAI_API_KEY: 'test-key', + DEXTO_LOG_LEVEL: 'debug', + }); + + const content = fs.readFileSync(envPath, 'utf-8'); + expect(content).toContain('OPENAI_API_KEY=test-key'); + expect(content).toContain('DEXTO_LOG_LEVEL=debug'); + } finally { + cleanupTempDir(tempDir); + } + }); + + it('updates existing variables in .env file', async () => { + const tempDir = createTempDir(); + const envPath = path.join(tempDir, '.env'); + + try { + // Create initial .env + fs.writeFileSync(envPath, 'OPENAI_API_KEY=old-key\nDEXTO_LOG_LEVEL=info'); + + // Update one variable + await updateEnvFile(envPath, { OPENAI_API_KEY: 'new-key' }); + + const content = fs.readFileSync(envPath, 'utf-8'); + expect(content).toContain('OPENAI_API_KEY=new-key'); + expect(content).toContain('DEXTO_LOG_LEVEL=info'); // Preserved + } finally { + cleanupTempDir(tempDir); + } + }); + + it('adds new variables to existing .env file', async () => { + const tempDir = createTempDir(); + const envPath = path.join(tempDir, '.env'); + + try { + // Create initial .env + fs.writeFileSync(envPath, 'OPENAI_API_KEY=test-key'); + + // Add new variable + await updateEnvFile(envPath, { ANTHROPIC_API_KEY: 'anthropic-key' }); + + const content = fs.readFileSync(envPath, 'utf-8'); + expect(content).toContain('OPENAI_API_KEY=test-key'); // Preserved + expect(content).toContain('ANTHROPIC_API_KEY=anthropic-key'); // Added + } finally { + cleanupTempDir(tempDir); + } + }); + + it('handles non-existent .env file', async () => { + const tempDir = createTempDir(); + const envPath = path.join(tempDir, '.env'); + + try { + await updateEnvFile(envPath, { OPENAI_API_KEY: 'test-key' }); + + expect(fs.existsSync(envPath)).toBe(true); + const content = fs.readFileSync(envPath, 'utf-8'); + expect(content).toContain('OPENAI_API_KEY=test-key'); + } finally { + cleanupTempDir(tempDir); + } + }); + }); +}); diff --git a/dexto/packages/cli/src/utils/env.ts b/dexto/packages/cli/src/utils/env.ts new file mode 100644 index 00000000..c7d439bb --- /dev/null +++ b/dexto/packages/cli/src/utils/env.ts @@ -0,0 +1,86 @@ +import * as path from 'path'; +import { homedir } from 'os'; +import dotenv from 'dotenv'; +import { + getExecutionContext, + ensureDextoGlobalDirectory, + getDextoEnvPath, +} from '@dexto/agent-management'; + +/** + * Multi-layer environment variable loading with context awareness. + * Loads environment variables in priority order: + * 1. Shell environment (highest priority) + * 2. Project .env (if in dexto project) + * 3. Global ~/.dexto/.env (fallback) + * + * @param startPath Starting directory for project detection + * @returns Combined environment variables object + */ +export async function loadEnvironmentVariables( + startPath: string = process.cwd() +): Promise> { + const context = getExecutionContext(startPath); + const env: Record = {}; + + const globalEnvPath = path.join(homedir(), '.dexto', '.env'); + try { + const globalResult = dotenv.config({ path: globalEnvPath, processEnv: {} }); + if (globalResult.parsed) { + Object.assign(env, globalResult.parsed); + } + } catch { + // Global .env is optional, ignore errors + } + + // Load .env from CWD if it exists (may differ from startPath) + const cwdEnvPath = path.join(process.cwd(), '.env'); + try { + const cwdResult = dotenv.config({ path: cwdEnvPath, processEnv: {} }); + if (cwdResult.parsed) { + Object.assign(env, cwdResult.parsed); + } + } catch { + // CWD .env is optional, ignore errors + } + + // For dexto projects, also load from project root (may differ from CWD) + if (context === 'dexto-source' || context === 'dexto-project') { + const projectEnvPath = getDextoEnvPath(startPath); + // Only load if different from cwdEnvPath to avoid double-loading + if (projectEnvPath !== cwdEnvPath) { + try { + const projectResult = dotenv.config({ path: projectEnvPath, processEnv: {} }); + if (projectResult.parsed) { + Object.assign(env, projectResult.parsed); + } + } catch { + // Project .env is optional, ignore errors + } + } + } + + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined && value !== '') { + env[key] = value; + } + } + + return env; +} + +/** + * Apply layered environment loading to process.env. + * This replaces the simple dotenv.config() with multi-layer loading. + * Should be called at CLI startup before any schema validation. + * + * @param startPath Starting directory for project detection + */ +export async function applyLayeredEnvironmentLoading( + startPath: string = process.cwd() +): Promise { + await ensureDextoGlobalDirectory(); + + const layeredEnv = await loadEnvironmentVariables(startPath); + Object.assign(process.env, layeredEnv); +} diff --git a/dexto/packages/cli/src/utils/graceful-shutdown.ts b/dexto/packages/cli/src/utils/graceful-shutdown.ts new file mode 100644 index 00000000..0449f264 --- /dev/null +++ b/dexto/packages/cli/src/utils/graceful-shutdown.ts @@ -0,0 +1,129 @@ +import type { DextoAgent } from '@dexto/core'; +import { logger } from '@dexto/core'; + +export interface GracefulShutdownOptions { + /** + * When true, the first SIGINT is ignored to let the application handle it + * (e.g., for Ink CLI which needs to handle Ctrl+C for cancellation/exit warning). + * A second SIGINT within the timeout will force exit. + */ + inkMode?: boolean; + /** + * Timeout in ms before force exit in ink mode (default: 3000ms) + */ + forceExitTimeout?: number; +} + +export function registerGracefulShutdown( + getCurrentAgent: () => DextoAgent, + options: GracefulShutdownOptions = {} +): void { + const { inkMode = false, forceExitTimeout = 3000 } = options; + const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGUSR2']; + + // For SIGINT, handle separately based on mode + if (!inkMode) { + signals.push('SIGINT'); + } + + let isShuttingDown = false; + + const performShutdown = async (signal: string) => { + if (isShuttingDown) return; + isShuttingDown = true; + + logger.info(`Received ${signal}, shutting down gracefully...`); + try { + const agent = getCurrentAgent(); + await agent.stop(); + process.exit(0); + } catch (error) { + logger.error( + `Shutdown error: ${error instanceof Error ? error.message : String(error)}`, + { error } + ); + process.exit(1); + } + }; + + signals.forEach((signal) => { + process.on(signal, () => performShutdown(signal)); + }); + + // In ink mode, handle SIGINT specially - allow first one to pass through to Ink + if (inkMode) { + let firstSigintTime: number | null = null; + + process.on('SIGINT', () => { + const now = Date.now(); + + // If already shutting down, ignore + if (isShuttingDown) return; + + // First SIGINT - record time and let Ink handle it + if (firstSigintTime === null) { + firstSigintTime = now; + + // Set timeout to clear the "first sigint" state + setTimeout(() => { + if ( + firstSigintTime !== null && + Date.now() - firstSigintTime >= forceExitTimeout + ) { + firstSigintTime = null; + } + }, forceExitTimeout); + + // Don't exit - let Ink handle it + return; + } + + // Second SIGINT within timeout - force exit + if (now - firstSigintTime < forceExitTimeout) { + void performShutdown('SIGINT (force)'); + } else { + // Timeout expired, treat as new first SIGINT + firstSigintTime = now; + } + }); + } + + // Handle uncaught exceptions + process.on('uncaughtException', async (error) => { + logger.error( + `Uncaught exception: ${error instanceof Error ? error.message : String(error)}`, + { error }, + 'red' + ); + if (!isShuttingDown) { + isShuttingDown = true; + try { + const agent = getCurrentAgent(); + await agent.stop(); + } catch (innerError) { + logger.error( + `Error during shutdown initiated by uncaughtException: ${innerError instanceof Error ? innerError.message : String(innerError)}`, + { error: innerError } + ); + } + } + process.exit(1); + }); + + process.on('unhandledRejection', async (reason) => { + logger.error(`Unhandled rejection: ${reason}`, { reason }, 'red'); + if (!isShuttingDown) { + isShuttingDown = true; + try { + const agent = getCurrentAgent(); + await agent.stop(); + } catch (innerError) { + logger.error( + `Error during shutdown initiated by unhandledRejection: ${innerError instanceof Error ? innerError.message : String(innerError)}`, + { error: innerError } + ); + } + } + process.exit(1); + }); +} diff --git a/dexto/packages/cli/src/utils/port-utils.spec.ts b/dexto/packages/cli/src/utils/port-utils.spec.ts new file mode 100644 index 00000000..1a0ac8d0 --- /dev/null +++ b/dexto/packages/cli/src/utils/port-utils.spec.ts @@ -0,0 +1,30 @@ +import { describe, it, expect } from 'vitest'; +import { getPort } from './port-utils.js'; + +describe('getPort', () => { + it('returns default if envVar is undefined', () => { + expect(getPort(undefined, 3000, 'TEST')).toBe(3000); + }); + + it('parses valid numeric string', () => { + expect(getPort('8080', 3000, 'TEST')).toBe(8080); + }); + + it('throws error for non-numeric string', () => { + expect(() => getPort('not-a-number', 3000, 'TEST')).toThrow( + 'Environment variable TEST value "not-a-number" is not a valid port' + ); + }); + + it('throws error for negative port', () => { + expect(() => getPort('-1', 3000, 'TEST')).toThrow( + 'Environment variable TEST value "-1" is not a valid port' + ); + }); + + it('throws error for port > 65535', () => { + expect(() => getPort('70000', 3000, 'TEST')).toThrow( + 'Environment variable TEST value "70000" is not a valid port' + ); + }); +}); diff --git a/dexto/packages/cli/src/utils/port-utils.ts b/dexto/packages/cli/src/utils/port-utils.ts new file mode 100644 index 00000000..4f98c1f0 --- /dev/null +++ b/dexto/packages/cli/src/utils/port-utils.ts @@ -0,0 +1,18 @@ +/** + * Parse an environment variable as a port number, with validation. + * @param envVar - the string value from process.env + * @param defaultPort - fallback port if envVar is undefined + * @param varName - the name of the environment variable (used in error messages) + * @returns a valid port number + * @throws if envVar is set but not a valid port number + */ +export function getPort(envVar: string | undefined, defaultPort: number, varName: string): number { + if (envVar === undefined) { + return defaultPort; + } + const port = parseInt(envVar, 10); + if (isNaN(port) || port <= 0 || port > 65535) { + throw new Error(`Environment variable ${varName} value "${envVar}" is not a valid port`); + } + return port; +} diff --git a/dexto/packages/cli/src/web.ts b/dexto/packages/cli/src/web.ts new file mode 100644 index 00000000..70019069 --- /dev/null +++ b/dexto/packages/cli/src/web.ts @@ -0,0 +1,33 @@ +/** + * Resolves the webRoot path for serving WebUI static files. + * + * In production builds, the WebUI dist is embedded at packages/cli/dist/webui. + * This function returns the absolute path if found, otherwise undefined. + */ +import { existsSync } from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +/** + * Discovers the webui path for embedded Vite build. + * @returns Absolute path to webui dist folder, or undefined if not found + */ +export function resolveWebRoot(): string | undefined { + // Path discovery logic for the built webui + const scriptDir = path.dirname(fileURLToPath(import.meta.url)); + + // Look for embedded webui in CLI's dist folder + const webuiPath = path.resolve(scriptDir, 'webui'); + + if (!existsSync(webuiPath)) { + return undefined; + } + + // Verify index.html exists (Vite output) + const indexPath = path.join(webuiPath, 'index.html'); + if (!existsSync(indexPath)) { + return undefined; + } + + return webuiPath; +} diff --git a/dexto/packages/cli/tsconfig.json b/dexto/packages/cli/tsconfig.json new file mode 100644 index 00000000..f5866a22 --- /dev/null +++ b/dexto/packages/cli/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": false, + "jsx": "react-jsx", + "paths": { + "@dexto/core": ["../core/dist/index.d.ts"], + "@dexto/core/*": ["../core/dist/*"], + "@core": ["../core/dist/index.d.ts"], + "@core/*": ["../core/dist/*"], + "@dexto/agent-management": ["../agent-management/dist/index.d.ts"], + "@dexto/analytics": ["../analytics/dist/index.d.ts"], + "@dexto/server": ["../server/dist/index.d.ts"] + } + }, + "include": ["src/**/*"], + "exclude": ["**/*.test.ts", "**/*.spec.ts", "dist", "node_modules"] +} diff --git a/dexto/packages/cli/tsconfig.typecheck.json b/dexto/packages/cli/tsconfig.typecheck.json new file mode 100644 index 00000000..90a93a82 --- /dev/null +++ b/dexto/packages/cli/tsconfig.typecheck.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["dist", "node_modules"] +} diff --git a/dexto/packages/client-sdk/CHANGELOG.md b/dexto/packages/client-sdk/CHANGELOG.md new file mode 100644 index 00000000..522c479c --- /dev/null +++ b/dexto/packages/client-sdk/CHANGELOG.md @@ -0,0 +1,186 @@ +# @dexto/client-sdk + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +## 1.5.5 + +## 1.5.4 + +## 1.5.3 + +## 1.5.2 + +## 1.5.1 + +## 1.5.0 + +### Minor Changes + +- e7722e5: Minor version bump for new release with bundler, custom tool pkgs, etc. + +## 1.4.0 + +## 1.3.0 + +### Minor Changes + +- eb266af: Migrate WebUI from next-js to vite. Fix any typing in web UI. Improve types in core. minor renames in event schemas + +## 1.2.6 + +## 1.2.5 + +### Patch Changes + +- 8f373cc: Migrate server API to Hono framework with feature flag + - Migrated Express server to Hono with OpenAPI schema generation + - Added DEXTO_USE_HONO environment variable flag (default: false for backward compatibility) + - Fixed WebSocket test isolation by adding sessionId filtering + - Fixed logger context to pass structured objects instead of stringified JSON + - Fixed CI workflow for OpenAPI docs synchronization + - Updated documentation links and fixed broken API references + +- f28ad7e: Migrate webUI to use client-sdk, add agents.md file to webui,improve types in apis for consumption +- a35a256: Migrate from WebSocket to Server-Sent Events (SSE) for real-time streaming + - Replace WebSocket with SSE for message streaming via new `/api/message-stream` endpoint + - Refactor approval system from event-based providers to simpler handler pattern + - Add new APIs for session approval + - Move session title generation to a separate API + - Add `ApprovalCoordinator` for multi-client SSE routing with sessionId mapping + - Add stream and generate methods to DextoAgent and integ tests for itq= + +- 5a26bdf: Update hono server to chain apis to keep type info, update client sdk to be fully typed +- ac649fd: Fix error handling and UI bugs, add gpt-5.1, gemini-3 + +## 1.2.4 + +### Patch Changes + +- cd706e7: bump up version after fixing node-machine-id +- Updated dependencies [cd706e7] + - @dexto/core@1.2.4 + +## 1.2.3 + +### Patch Changes + +- 5d6ae73: Bump up version to fix bugs +- Updated dependencies [5d6ae73] + - @dexto/core@1.2.3 + +## 1.2.2 + +### Patch Changes + +- @dexto/core@1.2.2 + +## 1.2.1 + +### Patch Changes + +- @dexto/core@1.2.1 + +## 1.2.0 + +### Patch Changes + +- Updated dependencies [b51e4d9] +- Updated dependencies [a27ddf0] +- Updated dependencies [155813c] +- Updated dependencies [1e25f91] +- Updated dependencies [3a65cde] +- Updated dependencies [5ba5d38] +- Updated dependencies [930a4ca] +- Updated dependencies [ecad345] +- Updated dependencies [930d75a] + - @dexto/core@1.2.0 + +## 1.1.11 + +### Patch Changes + +- 01167a2: Refactors +- Updated dependencies [c40b675] +- Updated dependencies [015100c] +- Updated dependencies [0760f8a] +- Updated dependencies [5cc6933] +- Updated dependencies [40f89f5] +- Updated dependencies [3a24d08] +- Updated dependencies [01167a2] +- Updated dependencies [a53b87a] +- Updated dependencies [24e5093] +- Updated dependencies [c695e57] +- Updated dependencies [0700f6f] +- Updated dependencies [0a5636c] +- Updated dependencies [35d48c5] + - @dexto/core@1.1.11 + +## 1.1.10 + +### Patch Changes + +- @dexto/core@1.1.10 + +## 1.1.9 + +### Patch Changes + +- Updated dependencies [27778ba] + - @dexto/core@1.1.9 + +## 1.1.8 + +### Patch Changes + +- Updated dependencies [d79d358] + - @dexto/core@1.1.8 + +## 1.1.7 + +### Patch Changes + +- @dexto/core@1.1.7 + +## 1.1.6 + +### Patch Changes + +- @dexto/core@1.1.6 + +## 1.1.5 + +### Patch Changes + +- 795c7f1: feat: Add @dexto/client-sdk package + - New lightweight cross-environment client SDK + - HTTP + optional WebSocket support for messaging + - Streaming and non-streaming message support + - Session management, LLM config/catalog access + - MCP tools integration and search functionality + - Real-time events support + - Comprehensive TypeScript types and validation + - Unit tests and documentation included + +- 09b8e33: Fix minor comments +- Updated dependencies [e2bd0ce] +- Updated dependencies [11cbec0] +- Updated dependencies [795c7f1] +- Updated dependencies [9d7541c] + - @dexto/core@1.1.5 diff --git a/dexto/packages/client-sdk/README.md b/dexto/packages/client-sdk/README.md new file mode 100644 index 00000000..33ab201b --- /dev/null +++ b/dexto/packages/client-sdk/README.md @@ -0,0 +1,107 @@ +# Dexto Client SDK + +An ultra-lightweight, zero-dependency HTTP client SDK for the Dexto API. + +## Features + +- 🚀 **Ultra-lightweight**: Only 80KB bundle size +- 🔌 **Zero dependencies**: No external libraries +- 🌐 **Universal**: Works in Node.js, browsers, and React Native +- 📡 **HTTP + SSE**: Full REST API and real-time SSE streaming support +- 🛡️ **TypeScript**: Full type safety +- 🔄 **Auto-retry**: Built-in retry logic with exponential backoff +- ⚡ **Fast**: Server-side validation, client-side pass-through + +## Installation + +```bash +npm install @dexto/client-sdk +``` + +## Quick Start + +```typescript +import { DextoClient } from '@dexto/client-sdk'; + +const client = new DextoClient({ + baseUrl: 'https://your-dexto-server.com', + apiKey: 'optional-api-key' +}); + +// Connect to Dexto server +await client.connect(); + +// Send a message +const response = await client.sendMessage({ + content: 'Hello, how can you help me?' +}); + +console.log(response.response); +``` + +## Configuration + +```typescript +const client = new DextoClient({ + baseUrl: 'https://your-dexto-server.com', // Required: Dexto API base URL + apiKey: 'your-api-key', // Optional: API key for auth + timeout: 30000, // Optional: Request timeout (ms) + retries: 3, // Optional: Retry attempts +}, { + reconnect: true, // Optional: Auto-reconnect + reconnectInterval: 5000, // Optional: Reconnect delay (ms) + debug: false // Optional: Debug logging +}); +``` + +## API Methods + +### Connection Management +- `connect()` - Establish connection to Dexto server +- `disconnect()` - Close connection +- `isConnected` - Check connection status + +### Messaging +- `sendMessage(input)` - Send message (HTTP) +- SSE streaming available via `/api/message-stream` endpoint + +### Session Management +- `listSessions()` - List all sessions +- `createSession(id?)` - Create new session +- `getSession(id)` - Get session details +- `deleteSession(id)` - Delete session + +### Real-time Events +- `on(eventType, handler)` - Subscribe to Dexto events +- `onConnectionState(handler)` - Connection state changes + +## Error Handling + +```typescript +try { + await client.sendMessage({ content: 'Hello' }); +} catch (error) { + if (error.name === 'ConnectionError') { + console.log('Failed to connect to Dexto server'); + } else if (error.name === 'HttpError') { + console.log(`HTTP ${error.status}: ${error.statusText}`); + } +} +``` + +## Philosophy + +This SDK follows the **thin client** philosophy: + +- ✅ **Pass-through**: Data goes directly to Dexto server +- ✅ **Server validation**: Let the Dexto server handle all validation +- ✅ **Simple errors**: Return server errors as-is +- ✅ **Type safety**: Full TypeScript support +- ✅ **Fast**: Minimal client-side processing + +## Bundle Size + +- **Total**: ~80KB +- **Main bundle**: ~25KB +- **Type definitions**: ~10KB +- **Zero external dependencies** diff --git a/dexto/packages/client-sdk/package.json b/dexto/packages/client-sdk/package.json new file mode 100644 index 00000000..c6e1459a --- /dev/null +++ b/dexto/packages/client-sdk/package.json @@ -0,0 +1,44 @@ +{ + "name": "@dexto/client-sdk", + "version": "1.5.6", + "private": false, + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.cjs", + "default": "./dist/index.js" + }, + "./package.json": "./package.json" + }, + "dependencies": { + "hono": "^4.6.14" + }, + "devDependencies": { + "@dexto/core": "workspace:*", + "@dexto/server": "workspace:*", + "tsup": "^8.0.2", + "typescript": "^5.4.2", + "vitest": "^1.3.1" + }, + "scripts": { + "build": "tsup && tsc -p tsconfig.json --emitDeclarationOnly", + "dev": "tsup --watch", + "typecheck": "tsc -p tsconfig.typecheck.json --noEmit", + "lint": "eslint . --ext .ts,.tsx", + "test": "vitest run", + "test:watch": "vitest watch" + }, + "files": [ + "dist", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "sideEffects": false +} \ No newline at end of file diff --git a/dexto/packages/client-sdk/src/client.ts b/dexto/packages/client-sdk/src/client.ts new file mode 100644 index 00000000..cdce370b --- /dev/null +++ b/dexto/packages/client-sdk/src/client.ts @@ -0,0 +1,99 @@ +import { hc } from 'hono/client'; +import type { AppType } from '@dexto/server'; +import type { ClientConfig } from './types.js'; + +/** + * Create a type-safe Dexto client using Hono's typed client + * + * @example + * ```typescript + * import { createDextoClient } from '@dexto/client-sdk'; + * + * const client = createDextoClient({ + * baseUrl: 'http://localhost:3001', + * apiKey: 'optional-api-key' + * }); + * + * // Create a session + * const session = await client.api.sessions.$post({ + * json: { sessionId: 'my-session' } + * }); + * + * // Send a synchronous message + * const response = await client.api['message-sync'].$post({ + * json: { message: 'Hello!', sessionId: 'my-session' } + * }); + * const { response: text } = await response.json(); + * + * // Search messages + * const searchResults = await client.api.search.messages.$get({ + * query: { q: 'hello', limit: 10 } + * }); + * + * // Streaming responses with SSE + * import { createMessageStream } from '@dexto/client-sdk'; + * + * const streamPromise = client.api['message-stream'].$post({ + * json: { message: 'Tell me a story', sessionId: 'my-session' } + * }); + * + * // Parse SSE events using createMessageStream + * const stream = createMessageStream(streamPromise); + * for await (const event of stream) { + * if (event.name === 'llm:chunk') { + * process.stdout.write(event.content); + * } + * } + * ``` + */ +export function createDextoClient(config: ClientConfig) { + const options: { headers?: Record } = {}; + + if (config.apiKey) { + options.headers = { + Authorization: `Bearer ${config.apiKey}`, + }; + } + + return hc(config.baseUrl, options); +} + +export * from './streaming.js'; + +// AGENTS DO NOT DELETE THE BELOW COMMENTS +// Uncomment for testing autofill in IDE +// const client1 = hc('http://localhost:3001'); +// let response1 = await client1.api.search.sessions.$get({ +// query: { +// q: "poop" +// } +// }) +// const client2 = createDextoClient({ +// baseUrl: 'http://localhost:3001', +// apiKey: 'optional-api-key' +// }) +// let response2 = await client2.api.sessions.$post({ +// json: { +// sessionId: 'session-123' +// } +// }) +// const body2 = await response2.json(); +// console.log(body2.session.id); + +// let response3 = await client2.health.$get(); +// console.log(response3.ok); + +// import { createMessageStream } from './streaming.js'; +// let response4 = client2.api['message-stream'].$post({ +// json: { +// message: 'Tell me a story', +// sessionId: 'my-session' +// } +// }); + +// const stream = createMessageStream(response4); +// for await (const event of stream) { +// if (event.type === 'llm:chunk') { +// process.stdout.write(event.content); +// } +// } diff --git a/dexto/packages/client-sdk/src/index.ts b/dexto/packages/client-sdk/src/index.ts new file mode 100644 index 00000000..7b05dcb5 --- /dev/null +++ b/dexto/packages/client-sdk/src/index.ts @@ -0,0 +1,17 @@ +/** + * Dexto Client SDK + * Lightweight type-safe client for Dexto API built on Hono's typed client + */ + +// Core client +export { createDextoClient } from './client.js'; + +// SSE streaming +export { stream, createStream, createMessageStream, SSEError } from './streaming.js'; +export type { SSEEvent, MessageStreamEvent } from './streaming.js'; + +// Client configuration +export type { ClientConfig } from './types.js'; + +// Server types for advanced usage +export type { AppType } from '@dexto/server'; diff --git a/dexto/packages/client-sdk/src/streaming.ts b/dexto/packages/client-sdk/src/streaming.ts new file mode 100644 index 00000000..bb233f09 --- /dev/null +++ b/dexto/packages/client-sdk/src/streaming.ts @@ -0,0 +1,197 @@ +import type { StreamingEvent } from '@dexto/core'; + +/** + * SSE (Server-Sent Events) streaming utilities for client SDK + * Adapted from @dexto/webui EventStreamClient + */ + +export type MessageStreamEvent = StreamingEvent; + +export interface SSEEvent { + event?: string; + data?: T; + id?: string; + retry?: number; +} + +export class SSEError extends Error { + constructor( + public status: number, + public body: any + ) { + super(`SSE Error: ${status}`); + this.name = 'SSEError'; + } +} + +/** + * Creates an async generator that yields SSE events from a Response object. + * + * @param response The fetch Response object containing the SSE stream + * @param options Optional configuration including AbortSignal + */ +export async function* stream( + response: Response, + options?: { signal?: AbortSignal } +): AsyncGenerator> { + if (!response.ok) { + const contentType = response.headers.get('content-type'); + let errorBody; + try { + if (contentType && contentType.includes('application/json')) { + errorBody = await response.json(); + } else { + errorBody = await response.text(); + } + } catch { + errorBody = 'Unknown error'; + } + throw new SSEError(response.status, errorBody); + } + + const reader = response.body?.getReader(); + if (!reader) { + throw new Error('Response body is null'); + } + + const decoder = new globalThis.TextDecoder(); + let buffer = ''; + const signal = options?.signal; + + // Handle abort signal + let aborted = false; + const abortHandler = () => { + aborted = true; + reader.cancel().catch(() => { + // Ignore errors during cancel + }); + }; + + if (signal) { + if (signal.aborted) { + reader.cancel().catch(() => {}); + return; + } + signal.addEventListener('abort', abortHandler); + } + + try { + while (true) { + if (aborted || signal?.aborted) { + return; + } + + // Check if we have a double newline in buffer + const parts = buffer.split('\n\n'); + if (parts.length > 1) { + const eventString = parts.shift()!; + buffer = parts.join('\n\n'); + + const event = parseSSE(eventString); + if (event) { + yield event as SSEEvent; + } + continue; + } + + // Need more data + const { done, value } = await reader.read(); + if (done) { + if (buffer.trim()) { + const event = parseSSE(buffer); + if (event) { + yield event as SSEEvent; + } + } + return; + } + + // Normalize CRLF to LF for spec-compliance + buffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, '\n'); + } + } finally { + if (signal) { + signal.removeEventListener('abort', abortHandler); + } + reader.cancel().catch(() => {}); + } +} + +/** + * Helper to create a stream from a promise that resolves to a Response. + * Useful for chaining with Hono client requests. + * + * @example + * ```typescript + * const stream = createStream(client.api.chat.$post({ json: { message: 'hi' } })); + * for await (const event of stream) { ... } + * ``` + */ +export async function* createStream( + responsePromise: Promise, + options?: { signal?: AbortSignal } +): AsyncGenerator> { + const response = await responsePromise; + yield* stream(response, options); +} + +/** + * Helper to create a typed message stream from a promise that resolves to a Response. + * Automatically parses JSON data and yields typed MessageStreamEvent objects. + * + * @example + * ```typescript + * const stream = createMessageStream(client.api['message-stream'].$post({ ... })); + * for await (const event of stream) { + * if (event.name === 'llm:chunk') { + * console.log(event.content); + * } + * } + * ``` + */ +export async function* createMessageStream( + responsePromise: Promise, + options?: { signal?: AbortSignal } +): AsyncGenerator { + const sseStream = createStream(responsePromise, options); + for await (const event of sseStream) { + if (event.data) { + try { + const parsed = JSON.parse(event.data); + // SSE event name becomes the 'name' discriminant for StreamingEvent + // Using 'name' (not 'type') to avoid collision with payload fields + if (event.event) { + parsed.name = event.event; + } + yield parsed as MessageStreamEvent; + } catch { + // Ignore parse errors for non-JSON data + } + } + } +} + +function parseSSE(raw: string): SSEEvent | null { + const lines = raw.split('\n').map((line) => line.replace(/\r$/, '')); + const event: SSEEvent = {}; + let hasData = false; + + for (const line of lines) { + if (line.startsWith(':')) continue; // Comment + + if (line.startsWith('data: ')) { + const data = line.slice(6); + event.data = event.data ? event.data + '\n' + data : data; + hasData = true; + } else if (line.startsWith('event: ')) { + event.event = line.slice(7); + } else if (line.startsWith('id: ')) { + event.id = line.slice(4); + } else if (line.startsWith('retry: ')) { + event.retry = parseInt(line.slice(7), 10); + } + } + + if (!hasData && !event.event && !event.id) return null; + return event; +} diff --git a/dexto/packages/client-sdk/src/types.ts b/dexto/packages/client-sdk/src/types.ts new file mode 100644 index 00000000..ca43597c --- /dev/null +++ b/dexto/packages/client-sdk/src/types.ts @@ -0,0 +1,9 @@ +/** + * Client configuration for Dexto SDK + */ +export interface ClientConfig { + /** Base URL of the Dexto server */ + baseUrl: string; + /** Optional API key for authentication */ + apiKey?: string; +} diff --git a/dexto/packages/client-sdk/tsconfig.json b/dexto/packages/client-sdk/tsconfig.json new file mode 100644 index 00000000..e292b0fc --- /dev/null +++ b/dexto/packages/client-sdk/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": false + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "**/*.test.ts", + "**/*.spec.ts", + "**/*.integration.test.ts", + "dist", + "node_modules" + ] +} diff --git a/dexto/packages/client-sdk/tsconfig.typecheck.json b/dexto/packages/client-sdk/tsconfig.typecheck.json new file mode 100644 index 00000000..90a93a82 --- /dev/null +++ b/dexto/packages/client-sdk/tsconfig.typecheck.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["dist", "node_modules"] +} diff --git a/dexto/packages/client-sdk/tsup.config.ts b/dexto/packages/client-sdk/tsup.config.ts new file mode 100644 index 00000000..054222d1 --- /dev/null +++ b/dexto/packages/client-sdk/tsup.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: ['src/index.ts'], + format: ['cjs', 'esm'], + outDir: 'dist', + dts: false, // Disable DTS generation in tsup to avoid worker memory issues + shims: true, + bundle: true, + platform: 'neutral', + target: 'es2018', + minify: false, + splitting: false, + treeshake: false, + clean: true, + sourcemap: false, +}); diff --git a/dexto/packages/client-sdk/vitest.config.ts b/dexto/packages/client-sdk/vitest.config.ts new file mode 100644 index 00000000..8c2e1588 --- /dev/null +++ b/dexto/packages/client-sdk/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'node', + }, +}); diff --git a/dexto/packages/core/CHANGELOG.md b/dexto/packages/core/CHANGELOG.md new file mode 100644 index 00000000..d3ea6851 --- /dev/null +++ b/dexto/packages/core/CHANGELOG.md @@ -0,0 +1,399 @@ +# @dexto/core + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +## 1.5.5 + +### Patch Changes + +- 63fa083: Session and context management fixes: + - Remove continuation session logic after compaction, now sticks to same session + - `/clear` continues same session and resets context (frees up AI context window) + - `/new` command creates new session with fresh context and clears screen + - Add context tokens remaining to footer, align context calculations everywhere + - Fix context calculation logic by including cache read tokens + + Other improvements: + - Fix code block syntax highlighting in terminal (uses cli-highlight) + - Make terminal the default mode during onboarding + - Reduce OTEL dependency bloat by replacing auto-instrumentation with specific packages (47 MB saved: 65 MB → 18 MB) + +- 6df3ca9: Updated readme. Removed stale filesystem and process tool from dexto/core. + +## 1.5.4 + +### Patch Changes + +- 0016cd3: Bug fixes and updates for compaction. Also added UI enhancements for compaction. +- 499b890: Fix model override persistence after compaction and improve context token tracking + + **Bug Fixes:** + - Fix model override resetting to config model after compaction (now respects session overrides) + + **Context Tracking Improvements:** + - New algorithm uses actual `input_tokens` and `output_tokens` from LLM responses as source of truth + - Self-correcting estimates: inaccuracies auto-correct when next LLM response arrives + - Handles pruning automatically (next response's input_tokens reflects pruned state) + - `/context` and compaction decisions now share common calculation logic + - Removed `outputBuffer` concept in favor of single configurable threshold + - Default compaction threshold lowered to 90% + + **New `/context` Command:** + - Interactive overlay with stacked token bar visualization + - Breakdown by component: system prompt, tools, messages, free space, auto-compact buffer + - Expandable per-tool token details + - Shows pruned tool count and compaction history + + **Observability:** + - Comparison logging between estimated vs actual tokens for calibration + - `dexto_llm_tokens_consumed` metric now includes estimated input tokens and accuracy metrics + +- aa2c9a0: - new --dev flag for using dev mode with the CLI (for maintainers) (sets DEXTO_DEV_MODE=true and ensures local files are used) + - improved bash tool descriptions + - fixed explore agent task description getting truncated + - fixed some alignment issues + - fix search/find tools not asking approval for working outside directory + - add sound feature (sounds when approval reqd, when loop done) + - configurable in `preferences.yml` (on by default) and in `~/.dexto/sounds`, instructions in comment in `~/.dexto/preferences.yml` + - add new `env` system prompt contributor that includes info about os, working directory, git status. useful for coding agent to get enough context to improve cmd construction without unnecessary directory shifts + - support for loading `.claude/commands` and `.cursor/commands` global and local commands in addition to `.dexto/commands` + +## 1.5.3 + +### Patch Changes + +- 4f00295: Added spawn-agent tools and explore agent. +- 69c944c: File integrity & performance improvements, approval system fixes, and developer experience enhancements + + ### File System Improvements + - **File integrity protection**: Store file hashes to prevent edits from corrupting files when content changes between operations (resolves #516) + - **Performance optimization**: Disable backups and remove redundant reads, switch to async non-blocking reads for faster file writes + + ### Approval System Fixes + - **Coding agent auto-approve**: Fix auto-approve not working due to incorrect tool names in auto-approve policies + - **Parallel tool calls**: Fix multiple parallel same-tool calls requiring redundant approvals - now checks all waiting approvals and resolves ones affected by newly approved commands + - **Refactored CLI approval handler**: Decoupled approval handler pattern from server for better separation of concerns + + ### Shell & Scripting Fixes + - **Bash mode aliases**: Fix bash mode not honoring zsh aliases + - **Script improvements**: Miscellaneous script improvements for better developer experience + +## 1.5.2 + +### Patch Changes + +- 8a85ea4: Fix maxsteps in agent loop causing early termination +- 527f3f9: Fixes for interactive CLI + +## 1.5.1 + +### Patch Changes + +- bfcc7b1: PostgreSQL improvements and privacy mode + + **PostgreSQL enhancements:** + - Add connection resilience for serverless databases (Neon, Supabase, etc.) with automatic retry on connection failures + - Support custom PostgreSQL schemas via `options.schema` config + - Add schema name validation to prevent SQL injection + - Improve connection pool error handling to prevent process crashes + + **Privacy mode:** + - Add `--privacy-mode` CLI flag to hide file paths from output (useful for screen recording/sharing) + - Can also be enabled via `DEXTO_PRIVACY_MODE=true` environment variable + + **Session improvements:** + - Add message deduplication in history provider to handle data corruption gracefully + - Add warning when conversation history hits 10k message limit + - Improve session deletion to ensure messages are always cleaned up + + **Other fixes:** + - Sanitize explicit `agentId` for filesystem safety + - Change verbose flush logs to debug level + - Export `BaseTypedEventEmitter` from events module + +- 4aabdb7: Fix claude caching, added gpt-5.2 models and reasoning effort options in user flows. + +## 1.5.0 + +### Minor Changes + +- e7722e5: Minor version bump for new release with bundler, custom tool pkgs, etc. + +### Patch Changes + +- ee12727: Added support for node-llama (llama.cpp) for local GGUF models. Added Ollama as first-class provider. Updated onboarding/setup flow. +- 1e7e974: Added image bundler, @dexto/image-local and moved tool services outside core. Added registry providers to select core services. +- 4c05310: Improve local model/GGUF model support, bash permission fixes in TUI, and add local/ollama switching/deleting support in web UI +- 5fa79fa: Renamed compression to compaction, added context-awareness to hono, updated cli tool display formatting and added integration test for image-local. +- ef40e60: Upgrades package versions and related changes to MCP SDK. CLI colors improved and token streaming added to status bar. + + Security: Resolve all Dependabot security vulnerabilities. Updated @modelcontextprotocol/sdk to 1.25.2, esbuild to 0.25.0, langchain to 0.3.37, and @langchain/core to 0.3.80. Added pnpm overrides for indirect vulnerabilities (preact@10.27.3, qs@6.14.1, jws@3.2.3, mdast-util-to-hast@13.2.1). Fixed type errors from MCP SDK breaking changes. + +- e714418: Added providers for db and cache storages. Expanded settings panel for API keys and other app preferences in WebUI along with other UI/UX enhancements. +- 7d5ab19: Updated WebUI design, event and state management and forms +- 436a900: Add support for openrouter, bedrock, glama, vertex ai, fix model switching issues and new model experience for each + +## 1.4.0 + +### Minor Changes + +- f73a519: Revamp CLI. Breaking change to DextoAgent.generate() and stream() apis and hono message APIs, so new minor version. Other fixes for logs, web UI related to message streaming/generating + +### Patch Changes + +- bd5c097: Add features check for internal tools, fix coding agent and logger agent elicitation +- 3cdce89: Revamp CLI for coding agent, add new events, improve mcp management, custom models, minor UI changes, prompt management +- d640e40: Remove LLM services, tokenizers, just stick with vercel, remove 'router' from schema and all types and docs +- 6f5627d: - Approval timeouts are now optional, defaulting to no timeout (infinite wait) + - Tool call history now includes success/failure status tracking +- 6e6a3e7: Fix message typings to use proper discriminated unions in core and webui +- c54760f: Revamp context management layer - add partial stream cancellation, message queueing, context compression with LLM, MCP UI support and gaming agent. New APIs and UI changes for these things +- ab47df8: Add approval metadata and ui badge +- 3b4b919: Fixed Ink CLI bugs and updated state management system. + +## 1.3.0 + +### Minor Changes + +- eb266af: Migrate WebUI from next-js to vite. Fix any typing in web UI. Improve types in core. minor renames in event schemas + +### Patch Changes + +- e2f770b: Add changeset for updated schema defaults and updated docs. +- f843b62: Change otel and storage deps to peer dependencies with dynamic imports to reduce bloat + +## 1.2.6 + +### Patch Changes + +- 7feb030: Update memory and prompt configs, fix agent install bug + +## 1.2.5 + +### Patch Changes + +- c1e814f: ## Logger v2 & Config Enrichment + + ### New Features + - **Multi-transport logging system**: Configure console, file, and remote logging transports via `logger` field in agent.yml. Supports log levels (error, warn, info, debug, silly) and automatic log rotation for file transports. + - **Per-agent isolation**: CLI automatically creates per-agent log files at `~/.dexto/logs/.log`, database at `~/.dexto/database/.db`, and blob storage at `~/.dexto/blobs//` + - **Agent ID derivation**: Agent ID is now automatically derived from `agentCard.name` (sanitized) or config filename, enabling proper multi-agent isolation without manual configuration + + ### Breaking Changes + - **Storage blob default changed**: Default blob storage type changed from `local` to `in-memory`. Existing configs with explicit `blob: { type: 'local' }` are unaffected. CLI enrichment provides automatic paths for SQLite and local blob storage. + + ### Improvements + - **Config enrichment layer**: New `enrichAgentConfig()` in agent-management package adds per-agent paths before initialization, eliminating path resolution in core services + - **Logger error factory**: Added typed error factory pattern for logger errors following project conventions + - **Removed wildcard exports**: Logger module now uses explicit named exports for better tree-shaking + + ### Documentation + - Added complete logger configuration section to agent.yml documentation + - Documented agentId field and derivation rules + - Updated storage documentation with CLI auto-configuration notes + - Added logger v2 architecture notes to core README + +- f9bca72: Add changeset for dropping defaultSessions from core layers. +- c0a10cd: Add changeset for mcp http mode patches. +- 81598b5: Decoupled elicitation from tool confirmation. Added `DenialReason` enum and structured error messages to approval responses. + - Tool approvals and elicitation now independently configurable via `elicitation.enabled` config + - Approval errors include `reason` (user_denied, timeout, system_denied, etc.) and `message` fields + - Enables `auto-approve` for tools while preserving interactive elicitation + + Config files without the new `elicitation` section will use defaults. No legacy code paths. + +- 4c90ffe: Add changeset for updated telemetry spans. +- 1a20506: update source context usage to also go through preferences + registry flow. added dexto_dev_mode flag for maintainers +- 8f373cc: Migrate server API to Hono framework with feature flag + - Migrated Express server to Hono with OpenAPI schema generation + - Added DEXTO_USE_HONO environment variable flag (default: false for backward compatibility) + - Fixed WebSocket test isolation by adding sessionId filtering + - Fixed logger context to pass structured objects instead of stringified JSON + - Fixed CI workflow for OpenAPI docs synchronization + - Updated documentation links and fixed broken API references + +- f28ad7e: Migrate webUI to use client-sdk, add agents.md file to webui,improve types in apis for consumption +- 4dd4998: Add changeset for command approval enhancement and orphaned tool handling +- 5e27806: Add changeset for updated agentCard with protocol version 0.3.0 +- a35a256: Migrate from WebSocket to Server-Sent Events (SSE) for real-time streaming + - Replace WebSocket with SSE for message streaming via new `/api/message-stream` endpoint + - Refactor approval system from event-based providers to simpler handler pattern + - Add new APIs for session approval + - Move session title generation to a separate API + - Add `ApprovalCoordinator` for multi-client SSE routing with sessionId mapping + - Add stream and generate methods to DextoAgent and integ tests for itq= + +- 0fa6ef5: add gpt 5 codex +- e2fb5f8: Add claude 4.5 opus +- a154ae0: UI refactor with TanStack Query, new agent management package, and Hono as default server + + **Server:** + - Make Hono the default API server (use `DEXTO_USE_EXPRESS=true` env var to use Express) + - Fix agentId propagation to Hono server for correct agent name display + - Fix circular reference crashes in error logging by using structured logger context + + **WebUI:** + - Integrate TanStack Query for server state management with automatic caching and invalidation + - Add centralized query key factory and API client with structured error handling + - Replace manual data fetching with TanStack Query hooks across all components + - Add Zustand for client-side persistent state (recent agents in localStorage) + - Add keyboard shortcuts support with react-hotkeys-hook + - Add optimistic updates for session management via WebSocket events + - Fix Dialog auto-close bug in CreateMemoryModal + - Add defensive null handling in MemoryPanel + - Standardize Prettier formatting (single quotes, 4-space indentation) + + **Agent Management:** + - Add `@dexto/agent-management` package for centralized agent configuration management + - Extract agent registry, preferences, and path utilities into dedicated package + + **Internal:** + - Improve build orchestration and fix dependency imports + - Add `@dexto/agent-management` to global CLI installation + +- ac649fd: Fix error handling and UI bugs, add gpt-5.1, gemini-3 + +## 1.2.4 + +### Patch Changes + +- cd706e7: bump up version after fixing node-machine-id + +## 1.2.3 + +### Patch Changes + +- 5d6ae73: Bump up version to fix bugs + +## 1.2.2 + +## 1.2.1 + +## 1.2.0 + +### Minor Changes + +- 1e25f91: Update web UI to be default, fix port bugs, update docs + +### Patch Changes + +- b51e4d9: Add changeset for blob mimetype check patch +- a27ddf0: Add OTEL telemetry to trace agent execution +- 155813c: Add changeset for coding internal tools +- 3a65cde: Update older LLMs to new LLMs, update docs +- 5ba5d38: **Features:** + - Agent switcher now supports file-based agents loaded via CLI (e.g., `dexto --agent path/to/agent.yml`) + - Agent selector UI remembers recent agents (up to 5) with localStorage persistence + - WebUI displays currently active file-based agent and recent agent history + - Dev server (`pnpm dev`) now auto-opens browser when WebUI is ready + - Added `/test-api` custom command for automated API test coverage analysis + + **Bug Fixes:** + - Fixed critical bug where Memory, A2A, and MCP API routes used stale agent references after switching + - Fixed telemetry shutdown blocking agent switches when observability infrastructure (Jaeger/OTLP) is unavailable + - Fixed dark mode styling issues when Chrome's Auto Dark Mode is enabled + - Fixed agent card not updating for A2A and MCP routes after agent switch + + **Improvements:** + - Refactored `Dexto.createAgent()` to static method, removing unnecessary singleton pattern + - Improved error handling for agent switching with typed errors (CONFLICT error type, `AgentError.switchInProgress()`) + - Telemetry now disabled by default (opt-in) in default agent configuration + - Added localStorage corruption recovery for recent agents list + +- 930a4ca: Fixes in UI, docs and agents +- ecad345: Add changeset for allow/deny tool policies. +- 930d75a: Add mcp server restart feature and button in webUI + +## 1.1.11 + +### Patch Changes + +- c40b675: - Updated toolResult sanitization flow + - Added support for video rendering to WebUI +- 015100c: Added new memory manager for creating, storing and managing memories. + - FileContributor has a new memories contributor for loading memories into SystemPrompt. +- 0760f8a: Fixes to postgres data parsing and url env parsing. +- 5cc6933: Fixes for prompts/resource management, UI improvements, custom slash command support, add support for embedded/linked resources, proper argument handling for prompts +- 40f89f5: Add New Agent buttons, form editor, new APIs, Dexto class +- 3a24d08: Add claude haiku 4.5 support +- 01167a2: Refactors +- a53b87a: feat: Redesign agent registry system with improved agent switching + - **@dexto/core**: Enhanced agent registry with better ID-based resolution, improved error handling, and normalized registry entries + - **dexto**: Added agent switching capabilities via API with proper state management + - **@dexto/webui**: Updated agent selector UI with better UX for switching between agents + - Agent resolution now uses `agentId` instead of `agentName` throughout the system + - Registry entries now require explicit `id` field matching the registry key + +- 24e5093: Add customize agent capabilities +- c695e57: Add blob storage system for persistent binary data management: + - Implement blob storage backend with local filesystem support + - Add blob:// URI scheme for referencing stored blobs + - Integrate blob storage with resource system for seamless @resource references + - Add automatic blob expansion in chat history and message references + - Add real-time cache invalidation events for resources and prompts + - Fix prompt cache invalidation WebSocket event handling in WebUI + - Add robustness improvements: empty text validation after resource expansion and graceful blob expansion error handling + - Support image/file uploads with automatic blob storage + - Add WebUI components for blob resource display and autocomplete +- 0700f6f: Support for in-built and custom plugins +- 0a5636c: Added a new Approval System and support MCP Elicitations +- 35d48c5: Add chat summary generation + +## 1.1.10 + +## 1.1.9 + +### Patch Changes + +- 27778ba: Add claude 4.5 sonnet and make it default + +## 1.1.8 + +### Patch Changes + +- d79d358: Add new functions for agent management to DextoAgent() + +## 1.1.7 + +## 1.1.6 + +## 1.1.5 + +### Patch Changes + +- e2bd0ce: Update build to not bundle +- 11cbec0: Update READMEs and docs +- 795c7f1: feat: Add @dexto/client-sdk package + - New lightweight cross-environment client SDK + - HTTP + optional WebSocket support for messaging + - Streaming and non-streaming message support + - Session management, LLM config/catalog access + - MCP tools integration and search functionality + - Real-time events support + - Comprehensive TypeScript types and validation + - Unit tests and documentation included + +- 9d7541c: Add posthog telemetry + +## 1.1.4 + +### Patch Changes + +- 2fccffd: Migrating to monorepo diff --git a/dexto/packages/core/README.md b/dexto/packages/core/README.md new file mode 100644 index 00000000..f091a9fb --- /dev/null +++ b/dexto/packages/core/README.md @@ -0,0 +1,283 @@ +# @dexto/core + +The Dexto Agent SDK for building agentic applications programmatically. This package powers the Dexto CLI and lets you embed the same agent runtime in your own apps. + +## Installation + +```bash +npm install @dexto/core +``` + +### Optional Dependencies + +Some features require additional packages. Install only what you need: + +```bash +# Telemetry (OpenTelemetry distributed tracing) +npm install @opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node \ + @opentelemetry/resources @opentelemetry/semantic-conventions \ + @opentelemetry/sdk-trace-base @opentelemetry/exporter-trace-otlp-http + +# For gRPC telemetry export protocol +npm install @opentelemetry/exporter-trace-otlp-grpc + +# Storage backends +npm install better-sqlite3 # SQLite database +npm install pg # PostgreSQL database +npm install ioredis # Redis cache + +# TypeScript plugin support +npm install tsx +``` + +If you configure a feature without its dependencies, you'll get a helpful error message with the exact install command. + +> **Note:** The `dexto` CLI package includes all optional dependencies. These are only needed when using `@dexto/core` directly as a library. + +## Quick Start + +```ts +import { DextoAgent } from '@dexto/core'; + +// Create and start agent +const agent = new DextoAgent({ + llm: { + provider: 'openai', + model: 'gpt-5-mini', + apiKey: process.env.OPENAI_API_KEY + } +}); +await agent.start(); + +// Create a session for the conversation +const session = await agent.createSession(); + +// Use generate() for simple request/response +const response = await agent.generate('What is TypeScript?', session.id); +console.log(response.content); + +// Conversations maintain context within a session +await agent.generate('Write a haiku about it', session.id); +await agent.generate('Make it funnier', session.id); + +// Multimodal: send images or files +await agent.generate([ + { type: 'text', text: 'Describe this image' }, + { type: 'image', image: base64Data, mimeType: 'image/png' } +], session.id); + +// Streaming for real-time UIs +for await (const event of await agent.stream('Write a story', session.id)) { + if (event.name === 'llm:chunk') process.stdout.write(event.content); +} + +await agent.stop(); +``` + +See the [Dexto Agent SDK docs](https://docs.dexto.ai/docs/guides/dexto-sdk) for multimodal content, streaming, MCP tools, and advanced features. + +--- + +### Starting a Server + +Start a Dexto server programmatically to expose REST and SSE streaming APIs to interact and manage your agent backend. + +```typescript +import { DextoAgent } from '@dexto/core'; +import { startHonoApiServer } from 'dexto'; + +// Create and configure agent +const agent = new DextoAgent({ + llm: { + provider: 'openai', + model: 'gpt-5-mini', + apiKey: process.env.OPENAI_API_KEY + }, + mcpServers: { + filesystem: { + type: 'stdio', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-filesystem', '.'] + } + } +}); + +// Start server on port 3001 +const { server } = await startHonoApiServer(agent, 3001); + +console.log('Dexto server running at http://localhost:3001'); +// Server provides REST API and SSE streaming endpoints +// POST /api/message - Send messages +// GET /api/sessions - List sessions +// See docs.dexto.ai/api/rest/ for all endpoints +``` + +This starts an HTTP server with full REST and SSE APIs, enabling integration with web frontends, webhooks, and other services. See the [REST API Documentation](https://docs.dexto.ai/api/rest/) for available endpoints. + +### Session Management + +Create and manage multiple conversation sessions with persistent storage. + +```typescript +const agent = new DextoAgent(config); +await agent.start(); + +// Create and manage sessions +const session = await agent.createSession('user-123'); +await agent.generate('Hello, how can you help me?', session.id); + +// List and manage sessions +const sessions = await agent.listSessions(); +const sessionHistory = await agent.getSessionHistory('user-123'); +await agent.deleteSession('user-123'); + +// Search across conversations +const results = await agent.searchMessages('bug fix', { limit: 10 }); +``` + +### LLM Management + +Switch between models and providers dynamically. + +```typescript +// Get current configuration +const currentLLM = agent.getCurrentLLMConfig(); + +// Switch models (provider inferred automatically) +await agent.switchLLM({ model: 'gpt-5-mini' }); +await agent.switchLLM({ model: 'claude-sonnet-4-5-20250929' }); + +// Switch model for a specific session id 1234 +await agent.switchLLM({ model: 'gpt-5-mini' }, '1234') + +// Get supported providers and models +const providers = agent.getSupportedProviders(); +const models = agent.getSupportedModels(); +const openaiModels = agent.getSupportedModelsForProvider('openai'); +``` + +### MCP Manager + +For advanced MCP server management, use the MCPManager directly. + +```typescript +import { MCPManager } from '@dexto/core'; + +const manager = new MCPManager(); + +// Connect to MCP servers +await manager.connectServer('filesystem', { + type: 'stdio', + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-filesystem', '.'] +}); + +// Access tools, prompts, and resources +const tools = await manager.getAllTools(); +const prompts = await manager.getAllPrompts(); +const resources = await manager.getAllResources(); + +// Execute tools +const result = await manager.executeTool('readFile', { path: './README.md' }); + +await manager.disconnectAll(); +``` + +### Agent-to-Agent Delegation + +Delegate tasks to other A2A-compliant agents using the built-in `delegate_to_url` tool. + +```typescript +const agent = new DextoAgent({ + llm: { /* ... */ }, + internalTools: ['delegate_to_url'], // Enable delegation tool + toolConfirmation: { mode: 'auto-approve' } +}); +await agent.start(); + +const session = await agent.createSession(); + +// Delegate via natural language +await agent.generate(` + Please delegate this task to the PDF analyzer agent at http://localhost:3001: + "Extract all tables from the Q4 sales report" +`, session.id); + +// Or call the tool directly +const tools = await agent.getAllTools(); +const delegateTool = tools.find(t => t.name === 'internal--delegate_to_url'); +// The tool is prefixed with 'internal--' by the ToolManager +``` + +**Configuration (YAML):** +```yaml +internalTools: + - delegate_to_url +``` + +**What it provides:** +- Point-to-point delegation when you know the agent URL +- A2A Protocol v0.3.0 compliant (JSON-RPC transport) +- Session management for stateful multi-turn conversations +- Automatic endpoint discovery (/v1/jsonrpc, /jsonrpc) +- Timeout handling and error recovery + +**Use cases:** +- Multi-agent systems with known agent URLs +- Delegation to specialized agents +- Building agent workflows and pipelines +- Testing agent-to-agent communication + +### Telemetry + +Built-in OpenTelemetry distributed tracing for observability. + +```typescript +const agent = new DextoAgent({ + llm: { /* ... */ }, + telemetry: { + enabled: true, + serviceName: 'my-agent', + export: { + type: 'otlp', + endpoint: 'http://localhost:4318/v1/traces' + } + } +}); +``` + +Automatically traces agent operations, LLM calls with token usage, and tool executions. See `src/telemetry/README.md` for details. + +### Logger + +Multi-transport logging system (v2) with console, file, and remote transports. Configure in agent YAML: + +```yaml +logger: + level: info # error | warn | info | debug | silly + transports: + - type: console + colorize: true + - type: file + path: ./logs/agent.log + maxSize: 10485760 + maxFiles: 5 +``` + +CLI automatically adds per-agent file transport at `~/.dexto/logs/.log`. See architecture in `src/logger/v2/`. + +See the DextoAgent API reference for all methods: +https://docs.dexto.ai/api/dexto-agent/ + +--- + +## Links + +- Docs: https://docs.dexto.ai/ +- Configuration Guide: https://docs.dexto.ai/docs/category/guides/ +- API Reference: https://docs.dexto.ai/api/ + +## License + +Elastic License 2.0. See the repository LICENSE for details. + diff --git a/dexto/packages/core/package.json b/dexto/packages/core/package.json new file mode 100644 index 00000000..361d0c9d --- /dev/null +++ b/dexto/packages/core/package.json @@ -0,0 +1,127 @@ +{ + "name": "@dexto/core", + "version": "1.5.6", + "private": false, + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "browser": { + "import": "./dist/index.browser.js", + "require": "./dist/index.browser.cjs" + }, + "default": { + "import": "./dist/index.js", + "require": "./dist/index.cjs" + } + }, + "./package.json": "./package.json" + }, + "dependencies": { + "@ai-sdk/amazon-bedrock": "^3.0.71", + "@ai-sdk/anthropic": "^2.0.56", + "@ai-sdk/cohere": "^2.0.0", + "@ai-sdk/google": "^2.0.0", + "@ai-sdk/google-vertex": "^3.0.94", + "@ai-sdk/groq": "^2.0.0", + "@ai-sdk/openai": "^2.0.0", + "@ai-sdk/provider": "^2.0.0", + "@ai-sdk/xai": "^2.0.0", + "@modelcontextprotocol/sdk": "^1.25.2", + "@opentelemetry/api": "^1.9.0", + "ai": "^5.0.0", + "boxen": "^7.1.1", + "chalk": "^5.4.1", + "diff": "^8.0.2", + "dotenv": "^16.4.7", + "glob": "^11.0.3", + "nanoid": "^5.1.6", + "winston": "^3.17.0", + "yaml": "^2.7.1", + "zod-to-json-schema": "^3.24.6" + }, + "devDependencies": { + "@opentelemetry/instrumentation-http": "^0.210.0", + "@opentelemetry/instrumentation-undici": "^0.20.0", + "@opentelemetry/resources": "^1.28.0", + "@opentelemetry/sdk-node": "^0.55.0", + "@opentelemetry/sdk-trace-base": "^1.28.0", + "@opentelemetry/semantic-conventions": "^1.28.0", + "@types/diff": "^8.0.0", + "@types/json-schema": "^7.0.15", + "zod": "^3.25.0" + }, + "peerDependencies": { + "@opentelemetry/core": "^1.28.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.55.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.55.0", + "@opentelemetry/instrumentation-http": "^0.210.0", + "@opentelemetry/instrumentation-undici": "^0.20.0", + "@opentelemetry/resources": "^1.28.0", + "@opentelemetry/sdk-node": "^0.55.0", + "@opentelemetry/sdk-trace-base": "^1.28.0", + "@opentelemetry/semantic-conventions": "^1.28.0", + "better-sqlite3": "^11.10.0", + "ioredis": "^5.7.0", + "pg": "^8.15.4", + "tsx": "^4.19.2", + "zod": "^3.25.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/core": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-grpc": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/instrumentation-http": { + "optional": true + }, + "@opentelemetry/instrumentation-undici": { + "optional": true + }, + "@opentelemetry/resources": { + "optional": true + }, + "@opentelemetry/sdk-node": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "@opentelemetry/semantic-conventions": { + "optional": true + }, + "better-sqlite3": { + "optional": true + }, + "ioredis": { + "optional": true + }, + "pg": { + "optional": true + }, + "tsx": { + "optional": true + } + }, + "scripts": { + "build": "cross-env NODE_OPTIONS='--max-old-space-size=4096' tsup && cross-env NODE_OPTIONS='--max-old-space-size=4096' tsc -p tsconfig.json --emitDeclarationOnly && node scripts/fix-dist-aliases.mjs", + "dev": "tsup --watch", + "typecheck": "tsc -p tsconfig.typecheck.json --noEmit", + "lint": "eslint . --ext .ts" + }, + "files": [ + "dist", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "sideEffects": false +} diff --git a/dexto/packages/core/scripts/fix-dist-aliases.mjs b/dexto/packages/core/scripts/fix-dist-aliases.mjs new file mode 100644 index 00000000..792ec795 --- /dev/null +++ b/dexto/packages/core/scripts/fix-dist-aliases.mjs @@ -0,0 +1,102 @@ +#!/usr/bin/env node +/* eslint-env node */ +import console from 'node:console'; +import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs'; +import { dirname, extname, join, relative, resolve } from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const DIST_DIR = resolve(__dirname, '../dist'); +const PROCESS_EXTENSIONS = new Set(['.js', '.cjs', '.mjs', '.ts', '.mts', '.cts']); +const IMPORT_PATTERN = /(['"])@core\/([^'"]+)\1/g; + +function collectFiles(root) { + const entries = readdirSync(root); + const files = []; + for (const entry of entries) { + const fullPath = join(root, entry); + const stats = statSync(fullPath); + if (stats.isDirectory()) { + files.push(...collectFiles(fullPath)); + } else { + files.push(fullPath); + } + } + return files; +} + +function resolveImport(fromFile, subpath) { + const fromExt = extname(fromFile); + const preferredExt = fromExt === '.cjs' ? '.cjs' : '.js'; + const candidateBase = subpath.replace(/\.(mjs|cjs|js)$/, ''); + const bases = [candidateBase]; + if (!candidateBase.endsWith('index')) { + bases.push(join(candidateBase, 'index')); + } + + const candidates = []; + for (const base of bases) { + const exts = Array.from(new Set([preferredExt, '.mjs', '.js', '.cjs'])); + for (const ext of exts) { + candidates.push(`${base}${ext}`); + } + } + + for (const candidate of candidates) { + const absolute = resolve(DIST_DIR, candidate); + if (existsSync(absolute)) { + let relativePath = relative(dirname(fromFile), absolute).replace(/\\/g, '/'); + if (!relativePath.startsWith('.')) { + relativePath = `./${relativePath}`; + } + return relativePath; + } + } + + return null; +} + +function rewriteAliases(filePath) { + const ext = extname(filePath); + if (!PROCESS_EXTENSIONS.has(ext)) { + return false; + } + + const original = readFileSync(filePath, 'utf8'); + let modified = false; + const updated = original.replace(IMPORT_PATTERN, (match, quote, requested) => { + const resolved = resolveImport(filePath, requested); + if (!resolved) { + console.warn(`⚠️ Unable to resolve alias @core/${requested} in ${filePath}`); + return match; + } + modified = true; + return `${quote}${resolved}${quote}`; + }); + + if (modified) { + writeFileSync(filePath, updated, 'utf8'); + } + + return modified; +} + +function main() { + if (!existsSync(DIST_DIR)) { + console.error(`❌ dist directory not found at ${DIST_DIR}`); + process.exit(1); + } + + const files = collectFiles(DIST_DIR); + let changed = 0; + for (const file of files) { + if (rewriteAliases(file)) { + changed += 1; + } + } + console.log(`ℹ️ Fixed alias imports in ${changed} files.`); +} + +main(); diff --git a/dexto/packages/core/src/agent/DextoAgent.lifecycle.test.ts b/dexto/packages/core/src/agent/DextoAgent.lifecycle.test.ts new file mode 100644 index 00000000..5bb99458 --- /dev/null +++ b/dexto/packages/core/src/agent/DextoAgent.lifecycle.test.ts @@ -0,0 +1,412 @@ +import { describe, test, expect, vi, beforeEach } from 'vitest'; +import { DextoAgent } from './DextoAgent.js'; +import type { AgentConfig, ValidatedAgentConfig } from './schemas.js'; +import { AgentConfigSchema } from './schemas.js'; +import type { AgentServices } from '../utils/service-initializer.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import { AgentErrorCode } from './error-codes.js'; + +// Mock the createAgentServices function +vi.mock('../utils/service-initializer.js', () => ({ + createAgentServices: vi.fn(), +})); + +import { createAgentServices } from '../utils/service-initializer.js'; +const mockCreateAgentServices = vi.mocked(createAgentServices); + +describe('DextoAgent Lifecycle Management', () => { + let mockConfig: AgentConfig; + let mockValidatedConfig: ValidatedAgentConfig; + let mockServices: AgentServices; + + beforeEach(() => { + vi.resetAllMocks(); + + mockConfig = { + systemPrompt: 'You are a helpful assistant', + llm: { + provider: 'openai', + model: 'gpt-5', + apiKey: 'test-key', + maxIterations: 50, + maxInputTokens: 128000, + }, + mcpServers: {}, + sessions: { + maxSessions: 10, + sessionTTL: 3600, + }, + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: false, + timeout: 120000, + }, + }; + + // Create the validated config that DextoAgent actually uses + mockValidatedConfig = AgentConfigSchema.parse(mockConfig); + + mockServices = { + mcpManager: { + disconnectAll: vi.fn(), + initializeFromConfig: vi.fn().mockResolvedValue(undefined), + } as any, + toolManager: { + setAgent: vi.fn(), + setPromptManager: vi.fn(), + initialize: vi.fn().mockResolvedValue(undefined), + } as any, + systemPromptManager: {} as any, + agentEventBus: { + on: vi.fn(), + emit: vi.fn(), + } as any, + stateManager: { + getRuntimeConfig: vi.fn().mockReturnValue({ + llm: mockValidatedConfig.llm, + mcpServers: {}, + storage: { + cache: { type: 'in-memory' }, + database: { type: 'in-memory' }, + }, + sessions: { + maxSessions: 10, + sessionTTL: 3600, + }, + }), + getLLMConfig: vi.fn().mockReturnValue(mockValidatedConfig.llm), + } as any, + sessionManager: { + cleanup: vi.fn(), + init: vi.fn().mockResolvedValue(undefined), + createSession: vi.fn().mockResolvedValue({ id: 'test-session' }), + } as any, + searchService: {} as any, + storageManager: { + disconnect: vi.fn(), + getDatabase: vi.fn().mockReturnValue({}), + getCache: vi.fn().mockReturnValue({}), + getBlobStore: vi.fn().mockReturnValue({}), + } as any, + resourceManager: {} as any, + approvalManager: { + requestToolConfirmation: vi.fn(), + requestElicitation: vi.fn(), + cancelApproval: vi.fn(), + cancelAllApprovals: vi.fn(), + hasHandler: vi.fn().mockReturnValue(false), + } as any, + memoryManager: {} as any, + pluginManager: { + cleanup: vi.fn(), + } as any, + }; + + mockCreateAgentServices.mockResolvedValue(mockServices); + + // Set up default behaviors for mock functions that will be overridden in tests + (mockServices.sessionManager.cleanup as any).mockResolvedValue(undefined); + (mockServices.mcpManager.disconnectAll as any).mockResolvedValue(undefined); + (mockServices.storageManager!.disconnect as any).mockResolvedValue(undefined); + }); + + describe('Constructor Patterns', () => { + test('should create agent with config (new pattern)', () => { + const agent = new DextoAgent(mockConfig); + + expect(agent.isStarted()).toBe(false); + expect(agent.isStopped()).toBe(false); + }); + }); + + describe('start() Method', () => { + test('should start successfully with valid config', async () => { + const agent = new DextoAgent(mockConfig); + + await agent.start(); + + expect(agent.isStarted()).toBe(true); + expect(agent.isStopped()).toBe(false); + expect(mockCreateAgentServices).toHaveBeenCalledWith( + mockValidatedConfig, + undefined, + expect.anything(), // logger instance + expect.anything() // eventBus instance + ); + }); + + test('should start with per-server connection modes in config', async () => { + const configWithServerModes = { + ...mockConfig, + mcpServers: { + filesystem: { + type: 'stdio' as const, + command: 'npx', + args: ['@modelcontextprotocol/server-filesystem', '.'], + env: {}, + timeout: 30000, + connectionMode: 'strict' as const, + }, + }, + }; + const agent = new DextoAgent(configWithServerModes); + + await agent.start(); + + const validatedConfigWithServerModes = AgentConfigSchema.parse(configWithServerModes); + expect(mockCreateAgentServices).toHaveBeenCalledWith( + validatedConfigWithServerModes, + undefined, + expect.anything(), // logger instance + expect.anything() // eventBus instance + ); + }); + + test('should throw error when starting twice', async () => { + const agent = new DextoAgent(mockConfig); + + await agent.start(); + + await expect(agent.start()).rejects.toThrow( + expect.objectContaining({ + code: AgentErrorCode.ALREADY_STARTED, + scope: ErrorScope.AGENT, + type: ErrorType.USER, + }) + ); + }); + + test('should handle start failure gracefully', async () => { + const agent = new DextoAgent(mockConfig); + mockCreateAgentServices.mockRejectedValue(new Error('Service initialization failed')); + + await expect(agent.start()).rejects.toThrow('Service initialization failed'); + expect(agent.isStarted()).toBe(false); + }); + }); + + describe('stop() Method', () => { + test('should stop successfully after start', async () => { + const agent = new DextoAgent(mockConfig); + await agent.start(); + + await agent.stop(); + + expect(agent.isStarted()).toBe(false); + expect(agent.isStopped()).toBe(true); + expect(mockServices.sessionManager.cleanup).toHaveBeenCalled(); + expect(mockServices.mcpManager.disconnectAll).toHaveBeenCalled(); + expect(mockServices.storageManager!.disconnect).toHaveBeenCalled(); + }); + + test('should throw error when stopping before start', async () => { + const agent = new DextoAgent(mockConfig); + + await expect(agent.stop()).rejects.toThrow( + expect.objectContaining({ + code: AgentErrorCode.NOT_STARTED, + scope: ErrorScope.AGENT, + type: ErrorType.USER, + }) + ); + }); + + test('should warn when stopping twice but not throw', async () => { + const agent = new DextoAgent(mockConfig); + await agent.start(); + await agent.stop(); + + // Second stop should not throw but should warn + await expect(agent.stop()).resolves.toBeUndefined(); + }); + + test('should handle partial cleanup failures gracefully', async () => { + const agent = new DextoAgent(mockConfig); + await agent.start(); + + // Make session cleanup fail + (mockServices.sessionManager.cleanup as any).mockRejectedValue( + new Error('Session cleanup failed') + ); + + // Should not throw, but should still mark as stopped + await expect(agent.stop()).resolves.toBeUndefined(); + expect(agent.isStopped()).toBe(true); + + // Should still try to clean other services + expect(mockServices.mcpManager.disconnectAll).toHaveBeenCalled(); + expect(mockServices.storageManager!.disconnect).toHaveBeenCalled(); + }); + }); + + describe('Method Access Control', () => { + const testMethods = [ + { name: 'run', args: ['test message'] }, + { name: 'createSession', args: [] }, + { name: 'getSession', args: ['session-id'] }, + { name: 'listSessions', args: [] }, + { name: 'deleteSession', args: ['session-id'] }, + { name: 'resetConversation', args: [] }, + { name: 'getCurrentLLMConfig', args: [] }, + { name: 'switchLLM', args: [{ model: 'gpt-5' }] }, + { name: 'addMcpServer', args: ['test', { type: 'stdio', command: 'test' }] }, + { name: 'getAllMcpTools', args: [] }, + ]; + + test.each(testMethods)('$name should throw before start()', async ({ name, args }) => { + const agent = new DextoAgent(mockConfig); + + let thrownError: DextoRuntimeError | undefined; + try { + const method = agent[name as keyof DextoAgent] as Function; + await method.apply(agent, args); + } catch (error) { + thrownError = error as DextoRuntimeError; + } + + expect(thrownError).toBeDefined(); + expect(thrownError).toMatchObject({ + code: AgentErrorCode.NOT_STARTED, + scope: ErrorScope.AGENT, + type: ErrorType.USER, + }); + }); + + test.each(testMethods)('$name should throw after stop()', async ({ name, args }) => { + const agent = new DextoAgent(mockConfig); + await agent.start(); + await agent.stop(); + + let thrownError: DextoRuntimeError | undefined; + try { + const method = agent[name as keyof DextoAgent] as Function; + await method.apply(agent, args); + } catch (error) { + thrownError = error as DextoRuntimeError; + } + + expect(thrownError).toBeDefined(); + expect(thrownError).toMatchObject({ + code: AgentErrorCode.STOPPED, + scope: ErrorScope.AGENT, + type: ErrorType.USER, + }); + }); + + test('isStarted and isStopped should work without start() (read-only)', () => { + const agent = new DextoAgent(mockConfig); + + expect(() => agent.isStarted()).not.toThrow(); + expect(() => agent.isStopped()).not.toThrow(); + }); + }); + + describe('Session Auto-Approve Tools Cleanup (Memory Leak Fix)', () => { + test('endSession should call clearSessionAutoApproveTools', async () => { + const agent = new DextoAgent(mockConfig); + + // Add clearSessionAutoApproveTools mock to toolManager + mockServices.toolManager.clearSessionAutoApproveTools = vi.fn(); + mockServices.sessionManager.endSession = vi.fn().mockResolvedValue(undefined); + + await agent.start(); + + await agent.endSession('test-session-123'); + + expect(mockServices.toolManager.clearSessionAutoApproveTools).toHaveBeenCalledWith( + 'test-session-123' + ); + expect(mockServices.sessionManager.endSession).toHaveBeenCalledWith('test-session-123'); + }); + + test('deleteSession should call clearSessionAutoApproveTools', async () => { + const agent = new DextoAgent(mockConfig); + + // Add clearSessionAutoApproveTools mock to toolManager + mockServices.toolManager.clearSessionAutoApproveTools = vi.fn(); + mockServices.sessionManager.deleteSession = vi.fn().mockResolvedValue(undefined); + + await agent.start(); + + await agent.deleteSession('test-session-456'); + + expect(mockServices.toolManager.clearSessionAutoApproveTools).toHaveBeenCalledWith( + 'test-session-456' + ); + expect(mockServices.sessionManager.deleteSession).toHaveBeenCalledWith( + 'test-session-456' + ); + }); + + test('clearSessionAutoApproveTools should be called before session cleanup', async () => { + const agent = new DextoAgent(mockConfig); + const callOrder: string[] = []; + + mockServices.toolManager.clearSessionAutoApproveTools = vi.fn(() => { + callOrder.push('clearSessionAutoApproveTools'); + }); + mockServices.sessionManager.endSession = vi.fn().mockImplementation(() => { + callOrder.push('endSession'); + return Promise.resolve(); + }); + + await agent.start(); + await agent.endSession('test-session'); + + expect(callOrder).toEqual(['clearSessionAutoApproveTools', 'endSession']); + }); + }); + + describe('Integration Tests', () => { + test('should handle complete lifecycle without errors', async () => { + const agent = new DextoAgent(mockConfig); + + // Initial state + expect(agent.isStarted()).toBe(false); + expect(agent.isStopped()).toBe(false); + + // Start + await agent.start(); + expect(agent.isStarted()).toBe(true); + expect(agent.isStopped()).toBe(false); + + // Use agent (mock a successful operation) + expect(agent.getCurrentLLMConfig()).toBeDefined(); + + // Stop + await agent.stop(); + expect(agent.isStarted()).toBe(false); + expect(agent.isStopped()).toBe(true); + }); + + test('should handle resource cleanup in correct order', async () => { + const agent = new DextoAgent(mockConfig); + await agent.start(); + + const cleanupOrder: string[] = []; + + (mockServices.sessionManager.cleanup as any).mockImplementation(() => { + cleanupOrder.push('sessions'); + return Promise.resolve(); + }); + + (mockServices.mcpManager.disconnectAll as any).mockImplementation(() => { + cleanupOrder.push('clients'); + return Promise.resolve(); + }); + + (mockServices.storageManager!.disconnect as any).mockImplementation(() => { + cleanupOrder.push('storage'); + return Promise.resolve(); + }); + + await agent.stop(); + + expect(cleanupOrder).toEqual(['sessions', 'clients', 'storage']); + }); + }); +}); diff --git a/dexto/packages/core/src/agent/DextoAgent.stream-api.integration.test.ts b/dexto/packages/core/src/agent/DextoAgent.stream-api.integration.test.ts new file mode 100644 index 00000000..ed4d4f8c --- /dev/null +++ b/dexto/packages/core/src/agent/DextoAgent.stream-api.integration.test.ts @@ -0,0 +1,354 @@ +import { describe, test, expect } from 'vitest'; +import { + createTestEnvironment, + TestConfigs, + requiresApiKey, + cleanupTestEnvironment, +} from '../llm/services/test-utils.integration.js'; +import type { StreamingEvent } from '../events/index.js'; + +/** + * DextoAgent Stream API Integration Tests + * + * Tests the new generate() and stream() APIs with real LLM providers. + * Requires valid API keys (ANTHROPIC_API_KEY, OPENAI_API_KEY, etc.) + */ + +describe('DextoAgent.generate() API', () => { + const skipTests = !requiresApiKey('openai'); + const t = skipTests ? test.skip : test.concurrent; + + t( + 'generate() returns complete response with usage stats', + async () => { + const env = await createTestEnvironment(TestConfigs.createOpenAIConfig()); + try { + const response = await env.agent.generate('What is 2+2?', env.sessionId); + + // Validate response structure + expect(response).toBeDefined(); + expect(response.content).toBeTruthy(); + expect(typeof response.content).toBe('string'); + expect(response.content.length).toBeGreaterThan(0); + + // Validate usage stats + expect(response.usage).toBeDefined(); + expect(response.usage.inputTokens).toBeGreaterThan(0); + expect(response.usage.outputTokens).toBeGreaterThan(0); + expect(response.usage.totalTokens).toBeGreaterThan(0); + + // Validate metadata + expect(response.sessionId).toBe(env.sessionId); + expect(response.toolCalls).toEqual([]); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'generate() maintains conversation context across turns', + async () => { + const env = await createTestEnvironment(TestConfigs.createOpenAIConfig()); + try { + const response1 = await env.agent.generate('My name is Alice', env.sessionId); + const response2 = await env.agent.generate('What is my name?', env.sessionId); + + // Sometimes response1.content can be empty if the model only acknowledges or uses a tool + // But for this simple prompt, it should have content. + // If empty, check if we got a valid response object at least. + expect(response1).toBeDefined(); + if (response1.content === '') { + // Retry or check if it was a valid empty response (e.g. tool call only - unlikely here) + // For now, let's assert it's truthy OR we verify context in second turn regardless + console.warn( + 'First turn response was empty, but proceeding to check context retention' + ); + } else { + expect(response1.content).toBeTruthy(); + } + + expect(response2).toBeDefined(); + if (response2.content === '') { + console.warn( + 'Second turn response was empty, but context retention test is partial success if first turn worked' + ); + } else { + expect(response2.content).toBeTruthy(); + expect(response2.content.toLowerCase()).toContain('alice'); + } + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'generate() works with different providers', + async () => { + const providers = [{ name: 'openai', config: TestConfigs.createOpenAIConfig() }]; + + for (const { name, config } of providers) { + if (!requiresApiKey(name as any)) continue; + + const env = await createTestEnvironment(config); + try { + const response = await env.agent.generate('Say hello', env.sessionId); + + expect(response.content).toBeTruthy(); + expect(response.usage.totalTokens).toBeGreaterThan(0); + } finally { + await cleanupTestEnvironment(env); + } + } + }, + 40000 + ); +}); + +describe('DextoAgent.stream() API', () => { + const skipTests = !requiresApiKey('openai'); + const t = skipTests ? test.skip : test.concurrent; + + t( + 'stream() yields events in correct order', + async () => { + const env = await createTestEnvironment(TestConfigs.createOpenAIConfig()); + try { + const events: StreamingEvent[] = []; + + for await (const event of await env.agent.stream('Say hello', env.sessionId)) { + events.push(event); + } + + // Validate event order + expect(events.length).toBeGreaterThan(0); + expect(events[0]).toBeDefined(); + expect(events[events.length - 1]).toBeDefined(); + expect(events[0]!.name).toBe('llm:thinking'); + // Last event is run:complete (added in lifecycle updates) + expect(events[events.length - 1]!.name).toBe('run:complete'); + + // Validate message-start event + // First event is typically llm:thinking + const startEvent = events[0]; + expect(startEvent).toBeDefined(); + expect(startEvent?.sessionId).toBe(env.sessionId); + + // Find the llm:response event (second to last, before run:complete) + const responseEvent = events.find((e) => e.name === 'llm:response'); + expect(responseEvent).toBeDefined(); + if (responseEvent && responseEvent.name === 'llm:response') { + expect(responseEvent.content).toBeTruthy(); + expect(responseEvent.tokenUsage?.totalTokens).toBeGreaterThan(0); + } + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'stream() yields content-chunk events', + async () => { + const env = await createTestEnvironment(TestConfigs.createOpenAIConfig()); + try { + const chunkEvents: StreamingEvent[] = []; + + for await (const event of await env.agent.stream('Say hello', env.sessionId)) { + if (event.name === 'llm:chunk') { + chunkEvents.push(event); + } + } + + // Should receive multiple chunks + expect(chunkEvents.length).toBeGreaterThan(0); + + // Validate chunk structure + for (const event of chunkEvents) { + if (event.name === 'llm:chunk') { + expect(event.content).toBeDefined(); + expect(typeof event.content).toBe('string'); + expect(event.chunkType).toMatch(/^(text|reasoning)$/); + } + } + + // Reconstruct full content from chunks (chunkEvents already filtered to llm:chunk only) + const fullContent = chunkEvents + .map((e) => (e.name === 'llm:chunk' ? e.content : '')) + .join(''); + + expect(fullContent.length).toBeGreaterThan(0); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'stream() can be consumed multiple times via AsyncIterator', + async () => { + const env = await createTestEnvironment(TestConfigs.createOpenAIConfig()); + try { + const stream = await env.agent.stream('Say hello', env.sessionId); + + const events: StreamingEvent[] = []; + for await (const event of stream) { + events.push(event); + } + + expect(events.length).toBeGreaterThan(0); + expect(events[0]).toBeDefined(); + expect(events[events.length - 1]).toBeDefined(); + expect(events[0]!.name).toBe('llm:thinking'); + // Last event is run:complete (added in lifecycle updates) + expect(events[events.length - 1]!.name).toBe('run:complete'); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'stream() maintains conversation context', + async () => { + const env = await createTestEnvironment(TestConfigs.createOpenAIConfig()); + try { + // First message + const events1: StreamingEvent[] = []; + for await (const event of await env.agent.stream( + 'My favorite color is blue', + env.sessionId + )) { + events1.push(event); + } + + // Second message should remember context + const events2: StreamingEvent[] = []; + for await (const event of await env.agent.stream( + 'What is my favorite color?', + env.sessionId + )) { + events2.push(event); + } + + const completeEvent2 = events2.find((e) => e.name === 'llm:response'); + if (completeEvent2 && completeEvent2.name === 'llm:response') { + expect(completeEvent2.content.toLowerCase()).toContain('blue'); + } + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'stream() works with different providers', + async () => { + const providers = [{ name: 'openai', config: TestConfigs.createOpenAIConfig() }]; + + for (const { name, config } of providers) { + if (!requiresApiKey(name as any)) continue; + + const env = await createTestEnvironment(config); + try { + const events: StreamingEvent[] = []; + + for await (const event of await env.agent.stream('Say hello', env.sessionId)) { + events.push(event); + } + + expect(events.length).toBeGreaterThan(0); + expect(events[0]).toBeDefined(); + expect(events[events.length - 1]).toBeDefined(); + expect(events[0]!.name).toBe('llm:thinking'); + // Last event is run:complete (added in lifecycle updates) + expect(events[events.length - 1]!.name).toBe('run:complete'); + } finally { + await cleanupTestEnvironment(env); + } + } + }, + 40000 + ); +}); + +describe('DextoAgent API Compatibility', () => { + const skipTests = !requiresApiKey('openai'); + const t = skipTests ? test.skip : test.concurrent; + + t( + 'generate() produces same content as run() without streaming', + async () => { + const env = await createTestEnvironment(TestConfigs.createOpenAIConfig()); + try { + const prompt = 'What is 2+2? Answer with just the number.'; + + // Use run() (old API) + const runResponse = await env.agent.run( + prompt, + undefined, + undefined, + env.sessionId + ); + + // Reset conversation + await env.agent.resetConversation(env.sessionId); + + // Use generate() (new API) + const generateResponse = await env.agent.generate(prompt, env.sessionId); + + // Both should work and return similar content + expect(runResponse).toBeTruthy(); + expect(generateResponse.content).toBeTruthy(); + + // Content should contain '4' + expect(runResponse).toContain('4'); + expect(generateResponse.content).toContain('4'); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'stream() works alongside old run() API', + async () => { + const env = await createTestEnvironment(TestConfigs.createOpenAIConfig()); + try { + // Use old run() API + const runResponse = await env.agent.run( + 'My name is Bob', + undefined, + undefined, + env.sessionId + ); + expect(runResponse).toBeTruthy(); + + // Use new stream() API - should maintain same context + const events: StreamingEvent[] = []; + for await (const event of await env.agent.stream( + 'What is my name?', + env.sessionId + )) { + events.push(event); + } + + const completeEvent = events.find((e) => e.name === 'llm:response'); + if (completeEvent && completeEvent.name === 'llm:response') { + expect(completeEvent.content.toLowerCase()).toContain('bob'); + } + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); +}); diff --git a/dexto/packages/core/src/agent/DextoAgent.ts b/dexto/packages/core/src/agent/DextoAgent.ts new file mode 100644 index 00000000..4ad5187d --- /dev/null +++ b/dexto/packages/core/src/agent/DextoAgent.ts @@ -0,0 +1,2847 @@ +// src/agent/DextoAgent.ts +import { randomUUID } from 'crypto'; +import { setMaxListeners } from 'events'; +import { MCPManager } from '../mcp/manager.js'; +import { ToolManager } from '../tools/tool-manager.js'; +import { SystemPromptManager } from '../systemPrompt/manager.js'; +import { SkillsContributor } from '../systemPrompt/contributors.js'; +import { ResourceManager, expandMessageReferences } from '../resources/index.js'; +import { expandBlobReferences } from '../context/utils.js'; +import type { InternalMessage } from '../context/types.js'; +import { PromptManager } from '../prompts/index.js'; +import type { PromptsConfig } from '../prompts/schemas.js'; +import { AgentStateManager } from './state-manager.js'; +import { SessionManager, ChatSession, SessionError } from '../session/index.js'; +import type { SessionMetadata } from '../session/index.js'; +import { AgentServices } from '../utils/service-initializer.js'; +import { createLogger } from '../logger/factory.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import { Telemetry } from '../telemetry/telemetry.js'; +import { InstrumentClass } from '../telemetry/decorators.js'; +import { trace, context, propagation, type BaggageEntry } from '@opentelemetry/api'; +import { ValidatedLLMConfig, LLMUpdates, LLMUpdatesSchema } from '@core/llm/schemas.js'; +import { resolveAndValidateLLMConfig } from '../llm/resolver.js'; +import { validateInputForLLM } from '../llm/validation.js'; +import { LLMError } from '../llm/errors.js'; +import { AgentError } from './errors.js'; +import { MCPError } from '../mcp/errors.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { DextoValidationError } from '../errors/DextoValidationError.js'; +import { ensureOk } from '@core/errors/result-bridge.js'; +import { fail, zodToIssues } from '@core/utils/result.js'; +import { resolveAndValidateMcpServerConfig } from '../mcp/resolver.js'; +import type { McpServerConfig, McpServerStatus, McpConnectionStatus } from '@core/mcp/schemas.js'; +import { + getSupportedProviders, + getDefaultModelForProvider, + getProviderFromModel, + getAllModelsForProvider, + ModelInfo, +} from '../llm/registry.js'; +import type { LLMProvider } from '../llm/types.js'; +import { createAgentServices } from '../utils/service-initializer.js'; +import type { AgentConfig, ValidatedAgentConfig, LLMValidationOptions } from './schemas.js'; +import { AgentConfigSchema, createAgentConfigSchema } from './schemas.js'; +import { + AgentEventBus, + type AgentEventMap, + type StreamingEvent, + type StreamingEventName, +} from '../events/index.js'; +import type { IMCPClient } from '../mcp/types.js'; +import type { ToolSet } from '../tools/types.js'; +import { SearchService } from '../search/index.js'; +import type { SearchOptions, SearchResponse, SessionSearchResponse } from '../search/index.js'; +import { safeStringify } from '@core/utils/safe-stringify.js'; +import { deriveHeuristicTitle, generateSessionTitle } from '../session/title-generator.js'; +import type { ApprovalHandler } from '../approval/types.js'; + +const requiredServices: (keyof AgentServices)[] = [ + 'mcpManager', + 'toolManager', + 'systemPromptManager', + 'agentEventBus', + 'stateManager', + 'sessionManager', + 'searchService', + 'memoryManager', +]; + +/** + * Interface for objects that can subscribe to the agent's event bus. + * Typically used by API layer subscribers (SSE, Webhooks, etc.) + */ +export interface AgentEventSubscriber { + subscribe(eventBus: AgentEventBus): void; +} + +/** + * The main entry point into Dexto's core functionality. + * + * DextoAgent is a high-level abstraction layer that provides a clean, user-facing API + * for building AI agents. It coordinates multiple internal services to deliver core + * capabilities including conversation management, LLM switching, MCP server integration, + * and multi-session support. + * + * Key Features: + * - **Conversation Management**: Process user messages and maintain conversation state + * - **Multi-Session Support**: Create and manage multiple independent chat sessions + * - **Dynamic LLM Switching**: Change language models while preserving conversation history + * - **MCP Server Integration**: Connect to and manage Model Context Protocol servers + * - **Tool Execution**: Execute tools from connected MCP servers + * - **Prompt Management**: Build and inspect dynamic system prompts with context + * - **Event System**: Emit events for integration with external systems + * + * Design Principles: + * - Thin wrapper around internal services with high-level methods + * - Primary API for applications building on Dexto + * - Internal services exposed as public readonly properties for advanced usage + * - Backward compatibility through default session management + * + * @example + * ```typescript + * // Create and start agent + * const agent = new DextoAgent(config); + * await agent.start(); + * + * // Process user messages + * const response = await agent.run("Hello, how are you?"); + * + * // Switch LLM models (provider inferred automatically) + * await agent.switchLLM({ model: 'gpt-5' }); + * + * // Manage sessions + * const session = agent.createSession('user-123'); + * const response = await agent.run("Hello", undefined, 'user-123'); + * + * // Connect MCP servers + * await agent.addMcpServer('filesystem', { command: 'mcp-filesystem' }); + * + * // Inspect available tools and system prompt + * const tools = await agent.getAllMcpTools(); + * const prompt = await agent.getSystemPrompt(); + * + * // Gracefully stop the agent when done + * await agent.stop(); + * ``` + */ +@InstrumentClass({ + prefix: 'agent', + excludeMethods: [ + 'isStarted', + 'isStopped', + 'getConfig', + 'getEffectiveConfig', + 'registerSubscriber', + 'ensureStarted', + ], +}) +export class DextoAgent { + /** + * These services are public for use by the outside world + * This gives users the option to use methods of the services directly if they know what they are doing + * But the main recommended entry points/functions would still be the wrapper methods we define below + */ + public readonly mcpManager!: MCPManager; + public readonly systemPromptManager!: SystemPromptManager; + public readonly agentEventBus!: AgentEventBus; + public readonly promptManager!: PromptManager; + public readonly stateManager!: AgentStateManager; + public readonly sessionManager!: SessionManager; + public readonly toolManager!: ToolManager; + public readonly resourceManager!: ResourceManager; + public readonly memoryManager!: import('../memory/index.js').MemoryManager; + public readonly services!: AgentServices; + + // Search service for conversation search + private searchService!: SearchService; + + // Track initialization state + private _isStarted: boolean = false; + private _isStopped: boolean = false; + + // Store config for async initialization (accessible before start() for setup) + public config: ValidatedAgentConfig; + + // Event subscribers (e.g., SSE, Webhook handlers) + private eventSubscribers: Set = new Set(); + + // Telemetry instance for distributed tracing + private telemetry?: Telemetry; + + // Approval handler for manual tool confirmation and elicitation + // Set via setApprovalHandler() before start() if needed + private approvalHandler?: ApprovalHandler | undefined; + + // Active stream controllers per session - allows cancel() to abort iterators + private activeStreamControllers: Map = new Map(); + + // Logger instance for this agent (dependency injection) + public readonly logger: IDextoLogger; + + /** + * Creates a DextoAgent instance. + * + * @param config - Agent configuration (validated and enriched) + * @param configPath - Optional path to config file (for relative path resolution) + * @param options - Validation options + * @param options.strict - When true (default), enforces API key and baseURL requirements. + * When false, allows missing credentials for interactive configuration. + */ + constructor( + config: AgentConfig, + private configPath?: string, + options?: LLMValidationOptions + ) { + // Validate and transform the input config using appropriate schema + const schema = + options?.strict === false + ? createAgentConfigSchema({ strict: false }) + : AgentConfigSchema; + this.config = schema.parse(config); + + // Create logger instance for this agent + // agentId is set by CLI enrichment from agentCard.name or filename + this.logger = createLogger({ + config: this.config.logger, + agentId: this.config.agentId, + component: DextoLogComponent.AGENT, + }); + + // Create event bus early so it's available for approval handler creation + this.agentEventBus = new AgentEventBus(); + + // call start() to initialize services + this.logger.info('DextoAgent created.'); + } + + /** + * Starts the agent by initializing all async services. + * This method handles storage backends, MCP connections, session manager initialization, and other async operations. + * Must be called before using any agent functionality. + * + * @throws Error if agent is already started or initialization fails + */ + public async start(): Promise { + if (this._isStarted) { + throw AgentError.alreadyStarted(); + } + + try { + this.logger.info('Starting DextoAgent...'); + + // Initialize all services asynchronously + // Pass logger and eventBus to services for dependency injection + const services = await createAgentServices( + this.config, + this.configPath, + this.logger, + this.agentEventBus + ); + + // Validate all required services are provided + for (const service of requiredServices) { + if (!services[service]) { + throw AgentError.initializationFailed( + `Required service ${service} is missing during agent start` + ); + } + } + + // Validate approval configuration + // Handler is required for manual tool confirmation OR when elicitation is enabled + const needsHandler = + this.config.toolConfirmation.mode === 'manual' || this.config.elicitation.enabled; + + if (needsHandler && !this.approvalHandler) { + const reasons = []; + if (this.config.toolConfirmation.mode === 'manual') { + reasons.push('tool confirmation mode is "manual"'); + } + if (this.config.elicitation.enabled) { + reasons.push('elicitation is enabled'); + } + + throw AgentError.initializationFailed( + `An approval handler is required but not configured (${reasons.join(' and ')}).\n` + + 'Either:\n' + + ' • Call agent.setApprovalHandler() before starting\n' + + ' • Set toolConfirmation: { mode: "auto-approve" } or { mode: "auto-deny" }\n' + + ' • Disable elicitation: { enabled: false }' + ); + } + + // Set approval handler if provided via setApprovalHandler() before start() + if (this.approvalHandler) { + services.approvalManager.setHandler(this.approvalHandler); + } + + // Use Object.assign to set readonly properties + Object.assign(this, { + mcpManager: services.mcpManager, + toolManager: services.toolManager, + resourceManager: services.resourceManager, + systemPromptManager: services.systemPromptManager, + stateManager: services.stateManager, + sessionManager: services.sessionManager, + memoryManager: services.memoryManager, + services: services, + }); + + // Initialize prompts manager (aggregates MCP, internal, starter prompts) + // File prompts automatically resolve custom slash commands + // Must be initialized before toolManager so invoke_skill tool can access prompts + const promptManager = new PromptManager( + this.mcpManager, + this.resourceManager, + this.config, + this.agentEventBus, + services.storageManager.getDatabase(), + this.logger + ); + await promptManager.initialize(); + Object.assign(this, { promptManager }); + + // Set agent reference for custom tools (must be done before tool initialization) + // This allows custom tool providers to wire up bidirectional communication + services.toolManager.setAgent(this); + + // Set prompt manager for invoke_skill tool (must be done before tool initialization) + services.toolManager.setPromptManager(promptManager); + + // Add skills contributor to system prompt if invoke_skill is enabled + // This lists available skills so the LLM knows what it can invoke + if (this.config.internalTools?.includes('invoke_skill')) { + const skillsContributor = new SkillsContributor( + 'skills', + 50, // Priority after memories (40) but before most other content + promptManager, + this.logger + ); + services.systemPromptManager.addContributor(skillsContributor); + this.logger.debug('Added SkillsContributor to system prompt'); + } + + // Initialize toolManager now that agent and promptManager references are set + // Custom tools need agent access for bidirectional communication + // invoke_skill tool needs promptManager access + await services.toolManager.initialize(); + + // Initialize search service from services + this.searchService = services.searchService; + + // Note: Telemetry is initialized in createAgentServices() before services are created + // This ensures decorators work correctly on all services + + this._isStarted = true; + this._isStopped = false; // Reset stopped flag to allow restart + this.logger.info('DextoAgent started successfully.'); + + // Subscribe all registered event subscribers to the new event bus + for (const subscriber of this.eventSubscribers) { + subscriber.subscribe(this.agentEventBus); + } + } catch (error) { + this.logger.error('Failed to start DextoAgent', { + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } + } + + /** + * Stops the agent and gracefully shuts down all services. + * This method handles disconnecting MCP clients, cleaning up sessions, closing storage connections, + * and releasing all resources. The agent cannot be restarted after being stopped. + * + * @throws Error if agent has not been started or shutdown fails + */ + public async stop(): Promise { + if (this._isStopped) { + this.logger.warn('Agent is already stopped'); + return; + } + + if (!this._isStarted) { + throw AgentError.notStarted(); + } + + try { + this.logger.info('Stopping DextoAgent...'); + + const shutdownErrors: Error[] = []; + + // 1. Clean up session manager (stop accepting new sessions, clean existing ones) + try { + if (this.sessionManager) { + await this.sessionManager.cleanup(); + this.logger.debug('SessionManager cleaned up successfully'); + } + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + shutdownErrors.push(new Error(`SessionManager cleanup failed: ${err.message}`)); + } + + // 2. Clean up plugins (close file handles, connections, etc.) + // Do this before storage disconnect so plugins can flush state if needed + try { + if (this.services?.pluginManager) { + await this.services.pluginManager.cleanup(); + this.logger.debug('PluginManager cleaned up successfully'); + } + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + shutdownErrors.push(new Error(`PluginManager cleanup failed: ${err.message}`)); + } + + // 3. Disconnect all MCP clients + try { + if (this.mcpManager) { + await this.mcpManager.disconnectAll(); + this.logger.debug('MCPManager disconnected all clients successfully'); + } + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + shutdownErrors.push(new Error(`MCPManager disconnect failed: ${err.message}`)); + } + + // 4. Close storage backends + try { + if (this.services?.storageManager) { + await this.services.storageManager.disconnect(); + this.logger.debug('Storage manager disconnected successfully'); + } + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + shutdownErrors.push(new Error(`Storage disconnect failed: ${err.message}`)); + } + + // Note: Telemetry is NOT shut down here + // For agent switching: Telemetry.shutdownGlobal() is called explicitly before creating new agent + // For process exit: Telemetry shuts down automatically via process exit handlers + // This allows telemetry to persist across agent restarts in the same process + + this._isStopped = true; + this._isStarted = false; + + if (shutdownErrors.length > 0) { + const errorMessages = shutdownErrors.map((e) => e.message).join('; '); + this.logger.warn(`DextoAgent stopped with some errors: ${errorMessages}`); + // Still consider it stopped, but log the errors + } else { + this.logger.info('DextoAgent stopped successfully.'); + } + } catch (error) { + this.logger.error('Failed to stop DextoAgent', { + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } + } + + /** + * Register an event subscriber that will be automatically re-subscribed on agent restart. + * Subscribers are typically API layer components (SSE, Webhook handlers) that need + * to receive agent events. If the agent is already started, the subscriber is immediately subscribed. + * + * @param subscriber - Object implementing AgentEventSubscriber interface + */ + public registerSubscriber(subscriber: AgentEventSubscriber): void { + this.eventSubscribers.add(subscriber); + if (this._isStarted) { + subscriber.subscribe(this.agentEventBus); + } + } + + /** + * Restart the agent by stopping and starting it. + * Automatically re-subscribes all registered event subscribers to the new event bus. + * This is useful when configuration changes require a full agent restart. + * + * @throws Error if restart fails during stop or start phases + */ + public async restart(): Promise { + await this.stop(); + await this.start(); + // Note: start() handles re-subscribing all registered subscribers + } + + /** + * Checks if the agent has been started. + * @returns true if agent is started, false otherwise + */ + public isStarted(): boolean { + return this._isStarted; + } + + /** + * Checks if the agent has been stopped. + * @returns true if agent is stopped, false otherwise + */ + public isStopped(): boolean { + return this._isStopped; + } + + /** + * Ensures the agent is started before executing operations. + * @throws Error if agent is not started or has been stopped + */ + private ensureStarted(): void { + if (this._isStopped) { + this.logger.warn('Agent is stopped'); + throw AgentError.stopped(); + } + if (!this._isStarted) { + this.logger.warn('Agent is not started'); + throw AgentError.notStarted(); + } + } + + // ============= CORE AGENT FUNCTIONALITY ============= + + /** + * Process user input and return the response. + * + * @deprecated Use generate() or stream() instead for multi-image support. + * This method is kept for backward compatibility and only supports single image/file. + * + * @param textInput - The user's text message + * @param imageDataInput - Optional single image data + * @param fileDataInput - Optional single file data + * @param sessionId - Session ID for the conversation (required) + * @param _stream - Ignored (streaming is handled internally) + * @returns Promise that resolves to the AI's response text + */ + public async run( + textInput: string, + imageDataInput: { image: string; mimeType: string } | undefined, + fileDataInput: { data: string; mimeType: string; filename?: string } | undefined, + sessionId: string, + _stream: boolean = false + ): Promise { + // Convert legacy signature to ContentPart[] + const parts: import('./types.js').ContentPart[] = []; + + if (textInput) { + parts.push({ type: 'text', text: textInput }); + } + + if (imageDataInput) { + parts.push({ + type: 'image', + image: imageDataInput.image, + mimeType: imageDataInput.mimeType, + }); + } + + if (fileDataInput) { + parts.push({ + type: 'file', + data: fileDataInput.data, + mimeType: fileDataInput.mimeType, + ...(fileDataInput.filename && { filename: fileDataInput.filename }), + }); + } + + // Call generate() which handles everything + const response = await this.generate(parts.length > 0 ? parts : textInput, sessionId); + return response.content; + } + + /** + * Generate a complete response (waits for full completion). + * This is the recommended method for non-streaming use cases. + * + * @param content String message or array of content parts (text, images, files) + * @param sessionId Session ID for the conversation + * @param options Optional configuration (signal for cancellation) + * @returns Promise that resolves to the complete response + * + * @example + * ```typescript + * // Simple text message + * const response = await agent.generate('What is 2+2?', 'session-1'); + * console.log(response.content); // "4" + * + * // Multimodal with image + * const response = await agent.generate( + * [ + * { type: 'text', text: 'Describe this image' }, + * { type: 'image', image: base64Data, mimeType: 'image/png' } + * ], + * 'session-1' + * ); + * ``` + */ + public async generate( + content: import('./types.js').ContentInput, + sessionId: string, + options?: import('./types.js').GenerateOptions + ): Promise { + // Collect all events from stream + const events: StreamingEvent[] = []; + + for await (const event of await this.stream(content, sessionId, options)) { + events.push(event); + } + + // Check for non-recoverable error events - only throw on fatal errors + // Recoverable errors (like tool failures) are included in the response for the caller to handle + const fatalErrorEvent = events.find( + (e): e is Extract => + e.name === 'llm:error' && e.recoverable !== true + ); + if (fatalErrorEvent) { + // If it's already a Dexto error (Runtime or Validation), throw it directly + if ( + fatalErrorEvent.error instanceof DextoRuntimeError || + fatalErrorEvent.error instanceof DextoValidationError + ) { + throw fatalErrorEvent.error; + } + // Otherwise wrap plain Error in DextoRuntimeError for proper HTTP status handling + const llmConfig = this.stateManager.getLLMConfig(sessionId); + throw LLMError.generationFailed( + fatalErrorEvent.error.message, + llmConfig.provider, + llmConfig.model + ); + } + + // Find the last llm:response event (final response after all tool calls) + const responseEvents = events.filter( + (e): e is Extract => e.name === 'llm:response' + ); + const responseEvent = responseEvents[responseEvents.length - 1]; + if (!responseEvent || responseEvent.name !== 'llm:response') { + // Get current LLM config for error context + const llmConfig = this.stateManager.getLLMConfig(sessionId); + throw LLMError.generationFailed( + 'Stream did not complete successfully - no response received', + llmConfig.provider, + llmConfig.model + ); + } + + // Collect tool calls from llm:tool-call events + const toolCallEvents = events.filter( + (e): e is Extract => + e.name === 'llm:tool-call' + ); + const toolResultEvents = events.filter( + (e): e is Extract => + e.name === 'llm:tool-result' + ); + + const toolCalls: import('./types.js').AgentToolCall[] = toolCallEvents.map((tc) => { + const toolResult = toolResultEvents.find((tr) => tr.callId === tc.callId); + return { + toolName: tc.toolName, + args: tc.args, + callId: tc.callId || `tool_${Date.now()}`, + result: toolResult + ? { + success: toolResult.success, + data: toolResult.sanitized, + } + : undefined, + }; + }); + + // Ensure usage matches LLMTokenUsage type with all required fields + const defaultUsage: import('./types.js').TokenUsage = { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + }; + const usage = responseEvent.tokenUsage ?? defaultUsage; + + return { + content: responseEvent.content, + reasoning: responseEvent.reasoning, + usage: usage as import('./types.js').TokenUsage, + toolCalls, + sessionId, + }; + } + + /** + * Stream a response (yields events as they arrive). + * This is the recommended method for real-time streaming UI updates. + * + * TODO: Refactor to move AsyncIterator down to LLM service level (Option 1). + * Streaming message API that returns core AgentEvents in real-time. + * Only emits STREAMING_EVENTS (tier 1 visibility) - events designed for real-time chat UIs. + * + * Events are forwarded directly from the AgentEventBus with no mapping layer, + * providing a unified event system across all API layers. + * + * @param content String message or array of content parts (text, images, files) + * @param sessionId Session ID for the conversation + * @param options Optional configuration (signal for cancellation) + * @returns AsyncIterator that yields StreamingEvent objects (core events with name property) + * + * @example + * ```typescript + * // Simple text + * for await (const event of await agent.stream('Write a poem', 'session-1')) { + * if (event.name === 'llm:chunk') process.stdout.write(event.content); + * } + * + * // Multimodal + * for await (const event of await agent.stream( + * [{ type: 'text', text: 'Describe this' }, { type: 'image', image: data, mimeType: 'image/png' }], + * 'session-1' + * )) { ... } + * ``` + */ + public async stream( + content: import('./types.js').ContentInput, + sessionId: string, + options?: import('./types.js').StreamOptions + ): Promise> { + this.ensureStarted(); + + // Validate sessionId is provided + if (!sessionId) { + throw AgentError.apiValidationError('sessionId is required'); + } + + const signal = options?.signal; + + // Normalize content: string -> [{ type: 'text', text: string }] + let contentParts: import('./types.js').ContentPart[] = + typeof content === 'string' ? [{ type: 'text', text: content }] : [...content]; + + // Event queue for aggregation - now holds core events directly + const eventQueue: StreamingEvent[] = []; + let completed = false; + + // Create AbortController for cleanup + const controller = new AbortController(); + const cleanupSignal = controller.signal; + + // Store controller so cancel() can abort this stream + this.activeStreamControllers.set(sessionId, controller); + + // Increase listener limit - stream() registers 12+ event listeners on this signal + setMaxListeners(30, cleanupSignal); + + // Track listener references for manual cleanup + // Using Function type here because listeners have different signatures per event + const listeners: Array<{ + event: StreamingEventName; + listener: Function; + }> = []; + + // Cleanup function to remove all listeners and stream controller + const cleanupListeners = () => { + if (listeners.length === 0) { + return; // Already cleaned up + } + for (const { event, listener } of listeners) { + this.agentEventBus.off( + event, + listener as Parameters[1] + ); + } + listeners.length = 0; + // Remove from active controllers map + this.activeStreamControllers.delete(sessionId); + }; + + // Wire external signal to trigger cleanup + if (signal) { + const abortHandler = () => { + cleanupListeners(); + controller.abort(); + }; + signal.addEventListener('abort', abortHandler, { once: true }); + } + + // TODO: Simplify these explicit subscriptions while keeping full type safety. + // The verbose approach avoids `any` and casts that were in the original loop. + // Potential approaches: function overloads on EventBus, or mapped type helpers. + // Subscribe to each streaming event with concrete types (spread works without cast) + const thinkingListener = (data: AgentEventMap['llm:thinking']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'llm:thinking', ...data }); + }; + this.agentEventBus.on('llm:thinking', thinkingListener, { signal: cleanupSignal }); + listeners.push({ event: 'llm:thinking', listener: thinkingListener }); + + const chunkListener = (data: AgentEventMap['llm:chunk']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'llm:chunk', ...data }); + }; + this.agentEventBus.on('llm:chunk', chunkListener, { signal: cleanupSignal }); + listeners.push({ event: 'llm:chunk', listener: chunkListener }); + + const responseListener = (data: AgentEventMap['llm:response']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'llm:response', ...data }); + // NOTE: We no longer set completed=true here. + // The iterator closes when run:complete is received, not llm:response. + // This allows queued messages to be processed after an LLM response. + }; + this.agentEventBus.on('llm:response', responseListener, { signal: cleanupSignal }); + listeners.push({ event: 'llm:response', listener: responseListener }); + + const toolCallListener = (data: AgentEventMap['llm:tool-call']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'llm:tool-call', ...data }); + }; + this.agentEventBus.on('llm:tool-call', toolCallListener, { signal: cleanupSignal }); + listeners.push({ event: 'llm:tool-call', listener: toolCallListener }); + + const toolResultListener = (data: AgentEventMap['llm:tool-result']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'llm:tool-result', ...data }); + }; + this.agentEventBus.on('llm:tool-result', toolResultListener, { signal: cleanupSignal }); + listeners.push({ event: 'llm:tool-result', listener: toolResultListener }); + + const errorListener = (data: AgentEventMap['llm:error']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'llm:error', ...data }); + if (!data.recoverable) { + completed = true; + } + }; + this.agentEventBus.on('llm:error', errorListener, { signal: cleanupSignal }); + listeners.push({ event: 'llm:error', listener: errorListener }); + + const unsupportedInputListener = (data: AgentEventMap['llm:unsupported-input']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'llm:unsupported-input', ...data }); + }; + this.agentEventBus.on('llm:unsupported-input', unsupportedInputListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'llm:unsupported-input', listener: unsupportedInputListener }); + + const titleUpdatedListener = (data: AgentEventMap['session:title-updated']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'session:title-updated', ...data }); + }; + this.agentEventBus.on('session:title-updated', titleUpdatedListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'session:title-updated', listener: titleUpdatedListener }); + + const approvalRequestListener = (data: AgentEventMap['approval:request']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'approval:request', ...data }); + }; + this.agentEventBus.on('approval:request', approvalRequestListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'approval:request', listener: approvalRequestListener }); + + const approvalResponseListener = (data: AgentEventMap['approval:response']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'approval:response', ...data }); + }; + this.agentEventBus.on('approval:response', approvalResponseListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'approval:response', listener: approvalResponseListener }); + + // Tool running event - emitted when tool execution starts (after approval if needed) + const toolRunningListener = (data: AgentEventMap['tool:running']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'tool:running', ...data }); + }; + this.agentEventBus.on('tool:running', toolRunningListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'tool:running', listener: toolRunningListener }); + + // Context compaction events - emitted when context is being compacted + const contextCompactingListener = (data: AgentEventMap['context:compacting']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'context:compacting', ...data }); + }; + this.agentEventBus.on('context:compacting', contextCompactingListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'context:compacting', listener: contextCompactingListener }); + + const contextCompactedListener = (data: AgentEventMap['context:compacted']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'context:compacted', ...data }); + }; + this.agentEventBus.on('context:compacted', contextCompactedListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'context:compacted', listener: contextCompactedListener }); + + // Message queue events (for mid-task user guidance) + const messageQueuedListener = (data: AgentEventMap['message:queued']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'message:queued', ...data }); + }; + this.agentEventBus.on('message:queued', messageQueuedListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'message:queued', listener: messageQueuedListener }); + + const messageDequeuedListener = (data: AgentEventMap['message:dequeued']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'message:dequeued', ...data }); + }; + this.agentEventBus.on('message:dequeued', messageDequeuedListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'message:dequeued', listener: messageDequeuedListener }); + + // Service events - extensible pattern for non-core services (e.g., sub-agent progress) + const serviceEventListener = (data: AgentEventMap['service:event']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'service:event', ...data }); + }; + this.agentEventBus.on('service:event', serviceEventListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'service:event', listener: serviceEventListener }); + + // Run lifecycle event - emitted when TurnExecutor truly finishes + // This is when we close the iterator (not on llm:response) + const runCompleteListener = (data: AgentEventMap['run:complete']) => { + if (data.sessionId !== sessionId) return; + eventQueue.push({ name: 'run:complete', ...data }); + completed = true; // NOW close the iterator + }; + this.agentEventBus.on('run:complete', runCompleteListener, { + signal: cleanupSignal, + }); + listeners.push({ event: 'run:complete', listener: runCompleteListener }); + + // Start streaming in background (fire-and-forget) + (async () => { + // Propagate sessionId through OpenTelemetry context for distributed tracing + const activeContext = context.active(); + const activeSpan = trace.getActiveSpan(); + + // Add sessionId to span attributes + if (activeSpan) { + activeSpan.setAttribute('sessionId', sessionId); + } + + // Preserve existing baggage entries and add sessionId + const existingBaggage = propagation.getBaggage(activeContext); + const baggageEntries: Record = {}; + if (existingBaggage) { + existingBaggage.getAllEntries().forEach(([key, entry]) => { + baggageEntries[key] = { ...entry }; + }); + } + baggageEntries.sessionId = { ...baggageEntries.sessionId, value: sessionId }; + + const updatedContext = propagation.setBaggage( + activeContext, + propagation.createBaggage(baggageEntries) + ); + + // Execute within updated OpenTelemetry context + await context.with(updatedContext, async () => { + try { + // Get session-specific LLM config for validation + const llmConfig = this.stateManager.getLLMConfig(sessionId); + + // Extract parts for validation + const textParts = contentParts.filter( + (p): p is import('./types.js').TextPart => p.type === 'text' + ); + const textContent = textParts.map((p) => p.text).join('\n'); + const imageParts = contentParts.filter( + (p): p is import('./types.js').ImagePart => p.type === 'image' + ); + const fileParts = contentParts.filter( + (p): p is import('./types.js').FilePart => p.type === 'file' + ); + + this.logger.debug( + `DextoAgent.stream: sessionId=${sessionId}, textLength=${textContent?.length ?? 0}, imageCount=${imageParts.length}, fileCount=${fileParts.length}` + ); + + // Validate ALL inputs early using session-specific config + // Validate text first (once) + const textValidation = validateInputForLLM( + { text: textContent }, + { provider: llmConfig.provider, model: llmConfig.model }, + this.logger + ); + ensureOk(textValidation, this.logger); + + // Validate each image + for (const imagePart of imageParts) { + const imageValidation = validateInputForLLM( + { + imageData: { + image: + typeof imagePart.image === 'string' + ? imagePart.image + : imagePart.image.toString(), + mimeType: imagePart.mimeType || 'image/png', + }, + }, + { provider: llmConfig.provider, model: llmConfig.model }, + this.logger + ); + ensureOk(imageValidation, this.logger); + } + + // Validate each file + for (const filePart of fileParts) { + const fileValidation = validateInputForLLM( + { + fileData: { + data: + typeof filePart.data === 'string' + ? filePart.data + : filePart.data.toString(), + mimeType: filePart.mimeType, + }, + }, + { provider: llmConfig.provider, model: llmConfig.model }, + this.logger + ); + ensureOk(fileValidation, this.logger); + } + + // Expand @resource mentions - returns ALL images as ContentPart[] + if (textContent.includes('@')) { + try { + const resources = await this.resourceManager.list(); + const expansion = await expandMessageReferences( + textContent, + resources, + (uri) => this.resourceManager.read(uri) + ); + + // Warn about unresolved references + if (expansion.unresolvedReferences.length > 0) { + const unresolvedNames = expansion.unresolvedReferences + .map((ref) => ref.originalRef) + .join(', '); + this.logger.warn( + `Could not resolve ${expansion.unresolvedReferences.length} resource reference(s): ${unresolvedNames}` + ); + } + + // Validate expanded message size (5MB limit) + const MAX_EXPANDED_SIZE = 5 * 1024 * 1024; // 5MB + const expandedSize = Buffer.byteLength( + expansion.expandedMessage, + 'utf-8' + ); + if (expandedSize > MAX_EXPANDED_SIZE) { + this.logger.warn( + `Expanded message size (${(expandedSize / 1024 / 1024).toFixed(2)}MB) exceeds limit (${MAX_EXPANDED_SIZE / 1024 / 1024}MB). Content may be truncated.` + ); + } + + // Update text parts with expanded message + contentParts = contentParts.filter((p) => p.type !== 'text'); + if (expansion.expandedMessage.trim()) { + contentParts.unshift({ + type: 'text', + text: expansion.expandedMessage, + }); + } + + // Add ALL extracted images to content parts + for (const img of expansion.extractedImages) { + contentParts.push({ + type: 'image', + image: img.image, + mimeType: img.mimeType, + }); + this.logger.debug( + `Added extracted image: ${img.name} (${img.mimeType})` + ); + } + } catch (error) { + this.logger.error( + `Failed to expand resource references: ${error instanceof Error ? error.message : String(error)}. Continuing with original message.` + ); + } + } + + // Validate that we have content after expansion - fallback to original if empty + const hasTextContent = contentParts.some( + (p) => p.type === 'text' && p.text.trim() + ); + const hasMediaContent = contentParts.some( + (p) => p.type === 'image' || p.type === 'file' + ); + if (!hasTextContent && !hasMediaContent) { + this.logger.warn( + 'Resource expansion resulted in empty content. Using original message.' + ); + contentParts = [{ type: 'text', text: textContent }]; + } + + // Get or create session + const session: ChatSession = + (await this.sessionManager.getSession(sessionId)) || + (await this.sessionManager.createSession(sessionId)); + + // Call session.stream() directly with ALL content parts + const _streamResult = await session.stream( + contentParts, + signal ? { signal } : undefined + ); + + // Increment message count + this.sessionManager + .incrementMessageCount(session.id) + .catch((error) => + this.logger.warn( + `Failed to increment message count: ${error instanceof Error ? error.message : String(error)}` + ) + ); + } catch (err) { + // Preserve typed errors, wrap unknown values in AgentError.streamFailed + const error = + err instanceof DextoRuntimeError || err instanceof DextoValidationError + ? err + : err instanceof Error + ? err + : AgentError.streamFailed(String(err)); + completed = true; + this.logger.error(`Error in DextoAgent.stream: ${error.message}`); + + const errorEvent: { name: 'llm:error' } & AgentEventMap['llm:error'] = { + name: 'llm:error', + error, + recoverable: false, + context: 'run_failed', + sessionId, + }; + eventQueue.push(errorEvent); + } + }); + })(); + + // Return async iterable iterator + const iterator: AsyncIterableIterator = { + async next(): Promise> { + // Wait for events + while (!completed && eventQueue.length === 0) { + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Check for abort (external signal OR internal via cancel()) + if (signal?.aborted || cleanupSignal.aborted) { + cleanupListeners(); + controller.abort(); + return { done: true, value: undefined }; + } + } + + // Return queued events + if (eventQueue.length > 0) { + return { done: false, value: eventQueue.shift()! }; + } + + // Stream completed + if (completed) { + cleanupListeners(); + controller.abort(); + return { done: true, value: undefined }; + } + + // Shouldn't reach here + cleanupListeners(); + return { done: true, value: undefined }; + }, + + async return(): Promise> { + // Called when consumer breaks out early or explicitly calls return() + cleanupListeners(); + controller.abort(); + return { done: true, value: undefined }; + }, + + [Symbol.asyncIterator]() { + return iterator; + }, + }; + + return iterator; + } + + /** + * Check if a session is currently processing a message. + * @param sessionId Session id + * @returns true if the session is busy processing; false otherwise + */ + public async isSessionBusy(sessionId: string): Promise { + this.ensureStarted(); + const session = await this.sessionManager.getSession(sessionId, false); + return session?.isBusy() ?? false; + } + + /** + * Queue a message for processing when a session is busy. + * The message will be injected into the conversation when the current turn completes. + * + * @param sessionId Session id + * @param message The user message to queue + * @returns Queue position and message ID + * @throws Error if session doesn't support message queueing + */ + public async queueMessage( + sessionId: string, + message: import('../session/message-queue.js').UserMessageInput + ): Promise<{ queued: true; position: number; id: string }> { + this.ensureStarted(); + const session = await this.sessionManager.getSession(sessionId, false); + if (!session) { + throw SessionError.notFound(sessionId); + } + return session.queueMessage(message); + } + + /** + * Get all queued messages for a session. + * @param sessionId Session id + * @returns Array of queued messages + */ + public async getQueuedMessages( + sessionId: string + ): Promise { + this.ensureStarted(); + const session = await this.sessionManager.getSession(sessionId, false); + if (!session) { + throw SessionError.notFound(sessionId); + } + return session.getQueuedMessages(); + } + + /** + * Remove a queued message. + * @param sessionId Session id + * @param messageId The ID of the queued message to remove + * @returns true if message was found and removed, false otherwise + */ + public async removeQueuedMessage(sessionId: string, messageId: string): Promise { + this.ensureStarted(); + const session = await this.sessionManager.getSession(sessionId, false); + if (!session) { + throw SessionError.notFound(sessionId); + } + return session.removeQueuedMessage(messageId); + } + + /** + * Clear all queued messages for a session. + * @param sessionId Session id + * @returns Number of messages that were cleared + */ + public async clearMessageQueue(sessionId: string): Promise { + this.ensureStarted(); + const session = await this.sessionManager.getSession(sessionId, false); + if (!session) { + throw SessionError.notFound(sessionId); + } + return session.clearMessageQueue(); + } + + /** + * Cancels the currently running turn for a session. + * Safe to call even if no run is in progress. + * @param sessionId Session id (required) + * @returns true if a run was in progress and was signaled to abort; false otherwise + */ + public async cancel(sessionId: string): Promise { + this.ensureStarted(); + + // Defensive runtime validation (protects against JavaScript callers, any types, @ts-ignore) + if (!sessionId || typeof sessionId !== 'string') { + throw AgentError.apiValidationError( + 'sessionId is required and must be a non-empty string' + ); + } + + // Abort the stream iterator first (so consumer's for-await loop exits cleanly) + const streamController = this.activeStreamControllers.get(sessionId); + if (streamController) { + streamController.abort(); + this.activeStreamControllers.delete(sessionId); + } + + // Then cancel the session's LLM/tool execution + const existing = await this.sessionManager.getSession(sessionId, false); + if (existing) { + return existing.cancel(); + } + // If no session found but stream was aborted, still return true + return !!streamController; + } + + // ============= SESSION MANAGEMENT ============= + + /** + * Creates a new chat session or returns an existing one. + * @param sessionId Optional session ID. If not provided, a UUID will be generated. + * @returns The created or existing ChatSession + */ + public async createSession(sessionId?: string): Promise { + this.ensureStarted(); + return await this.sessionManager.createSession(sessionId); + } + + /** + * Retrieves an existing session by ID. + * @param sessionId The session ID to retrieve + * @returns The ChatSession if found, undefined otherwise + */ + public async getSession(sessionId: string): Promise { + this.ensureStarted(); + return await this.sessionManager.getSession(sessionId); + } + + /** + * Lists all active session IDs. + * @returns Array of session IDs + */ + public async listSessions(): Promise { + this.ensureStarted(); + return await this.sessionManager.listSessions(); + } + + /** + * Ends a session by removing it from memory without deleting conversation history. + * Used for cleanup, agent shutdown, and session expiry. + * @param sessionId The session ID to end + */ + public async endSession(sessionId: string): Promise { + this.ensureStarted(); + // Clear session-level auto-approve tools to prevent memory leak + this.toolManager.clearSessionAutoApproveTools(sessionId); + return this.sessionManager.endSession(sessionId); + } + + /** + * Deletes a session and its conversation history permanently. + * Used for user-initiated permanent deletion. + * @param sessionId The session ID to delete + */ + public async deleteSession(sessionId: string): Promise { + this.ensureStarted(); + // Clear session-level auto-approve tools to prevent memory leak + this.toolManager.clearSessionAutoApproveTools(sessionId); + return this.sessionManager.deleteSession(sessionId); + } + + /** + * Gets metadata for a specific session. + * @param sessionId The session ID + * @returns The session metadata if found, undefined otherwise + */ + public async getSessionMetadata(sessionId: string): Promise { + this.ensureStarted(); + return await this.sessionManager.getSessionMetadata(sessionId); + } + + /** + * Sets a human-friendly title for the given session. + */ + public async setSessionTitle(sessionId: string, title: string): Promise { + this.ensureStarted(); + await this.sessionManager.setSessionTitle(sessionId, title); + } + + /** + * Gets the human-friendly title for the given session, if any. + */ + public async getSessionTitle(sessionId: string): Promise { + this.ensureStarted(); + return await this.sessionManager.getSessionTitle(sessionId); + } + + /** + * Generate a title for a session on-demand. + * Uses the first user message content to generate a descriptive title. + * + * @param sessionId Session ID to generate title for + * @returns Promise that resolves to the generated title, or null if generation failed + */ + public async generateSessionTitle(sessionId: string): Promise { + this.ensureStarted(); + + // Get session metadata to check if title already exists + const metadata = await this.sessionManager.getSessionMetadata(sessionId); + if (!metadata) { + throw SessionError.notFound(sessionId); + } + if (metadata.title) { + this.logger.debug( + `[SessionTitle] Session ${sessionId} already has title '${metadata.title}'` + ); + return metadata.title; + } + + // Get first user message + const session = await this.sessionManager.getSession(sessionId); + if (!session) { + throw SessionError.notFound(sessionId); + } + const history = await session.getHistory(); + const firstUserMsg = history.find((m) => m.role === 'user'); + if (!firstUserMsg) { + this.logger.debug(`[SessionTitle] No user message found for session ${sessionId}`); + return null; + } + + const userText = + typeof firstUserMsg.content === 'string' + ? firstUserMsg.content + : firstUserMsg.content + ?.filter((p) => p.type === 'text') + .map((p: { type: string; text: string }) => p.text) + .join(' '); + + if (!userText || !userText.trim()) { + this.logger.debug(`[SessionTitle] Empty user text for session ${sessionId}`); + return null; + } + + // Get LLM config for the session + const llmConfig = this.getEffectiveConfig(sessionId).llm; + + // Generate title + const result = await generateSessionTitle( + llmConfig, + this.toolManager, + this.systemPromptManager, + this.resourceManager, + userText, + this.logger + ); + + let title = result.title; + if (!title) { + // Fallback to heuristic + title = deriveHeuristicTitle(userText); + if (title) { + this.logger.info(`[SessionTitle] Using heuristic title for ${sessionId}: ${title}`); + } else { + this.logger.debug(`[SessionTitle] No suitable title derived for ${sessionId}`); + return null; + } + } else { + this.logger.info(`[SessionTitle] Generated LLM title for ${sessionId}: ${title}`); + } + + // Save title + await this.sessionManager.setSessionTitle(sessionId, title, { ifUnsetOnly: true }); + + return title; + } + + /** + * Gets the conversation history for a specific session. + * @param sessionId The session ID + * @returns Promise that resolves to the session's conversation history + * @throws Error if session doesn't exist + */ + public async getSessionHistory(sessionId: string): Promise { + this.ensureStarted(); + const session = await this.sessionManager.getSession(sessionId); + if (!session) { + throw SessionError.notFound(sessionId); + } + const history = await session.getHistory(); + if (!this.resourceManager) { + return history; + } + + return (await Promise.all( + history.map(async (message) => ({ + ...message, + content: await expandBlobReferences( + message.content, + this.resourceManager, + this.logger + ).catch((error) => { + this.logger.warn( + `Failed to expand blob references in message: ${error instanceof Error ? error.message : String(error)}` + ); + return message.content; // Return original content on error + }), + })) + )) as InternalMessage[]; + } + + /** + * Search for messages across all sessions or within a specific session + * + * @param query The search query string + * @param options Search options including session filter, role filter, and pagination + * @returns Promise that resolves to search results + */ + public async searchMessages( + query: string, + options: SearchOptions = {} + ): Promise { + this.ensureStarted(); + return await this.searchService.searchMessages(query, options); + } + + /** + * Search for sessions that contain the specified query + * + * @param query The search query string + * @returns Promise that resolves to session search results + */ + public async searchSessions(query: string): Promise { + this.ensureStarted(); + return await this.searchService.searchSessions(query); + } + + /** + * Resets the conversation history for a specific session. + * Keeps the session alive but the conversation history is cleared. + * @param sessionId Session ID (required) + */ + public async resetConversation(sessionId: string): Promise { + this.ensureStarted(); + + // Defensive runtime validation (protects against JavaScript callers, any types, @ts-ignore) + if (!sessionId || typeof sessionId !== 'string') { + throw AgentError.apiValidationError( + 'sessionId is required and must be a non-empty string' + ); + } + + try { + // Use SessionManager's resetSession method for better consistency + await this.sessionManager.resetSession(sessionId); + + this.logger.info(`DextoAgent conversation reset for session: ${sessionId}`); + this.agentEventBus.emit('session:reset', { + sessionId: sessionId, + }); + } catch (error) { + this.logger.error( + `Error during DextoAgent.resetConversation: ${error instanceof Error ? error.message : String(error)}` + ); + throw error; + } + } + + /** + * Clears the context window for a session without deleting history. + * + * This adds a "context clear" marker to the conversation history. When the + * context is loaded for LLM, messages before this marker are filtered out + * (via filterCompacted). The full history remains in the database for + * review via /resume or session history. + * + * Use this for /clear command - it preserves history but gives a fresh + * context window to the LLM. + * + * @param sessionId Session ID (required) + */ + public async clearContext(sessionId: string): Promise { + this.ensureStarted(); + + if (!sessionId || typeof sessionId !== 'string') { + throw AgentError.apiValidationError( + 'sessionId is required and must be a non-empty string' + ); + } + + const session = await this.sessionManager.getSession(sessionId); + if (!session) { + throw SessionError.notFound(sessionId); + } + + const contextManager = session.getContextManager(); + await contextManager.clearContext(); + + this.logger.info(`Context cleared for session: ${sessionId}`); + this.agentEventBus.emit('context:cleared', { + sessionId, + }); + } + + /** + * Manually compact the context for a session. + * + * Compaction generates a summary of older messages and adds it to the conversation history. + * When the context is loaded, filterCompacted() will exclude messages before the summary, + * effectively reducing the context window while preserving the full history in storage. + * + * @param sessionId Session ID of the session to compact (required) + * @returns Compaction result with stats, or null if compaction was skipped + */ + public async compactContext(sessionId: string): Promise<{ + /** The session that was compacted */ + sessionId: string; + /** Estimated tokens in context after compaction (includes system prompt, tools, and messages) */ + compactedContextTokens: number; + /** Number of messages before compaction */ + originalMessages: number; + /** Number of messages after compaction (summary + preserved) */ + compactedMessages: number; + } | null> { + this.ensureStarted(); + + if (!sessionId || typeof sessionId !== 'string') { + throw AgentError.apiValidationError( + 'sessionId is required and must be a non-empty string' + ); + } + + const session = await this.sessionManager.getSession(sessionId); + if (!session) { + throw SessionError.notFound(sessionId); + } + + // Get compaction strategy from the session's LLM service + const llmService = session.getLLMService(); + const compactionStrategy = llmService.getCompactionStrategy(); + + if (!compactionStrategy) { + this.logger.warn( + `Compaction strategy not configured for session ${sessionId} - skipping manual compaction` + ); + return null; + } + + // Get history and generate summary + const contextManager = session.getContextManager(); + const history = await contextManager.getHistory(); + + if (history.length < 4) { + this.logger.debug(`Compaction skipped for session ${sessionId} - history too short`); + return null; + } + + // Get full context estimate BEFORE compaction (includes system prompt, tools, messages) + // This uses the same calculation as /context command for consistency + const contributorContext = { mcpManager: this.mcpManager }; + const tools = await llmService.getAllTools(); + const beforeEstimate = await contextManager.getContextTokenEstimate( + contributorContext, + tools + ); + const originalTokens = beforeEstimate.estimated; + const originalMessages = beforeEstimate.stats.filteredMessageCount; + + // Emit compacting event + this.agentEventBus.emit('context:compacting', { + estimatedTokens: originalTokens, + sessionId, + }); + + // Generate summary message(s) + const summaryMessages = await compactionStrategy.compact(history); + + if (summaryMessages.length === 0) { + this.logger.debug(`Compaction skipped for session ${sessionId} - nothing to compact`); + this.agentEventBus.emit('context:compacted', { + originalTokens, + compactedTokens: originalTokens, + originalMessages, + compactedMessages: originalMessages, + strategy: compactionStrategy.name, + reason: 'manual', + sessionId, + }); + return null; + } + + // Add summary to history - filterCompacted() will exclude pre-summary messages at read-time + for (const summary of summaryMessages) { + await contextManager.addMessage(summary); + } + + // Reset actual token tracking since context has fundamentally changed + // The formula (lastInput + lastOutput + newEstimate) is no longer valid after compaction + contextManager.resetActualTokenTracking(); + + // Get full context estimate AFTER compaction (uses pure estimation since actuals were reset) + // This ensures /context will show the same value + const afterEstimate = await contextManager.getContextTokenEstimate( + contributorContext, + tools + ); + const compactedTokens = afterEstimate.estimated; + const compactedMessages = afterEstimate.stats.filteredMessageCount; + + this.agentEventBus.emit('context:compacted', { + originalTokens, + compactedTokens, + originalMessages, + compactedMessages, + strategy: compactionStrategy.name, + reason: 'manual', + sessionId, + }); + + this.logger.info( + `Compaction complete for session ${sessionId}: ` + + `${originalMessages} messages → ${compactedMessages} messages (~${compactedTokens} tokens)` + ); + + return { + sessionId, + compactedContextTokens: compactedTokens, + originalMessages, + compactedMessages, + }; + } + + /** + * Get context usage statistics for a session. + * Useful for monitoring context window usage and compaction status. + * + * @param sessionId Session ID (required) + * @returns Context statistics including token estimates and message counts + */ + public async getContextStats(sessionId: string): Promise<{ + estimatedTokens: number; + /** Last actual token count from LLM API (null if no calls made yet) */ + actualTokens: number | null; + /** Effective max context tokens (after applying maxContextTokens override and thresholdPercent) */ + maxContextTokens: number; + /** The model's raw context window before any config overrides */ + modelContextWindow: number; + /** Configured threshold percent (0.0-1.0), defaults to 1.0 */ + thresholdPercent: number; + usagePercent: number; + messageCount: number; + filteredMessageCount: number; + prunedToolCount: number; + hasSummary: boolean; + /** Current model identifier */ + model: string; + /** Display name for the model */ + modelDisplayName: string; + /** Detailed breakdown of context usage by category */ + breakdown: { + systemPrompt: number; + tools: { + total: number; + /** Per-tool token estimates */ + perTool: Array<{ name: string; tokens: number }>; + }; + messages: number; + }; + /** Calculation basis showing how the estimate was computed */ + calculationBasis?: { + /** 'actuals' = used lastInput + lastOutput + newEstimate, 'estimate' = pure estimation */ + method: 'actuals' | 'estimate'; + /** Last actual input tokens from API (if method is 'actuals') */ + lastInputTokens?: number; + /** Last actual output tokens from API (if method is 'actuals') */ + lastOutputTokens?: number; + /** Estimated tokens for new messages since last call (if method is 'actuals') */ + newMessagesEstimate?: number; + }; + }> { + this.ensureStarted(); + + if (!sessionId || typeof sessionId !== 'string') { + throw AgentError.apiValidationError( + 'sessionId is required and must be a non-empty string' + ); + } + + const session = await this.sessionManager.getSession(sessionId); + if (!session) { + throw SessionError.notFound(sessionId); + } + + const contextManager = session.getContextManager(); + + // Get token estimate using ContextManager's method (single source of truth) + const contributorContext = { mcpManager: this.mcpManager }; + const llmService = session.getLLMService(); + const tools = await llmService.getAllTools(); + + const tokenEstimate = await contextManager.getContextTokenEstimate( + contributorContext, + tools + ); + + // Get raw history for hasSummary check + const history = await contextManager.getHistory(); + // Get the effective max context tokens (compaction threshold takes priority) + const runtimeConfig = this.stateManager.getRuntimeConfig(sessionId); + const compactionConfig = runtimeConfig.compaction; + const modelContextWindow = contextManager.getMaxInputTokens(); + let maxContextTokens = modelContextWindow; + + // Apply compaction config overrides (same logic as vercel.ts) + // 1. maxContextTokens caps the context window (e.g., use 50K even if model supports 200K) + if (compactionConfig?.maxContextTokens !== undefined) { + maxContextTokens = Math.min(maxContextTokens, compactionConfig.maxContextTokens); + } + // 2. thresholdPercent triggers compaction early (default 90% to avoid context rot) + const thresholdPercent = compactionConfig?.thresholdPercent ?? 0.9; + if (thresholdPercent < 1.0) { + maxContextTokens = Math.floor(maxContextTokens * thresholdPercent); + } + + // Check if there's a summary in history (old isSummary or new isSessionSummary marker) + const hasSummary = history.some( + (msg) => msg.metadata?.isSummary === true || msg.metadata?.isSessionSummary === true + ); + + // Get model info for display + const llmConfig = runtimeConfig.llm; + const { getModelDisplayName } = await import('../llm/registry.js'); + const modelDisplayName = getModelDisplayName(llmConfig.model, llmConfig.provider); + + // Always use the calculated estimate (which includes lastInput + lastOutput + newMessages when actuals available) + const estimatedTokens = tokenEstimate.estimated; + + const autoCompactBuffer = + thresholdPercent > 0 && thresholdPercent < 1.0 + ? Math.floor((maxContextTokens * (1 - thresholdPercent)) / thresholdPercent) + : 0; + const totalTokenSpace = maxContextTokens + autoCompactBuffer; + const usedTokens = estimatedTokens + autoCompactBuffer; + + return { + estimatedTokens, + actualTokens: tokenEstimate.actual, + maxContextTokens, + modelContextWindow, + thresholdPercent, + usagePercent: + totalTokenSpace > 0 ? Math.round((usedTokens / totalTokenSpace) * 100) : 0, + messageCount: tokenEstimate.stats.originalMessageCount, + filteredMessageCount: tokenEstimate.stats.filteredMessageCount, + prunedToolCount: tokenEstimate.stats.prunedToolCount, + hasSummary, + model: llmConfig.model, + modelDisplayName, + breakdown: { + systemPrompt: tokenEstimate.breakdown.systemPrompt, + tools: tokenEstimate.breakdown.tools, + messages: tokenEstimate.breakdown.messages, + }, + ...(tokenEstimate.calculationBasis && { + calculationBasis: tokenEstimate.calculationBasis, + }), + }; + } + + // ============= LLM MANAGEMENT ============= + + /** + * Gets the current LLM configuration with all defaults applied. + * @returns Current LLM configuration + */ + public getCurrentLLMConfig(): ValidatedLLMConfig { + this.ensureStarted(); + return structuredClone(this.stateManager.getLLMConfig()); + } + + /** + * Switches the LLM service while preserving conversation history. + * This is a comprehensive method that handles ALL validation, configuration building, and switching internally. + * + * Design: + * - Input: Partial (allows optional fields like maxIterations?) + * - Output: LLMConfig (user-friendly type with all defaults applied) + * + * Key features: + * - Accepts partial LLM configuration object + * - Extracts and validates parameters internally + * - Infers provider from model if not provided + * - Automatically resolves API keys from environment variables + * - Uses LLMConfigSchema for comprehensive validation + * - Prevents inconsistent partial updates + * - Smart defaults for missing configuration values + * + * @param llmUpdates Partial LLM configuration object containing the updates to apply + * @param sessionId Session ID to switch LLM for. If not provided, switches for default session. Use '*' for all sessions + * @returns Promise that resolves with the validated LLM configuration + * @throws DextoLLMError if validation fails or switching fails + * + * @example + * ```typescript + * // Switch to a different model (provider will be inferred, API key auto-resolved) + * await agent.switchLLM({ model: 'gpt-5' }); + * + * // Switch to a different provider with explicit API key + * await agent.switchLLM({ provider: 'anthropic', model: 'claude-4-sonnet-20250514', apiKey: 'sk-ant-...' }); + * + * // Switch with session options + * await agent.switchLLM({ provider: 'anthropic', model: 'claude-4-sonnet-20250514' }, 'user-123'); + * + * // Switch for all sessions + * await agent.switchLLM({ model: 'gpt-5' }, '*'); + * ``` + */ + public async switchLLM( + llmUpdates: LLMUpdates, + sessionId?: string + ): Promise { + this.ensureStarted(); + + // Validate input using schema (single source of truth) + this.logger.debug(`DextoAgent.switchLLM: llmUpdates: ${safeStringify(llmUpdates)}`); + const parseResult = LLMUpdatesSchema.safeParse(llmUpdates); + if (!parseResult.success) { + const validation = fail(zodToIssues(parseResult.error, 'error')); + ensureOk(validation, this.logger); // This will throw DextoValidationError + throw new Error('Unreachable'); // For TypeScript + } + const validatedUpdates = parseResult.data; + + // Get current config for the session + const currentLLMConfig = sessionId + ? this.stateManager.getRuntimeConfig(sessionId).llm + : this.stateManager.getRuntimeConfig().llm; + + // Build and validate the new configuration using Result pattern internally + const result = await resolveAndValidateLLMConfig( + currentLLMConfig, + validatedUpdates, + this.logger + ); + const validatedConfig = ensureOk(result, this.logger); + + // Perform the actual LLM switch with validated config + await this.performLLMSwitch(validatedConfig, sessionId); + this.logger.info( + `DextoAgent.switchLLM: LLM switched to: ${safeStringify(validatedConfig)}` + ); + + // Log warnings if present + const warnings = result.issues.filter((issue) => issue.severity === 'warning'); + if (warnings.length > 0) { + this.logger.warn( + `LLM switch completed with warnings: ${warnings.map((w) => w.message).join(', ')}` + ); + } + + // Return the validated config directly + return validatedConfig; + } + + /** + * Performs the actual LLM switch with a validated configuration. + * This is a helper method that handles state management and session switching. + * + * @param validatedConfig - The validated LLM configuration to apply + * @param sessionScope - Session ID, '*' for all sessions, or undefined for default session + */ + private async performLLMSwitch( + validatedConfig: ValidatedLLMConfig, + sessionScope?: string + ): Promise { + // Update state manager (no validation needed - already validated) + this.stateManager.updateLLM(validatedConfig, sessionScope); + + // Switch LLM in session(s) + if (sessionScope === '*') { + await this.sessionManager.switchLLMForAllSessions(validatedConfig); + } else if (sessionScope) { + // Verify session exists before switching LLM + const session = await this.sessionManager.getSession(sessionScope); + if (!session) { + throw SessionError.notFound(sessionScope); + } + await this.sessionManager.switchLLMForSpecificSession(validatedConfig, sessionScope); + } else { + // No sessionScope provided - this is a configuration-level switch only + // State manager has been updated, individual sessions will pick up changes when created + this.logger.debug('LLM config updated at agent level (no active session switches)'); + } + } + + /** + * Gets all supported LLM providers. + * Returns a strongly-typed array of valid provider names that can be used with the agent. + * + * @returns Array of supported provider names + * + * @example + * ```typescript + * const providers = agent.getSupportedProviders(); + * console.log(providers); // ['openai', 'anthropic', 'google', 'groq'] + * ``` + */ + public getSupportedProviders(): LLMProvider[] { + return getSupportedProviders(); + } + + /** + * Gets all supported models grouped by provider with detailed information. + * Returns a strongly-typed object mapping each provider to its available models, + * including model metadata such as token limits and default status. + * + * @returns Object mapping provider names to their model information + * + * @example + * ```typescript + * const models = agent.getSupportedModels(); + * console.log(models.openai); // Array of OpenAI models with metadata + * console.log(models.anthropic[0].maxInputTokens); // Token limit for first Anthropic model + * + * // Check if a model is the default for its provider + * const hasDefault = models.google.some(model => model.isDefault); + * ``` + */ + public getSupportedModels(): Record< + LLMProvider, + Array + > { + const result = {} as Record< + LLMProvider, + Array + >; + + for (const provider of this.getSupportedProviders()) { + result[provider] = this.getSupportedModelsForProvider(provider); + } + + return result; + } + + /** + * Gets supported models for a specific provider. + * Returns model information including metadata for the specified provider only. + * For gateway providers like 'dexto' with supportsAllRegistryModels, returns + * all models from all accessible providers with their original provider info. + * + * @param provider The provider to get models for + * @returns Array of model information for the specified provider + * @throws Error if provider is not supported + * + * @example + * ```typescript + * try { + * const openaiModels = agent.getSupportedModelsForProvider('openai'); + * const defaultModel = openaiModels.find(model => model.isDefault); + * console.log(`Default OpenAI model: ${defaultModel?.name}`); + * } catch (error) { + * console.error('Unsupported provider'); + * } + * ``` + */ + public getSupportedModelsForProvider( + provider: LLMProvider + ): Array { + const models = getAllModelsForProvider(provider); + + return models.map((model) => { + // For inherited models, get default from the original provider + const originalProvider = + 'originalProvider' in model ? model.originalProvider : provider; + const defaultModel = getDefaultModelForProvider(originalProvider ?? provider); + + return { + ...model, + isDefault: model.name === defaultModel, + }; + }); + } + + /** + * Infers the provider from a model name. + * Searches through all supported providers to find which one supports the given model. + * + * @param modelName The model name to search for + * @returns The provider name if found, null if the model is not supported + * + * @example + * ```typescript + * const provider = agent.inferProviderFromModel('gpt-5'); + * console.log(provider); // 'openai' + * + * const provider2 = agent.inferProviderFromModel('claude-4-sonnet-20250514'); + * console.log(provider2); // 'anthropic' + * + * const provider3 = agent.inferProviderFromModel('unknown-model'); + * console.log(provider3); // null + * ``` + */ + public inferProviderFromModel(modelName: string): LLMProvider | null { + try { + return getProviderFromModel(modelName) as LLMProvider; + } catch { + return null; + } + } + + // ============= MCP SERVER MANAGEMENT ============= + + /** + * Adds a new MCP server to the runtime configuration and connects it if enabled. + * This method handles validation, state management, and establishing the connection. + * + * @param name The name of the server to add. + * @param config The configuration object for the server. + * @throws DextoError if validation fails or connection fails + */ + public async addMcpServer(name: string, config: McpServerConfig): Promise { + this.ensureStarted(); + + // Validate the server configuration + const existingServerNames = Object.keys(this.stateManager.getRuntimeConfig().mcpServers); + const validation = resolveAndValidateMcpServerConfig(name, config, existingServerNames); + const validatedConfig = ensureOk(validation, this.logger); + + // Add to runtime state (no validation needed - already validated) + this.stateManager.setMcpServer(name, validatedConfig); + + // Only connect if server is enabled (default is true) + if (validatedConfig.enabled === false) { + this.logger.info(`MCP server '${name}' added but not connected (disabled)`); + return; + } + + try { + // Connect the server + await this.mcpManager.connectServer(name, validatedConfig); + + // Ensure tool cache reflects the newly connected server before notifying listeners + await this.toolManager.refresh(); + + this.agentEventBus.emit('mcp:server-connected', { + name, + success: true, + }); + this.agentEventBus.emit('tools:available-updated', { + tools: Object.keys(await this.toolManager.getAllTools()), + source: 'mcp', + }); + + this.logger.info(`MCP server '${name}' added and connected successfully`); + + // Log warnings if present + const warnings = validation.issues.filter((i) => i.severity === 'warning'); + if (warnings.length > 0) { + this.logger.warn( + `MCP server connected with warnings: ${warnings.map((w) => w.message).join(', ')}` + ); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`Failed to connect MCP server '${name}': ${errorMessage}`); + + // Clean up state if connection failed + this.stateManager.removeMcpServer(name); + + this.agentEventBus.emit('mcp:server-connected', { + name, + success: false, + error: errorMessage, + }); + + throw MCPError.connectionFailed(name, errorMessage); + } + } + + /** + * @deprecated Use `addMcpServer` instead. This method will be removed in a future version. + */ + public async connectMcpServer(name: string, config: McpServerConfig): Promise { + return this.addMcpServer(name, config); + } + + /** + * Enables a disabled MCP server and connects it. + * Updates the runtime state to enabled=true and establishes the connection. + * + * @param name The name of the server to enable. + * @throws MCPError if server is not found or connection fails + */ + public async enableMcpServer(name: string): Promise { + this.ensureStarted(); + + const currentConfig = this.stateManager.getRuntimeConfig().mcpServers[name]; + if (!currentConfig) { + throw MCPError.serverNotFound(name); + } + + // Update state with enabled=true + const updatedConfig = { ...currentConfig, enabled: true }; + this.stateManager.setMcpServer(name, updatedConfig); + + try { + // Connect the server + await this.mcpManager.connectServer(name, updatedConfig); + await this.toolManager.refresh(); + + this.agentEventBus.emit('mcp:server-connected', { name, success: true }); + this.logger.info(`MCP server '${name}' enabled and connected`); + } catch (error) { + // Revert state on failure + this.stateManager.setMcpServer(name, currentConfig); + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`Failed to enable MCP server '${name}': ${errorMessage}`); + throw MCPError.connectionFailed(name, errorMessage); + } + } + + /** + * Disables an MCP server and disconnects it. + * Updates the runtime state to enabled=false and closes the connection. + * + * @param name The name of the server to disable. + * @throws MCPError if server is not found or disconnect fails + */ + public async disableMcpServer(name: string): Promise { + this.ensureStarted(); + + const currentConfig = this.stateManager.getRuntimeConfig().mcpServers[name]; + if (!currentConfig) { + throw MCPError.serverNotFound(name); + } + + // Update state with enabled=false + const updatedConfig = { ...currentConfig, enabled: false }; + this.stateManager.setMcpServer(name, updatedConfig); + + try { + // Disconnect the server + await this.mcpManager.removeClient(name); + await this.toolManager.refresh(); + + this.logger.info(`MCP server '${name}' disabled and disconnected`); + } catch (error) { + // Revert state on failure + this.stateManager.setMcpServer(name, currentConfig); + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`Failed to disable MCP server '${name}': ${errorMessage}`); + throw MCPError.disconnectionFailed(name, errorMessage); + } + } + + /** + * Removes and disconnects an MCP server completely. + * Use this for deleting a server - removes from both runtime state and disconnects. + * @param name The name of the server to remove. + * @throws MCPError if disconnection fails + */ + public async removeMcpServer(name: string): Promise { + this.ensureStarted(); + + try { + // Disconnect the client first + await this.mcpManager.removeClient(name); + + // Then remove from runtime state + this.stateManager.removeMcpServer(name); + + // Refresh tool cache after server removal so the LLM sees updated set + await this.toolManager.refresh(); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`Failed to remove MCP server '${name}': ${errorMessage}`); + throw MCPError.disconnectionFailed(name, errorMessage); + } + } + + /** + * Restarts an MCP server by disconnecting and reconnecting with its original configuration. + * This is useful for recovering from server errors or applying configuration changes. + * @param name The name of the server to restart. + * @throws MCPError if server is not found or restart fails + */ + public async restartMcpServer(name: string): Promise { + this.ensureStarted(); + + try { + this.logger.info(`DextoAgent: Restarting MCP server '${name}'...`); + + // Restart the server using MCPManager + await this.mcpManager.restartServer(name); + + // Refresh tool cache after restart so the LLM sees updated toolset + await this.toolManager.refresh(); + + this.agentEventBus.emit('mcp:server-restarted', { + serverName: name, + }); + this.agentEventBus.emit('tools:available-updated', { + tools: Object.keys(await this.toolManager.getAllTools()), + source: 'mcp', + }); + + this.logger.info(`DextoAgent: Successfully restarted MCP server '${name}'.`); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error( + `DextoAgent: Failed to restart MCP server '${name}': ${errorMessage}` + ); + + // Note: No event emitted on failure since the error is thrown + // The calling layer (API) will handle error reporting + + throw error; + } + } + + /** + * Executes a tool from any source (MCP servers, custom tools, or internal tools). + * This is the unified interface for tool execution that can handle all tool types. + * + * Note: This is for direct/programmatic tool execution outside of LLM flow. + * A toolCallId is generated automatically for tracking purposes. + * + * @param toolName The name of the tool to execute + * @param args The arguments to pass to the tool + * @returns The result of the tool execution + */ + public async executeTool(toolName: string, args: any): Promise { + this.ensureStarted(); + // Generate a toolCallId for direct API calls (not from LLM) + const toolCallId = `direct-${randomUUID()}`; + return await this.toolManager.executeTool(toolName, args, toolCallId); + } + + /** + * Gets all available tools from all connected MCP servers. + * Useful for users to discover what tools are available. + * @returns Promise resolving to a map of tool names to tool definitions + */ + public async getAllMcpTools(): Promise { + this.ensureStarted(); + return await this.mcpManager.getAllTools(); + } + + /** + * Gets all MCP tools with their server metadata. + * Returns tool cache entries which include server names for proper grouping. + * @returns Map of tool names to cache entries with server info + */ + public getAllMcpToolsWithServerInfo() { + this.ensureStarted(); + return this.mcpManager.getAllToolsWithServerInfo(); + } + + /** + * Gets all available tools from all sources (MCP servers and custom tools). + * This is the unified interface for tool discovery that includes both MCP and custom tools. + * @returns Promise resolving to a map of tool names to tool definitions + */ + public async getAllTools(): Promise { + this.ensureStarted(); + return await this.toolManager.getAllTools(); + } + + /** + * Gets all connected MCP clients. + * Used by the API layer to inspect client status. + * @returns Map of client names to client instances + */ + public getMcpClients(): Map { + this.ensureStarted(); + return this.mcpManager.getClients(); + } + + /** + * Gets all failed MCP connections. + * Used by the API layer to report connection errors. + * @returns Record of failed connection names to error messages + */ + public getMcpFailedConnections(): Record { + this.ensureStarted(); + return this.mcpManager.getFailedConnections(); + } + + /** + * Gets the connection status of a single MCP server. + * @param name The server name + * @returns The connection status, or undefined if server not configured + * + * TODO: Move to MCPManager once it has access to server configs (enabled state). + * Currently here because MCPManager only tracks connections, not config. + */ + public getMcpServerStatus(name: string): McpServerStatus | undefined { + this.ensureStarted(); + const config = this.stateManager.getRuntimeConfig(); + const serverConfig = config.mcpServers[name]; + if (!serverConfig) return undefined; + + const enabled = serverConfig.enabled !== false; + const connectedClients = this.mcpManager.getClients(); + const failedConnections = this.mcpManager.getFailedConnections(); + + let status: McpConnectionStatus; + if (!enabled) { + status = 'disconnected'; + } else if (connectedClients.has(name)) { + status = 'connected'; + } else { + status = 'error'; + } + + const result: McpServerStatus = { + name, + type: serverConfig.type, + enabled, + status, + }; + if (failedConnections[name]) { + result.error = failedConnections[name]; + } + return result; + } + + /** + * Gets all configured MCP servers with their connection status. + * Centralizes the status computation logic used by CLI, server, and webui. + * @returns Array of server info with computed status + * + * TODO: Move to MCPManager once it has access to server configs (enabled state). + * Currently here because MCPManager only tracks connections, not config. + */ + public getMcpServersWithStatus(): McpServerStatus[] { + this.ensureStarted(); + const config = this.stateManager.getRuntimeConfig(); + const mcpServers = config.mcpServers || {}; + const connectedClients = this.mcpManager.getClients(); + const failedConnections = this.mcpManager.getFailedConnections(); + + const servers: McpServerStatus[] = []; + + for (const [name, serverConfig] of Object.entries(mcpServers)) { + const enabled = serverConfig.enabled !== false; + let status: McpConnectionStatus; + + if (!enabled) { + status = 'disconnected'; + } else if (connectedClients.has(name)) { + status = 'connected'; + } else { + status = 'error'; + } + + const server: McpServerStatus = { + name, + type: serverConfig.type, + enabled, + status, + }; + if (failedConnections[name]) { + server.error = failedConnections[name]; + } + servers.push(server); + } + + return servers; + } + + // ============= RESOURCE MANAGEMENT ============= + + /** + * Lists all available resources with their info. + * This includes resources from MCP servers and any custom resource providers. + */ + public async listResources(): Promise { + this.ensureStarted(); + return await this.resourceManager.list(); + } + + /** + * Checks if a resource exists by URI. + */ + public async hasResource(uri: string): Promise { + this.ensureStarted(); + return await this.resourceManager.has(uri); + } + + /** + * Reads the content of a specific resource by URI. + */ + public async readResource( + uri: string + ): Promise { + this.ensureStarted(); + return await this.resourceManager.read(uri); + } + + /** + * Lists resources for a specific MCP server. + */ + public async listResourcesForServer(serverId: string): Promise< + Array<{ + uri: string; + name: string; + originalUri: string; + serverName: string; + }> + > { + this.ensureStarted(); + const allResources = await this.resourceManager.list(); + const serverResources = Object.values(allResources) + .filter((resource) => resource.serverName === serverId) + .map((resource) => { + const original = (resource.metadata?.originalUri as string) ?? resource.uri; + const name = resource.name ?? resource.uri.split('/').pop() ?? resource.uri; + const serverName = resource.serverName ?? serverId; + return { uri: original, name, originalUri: original, serverName }; + }); + return serverResources; + } + + // ============= PROMPT MANAGEMENT ============= + + /** + * Gets the current system prompt with all dynamic content resolved. + * This method builds the complete prompt by invoking all configured prompt contributors + * (static content, dynamic placeholders, MCP resources, etc.) and returns the final + * prompt string that will be sent to the LLM. + * + * Useful for debugging prompt issues, inspecting what context the AI receives, + * and understanding how dynamic content is being incorporated. + * + * @returns Promise resolving to the complete system prompt string + * + * @example + * ```typescript + * // Get the current system prompt for inspection + * const prompt = await agent.getSystemPrompt(); + * console.log('Current system prompt:', prompt); + * + * // Useful for debugging prompt-related issues + * if (response.quality === 'poor') { + * const prompt = await agent.getSystemPrompt(); + * console.log('Check if prompt includes expected context:', prompt); + * } + * ``` + */ + public async getSystemPrompt(): Promise { + this.ensureStarted(); + const context = { + mcpManager: this.mcpManager, + }; + return await this.systemPromptManager.build(context); + } + + /** + * Lists all available prompts from all providers (MCP, internal, starter, custom). + * @returns Promise resolving to a PromptSet with all available prompts + */ + public async listPrompts(): Promise { + this.ensureStarted(); + return await this.promptManager.list(); + } + + /** + * Gets the definition of a specific prompt by name. + * @param name The name of the prompt + * @returns Promise resolving to the prompt definition or null if not found + */ + public async getPromptDefinition( + name: string + ): Promise { + this.ensureStarted(); + return await this.promptManager.getPromptDefinition(name); + } + + /** + * Checks if a prompt exists. + * @param name The name of the prompt to check + * @returns Promise resolving to true if the prompt exists, false otherwise + */ + public async hasPrompt(name: string): Promise { + this.ensureStarted(); + return await this.promptManager.has(name); + } + + /** + * Refreshes the prompts cache, reloading from all providers. + * Call this after adding/deleting prompts to make them immediately available. + * + * @param newPrompts Optional - if provided, updates the config prompts before refreshing. + * Use this when you've modified the agent config file and need to + * update both the runtime config and refresh the cache. + */ + public async refreshPrompts(newPrompts?: PromptsConfig): Promise { + this.ensureStarted(); + if (newPrompts) { + this.promptManager.updateConfigPrompts(newPrompts); + } + await this.promptManager.refresh(); + } + + /** + * Gets a prompt with its messages. + * @param name The name of the prompt + * @param args Optional arguments to pass to the prompt + * @returns Promise resolving to the prompt result with messages + */ + public async getPrompt( + name: string, + args?: Record + ): Promise { + this.ensureStarted(); + return await this.promptManager.getPrompt(name, args); + } + + /** + * Creates a new custom prompt. + * @param input The prompt creation input + * @returns Promise resolving to the created prompt info + */ + public async createCustomPrompt( + input: import('../prompts/index.js').CreateCustomPromptInput + ): Promise { + this.ensureStarted(); + return await this.promptManager.createCustomPrompt(input); + } + + /** + * Deletes a custom prompt by name. + * @param name The name of the custom prompt to delete + */ + public async deleteCustomPrompt(name: string): Promise { + this.ensureStarted(); + return await this.promptManager.deleteCustomPrompt(name); + } + + /** + * Resolves a prompt to its text content with all arguments applied. + * This is a high-level method that handles: + * - Prompt key resolution (resolving aliases) + * - Argument normalization (including special _context field) + * - Prompt execution and flattening + * - Returning per-prompt overrides (allowedTools, model) for the invoker to apply + * + * @param name The prompt name or alias + * @param options Optional configuration for prompt resolution + * @returns Promise resolving to the resolved text, resource URIs, and optional overrides + */ + public async resolvePrompt( + name: string, + options: { + context?: string; + args?: Record; + } = {} + ): Promise { + this.ensureStarted(); + return await this.promptManager.resolvePrompt(name, options); + } + + // ============= CONFIGURATION ACCESS ============= + + /** + * Gets the effective configuration for a session or the default configuration. + * @param sessionId Optional session ID. If not provided, returns default config. + * @returns The effective configuration object (validated with defaults applied) + * @remarks Requires agent to be started. Use `agent.config` for pre-start access. + */ + public getEffectiveConfig(sessionId?: string): Readonly { + this.ensureStarted(); + return sessionId + ? this.stateManager.getRuntimeConfig(sessionId) + : this.stateManager.getRuntimeConfig(); + } + + /** + * Gets the file path of the agent configuration currently in use. + * This returns the source agent file path, not session-specific overrides. + * @returns The path to the agent configuration file + * @throws AgentError if no config path is available + */ + public getAgentFilePath(): string { + if (!this.configPath) { + throw AgentError.noConfigPath(); + } + return this.configPath; + } + + /** + * Reloads the agent configuration with a new config object. + * Validates the new config, detects what changed, and automatically + * restarts the agent if necessary to apply the changes. + * + * @param newConfig The new agent configuration to apply + * @returns Object containing whether agent was restarted and list of changes applied + * @throws Error if config is invalid or restart fails + * + * TODO: improve hot reload capabilites so that we don't always require a restart + */ + public async reload(newConfig: AgentConfig): Promise<{ + restarted: boolean; + changesApplied: string[]; + }> { + this.logger.info('Reloading agent configuration'); + + const oldConfig = this.config; + const validated = AgentConfigSchema.parse(newConfig); + + // Detect what changed + const changesApplied = this.detectConfigChanges(oldConfig, validated); + + // Update the config reference + this.config = validated; + + let restarted = false; + if (changesApplied.length > 0) { + this.logger.info( + `Configuration changed. Restarting agent to apply: ${changesApplied.join(', ')}` + ); + await this.restart(); + restarted = true; + this.logger.info('Agent restarted successfully with new configuration'); + } else { + this.logger.info('Agent configuration reloaded successfully (no changes detected)'); + } + + return { + restarted, + changesApplied, + }; + } + + /** + * Detects configuration changes that require a full agent restart. + * Pure comparison logic - no file I/O. + * Returns an array of change descriptions. + * + * @param oldConfig Previous validated configuration + * @param newConfig New validated configuration + * @returns Array of restart-required change descriptions + */ + public detectConfigChanges( + oldConfig: ValidatedAgentConfig, + newConfig: ValidatedAgentConfig + ): string[] { + const changes: string[] = []; + + // Storage backend changes require restart + if (JSON.stringify(oldConfig.storage) !== JSON.stringify(newConfig.storage)) { + changes.push('Storage backend'); + } + + // Session config changes require restart (maxSessions, sessionTTL are readonly) + if (JSON.stringify(oldConfig.sessions) !== JSON.stringify(newConfig.sessions)) { + changes.push('Session configuration'); + } + + // System prompt changes require restart (PromptManager caches contributors) + if (JSON.stringify(oldConfig.systemPrompt) !== JSON.stringify(newConfig.systemPrompt)) { + changes.push('System prompt'); + } + + // Tool confirmation changes require restart (ConfirmationProvider caches config) + if ( + JSON.stringify(oldConfig.toolConfirmation) !== + JSON.stringify(newConfig.toolConfirmation) + ) { + changes.push('Tool confirmation'); + } + + // Internal tools changes require restart (InternalToolsProvider caches config) + if (JSON.stringify(oldConfig.internalTools) !== JSON.stringify(newConfig.internalTools)) { + changes.push('Internal tools'); + } + + // MCP server changes require restart + if (JSON.stringify(oldConfig.mcpServers) !== JSON.stringify(newConfig.mcpServers)) { + changes.push('MCP servers'); + } + + // LLM configuration changes require restart + if ( + oldConfig.llm.provider !== newConfig.llm.provider || + oldConfig.llm.model !== newConfig.llm.model || + oldConfig.llm.apiKey !== newConfig.llm.apiKey + ) { + changes.push('LLM configuration'); + } + + return changes; + } + + // ============= APPROVAL HANDLER API ============= + + /** + * Set a custom approval handler for manual approval mode. + * + * When `toolConfirmation.mode` is set to 'manual', an approval handler must be + * provided to process tool confirmation requests. The handler will be called + * whenever a tool execution requires user approval. + * + * The handler receives an approval request and must return a promise that resolves + * to an approval response with the user's decision (approved/denied/cancelled). + * + * @param handler The approval handler function + * + * @example + * ```typescript + * import { ApprovalStatus } from '@dexto/core'; + * + * agent.setApprovalHandler(async (request) => { + * // Present approval request to user (CLI, UI, webhook, etc.) + * console.log(`Approve tool: ${request.metadata.toolName}?`); + * console.log(`Args: ${JSON.stringify(request.metadata.args)}`); + * + * // Collect user's decision (this is just an example) + * const approved = await getUserInput(); + * + * return { + * approvalId: request.approvalId, + * status: approved ? ApprovalStatus.APPROVED : ApprovalStatus.DENIED, + * sessionId: request.sessionId, + * }; + * }); + * ``` + */ + public setApprovalHandler(handler: ApprovalHandler): void { + // Store handler for use during start() if not yet started + this.approvalHandler = handler; + + // If agent is already started, also set it on the service immediately + if (this._isStarted && this.services) { + this.services.approvalManager.setHandler(handler); + } + + this.logger.debug('Approval handler registered'); + } + + /** + * Clear the current approval handler. + * + * After calling this, manual approval mode will fail if a tool requires approval. + */ + public clearApprovalHandler(): void { + // Clear stored handler + this.approvalHandler = undefined; + + // If agent is already started, also clear it on the service + if (this._isStarted && this.services) { + this.services.approvalManager.clearHandler(); + } + + this.logger.debug('Approval handler cleared'); + } + + // ============= AGENT MANAGEMENT ============= + // Note: Agent management methods have been moved to the Dexto orchestrator class. + // See: /packages/core/src/Dexto.ts + // + // For agent lifecycle operations (list, install, uninstall, create), use: + // ```typescript + // await Dexto.listAgents(); // Static + // await Dexto.installAgent(name); // Static + // await Dexto.installCustomAgent(name, path, metadata); // Static + // await Dexto.uninstallAgent(name); // Static + // + // const dexto = new Dexto(); + // await dexto.createAgent(name); // Instance method + // ``` + + // Future methods could encapsulate more complex agent behaviors: + // - Multi-step task execution with progress tracking + // - Memory and context management across sessions + // - Tool chaining and workflow automation + // - Agent collaboration and delegation +} diff --git a/dexto/packages/core/src/agent/agentCard.ts b/dexto/packages/core/src/agent/agentCard.ts new file mode 100644 index 00000000..97e649c2 --- /dev/null +++ b/dexto/packages/core/src/agent/agentCard.ts @@ -0,0 +1,52 @@ +import type { AgentCard } from './schemas.js'; +import { AgentCardSchema } from '@core/agent/schemas.js'; + +/** + * Default agent description used when not provided + */ +const DEFAULT_AGENT_DESCRIPTION = + 'Dexto is an AI assistant capable of chat and task delegation, accessible via multiple protocols.'; + +/** + * Minimal runtime context needed to establish defaults + * if not provided in AgentCardOverride or by AgentCardSchema. + */ +export interface MinimalAgentCardContext { + defaultName: string; // Ultimate fallback name if not in overrides + defaultVersion: string; // Ultimate fallback version if not in overrides + defaultBaseUrl: string; // Used to construct default URL if not in overrides +} + +/** + * Creates the final AgentCard by merging context-defined values with user-provided overrides, + * then uses AgentCardSchema.parse() to apply schema-defined static defaults and perform validation. + */ +export function createAgentCard( + context: MinimalAgentCardContext, + overrides?: Partial // Updated type from AgentCardOverride to Partial +): AgentCard { + const { defaultName, defaultVersion, defaultBaseUrl } = context; + + // Start with overrides (which are now Partial or {}) + const effectiveInput: Record = { ...(overrides || {}) }; + + // Layer in context-dependent required fields if not already provided by overrides. + effectiveInput.name = overrides?.name ?? defaultName; + effectiveInput.version = overrides?.version ?? defaultVersion; + effectiveInput.url = overrides?.url ?? `${defaultBaseUrl}/mcp`; + effectiveInput.description = overrides?.description ?? DEFAULT_AGENT_DESCRIPTION; + + // Handle capabilities - pushNotifications defaults to false (no WebSocket support) + const capsFromInput = effectiveInput.capabilities; + effectiveInput.capabilities = { + ...(capsFromInput ?? {}), + pushNotifications: capsFromInput?.pushNotifications ?? false, + }; + + // If input specifies an empty skills array, this means "use schema default skills". + if (effectiveInput.skills && effectiveInput.skills.length === 0) { + effectiveInput.skills = undefined; + } + + return AgentCardSchema.parse(effectiveInput); +} diff --git a/dexto/packages/core/src/agent/error-codes.ts b/dexto/packages/core/src/agent/error-codes.ts new file mode 100644 index 00000000..00a7449e --- /dev/null +++ b/dexto/packages/core/src/agent/error-codes.ts @@ -0,0 +1,23 @@ +/** + * Agent-specific error codes + * Includes agent configuration and lifecycle errors only + * Domain-specific errors (LLM, Session, MCP, etc.) belong in their respective modules + */ +export enum AgentErrorCode { + // Lifecycle + NOT_STARTED = 'agent_not_started', + ALREADY_STARTED = 'agent_already_started', + STOPPED = 'agent_stopped', + INITIALIZATION_FAILED = 'agent_initialization_failed', + SWITCH_IN_PROGRESS = 'agent_switch_in_progress', + + // Configuration + NO_CONFIG_PATH = 'agent_no_config_path', + INVALID_CONFIG = 'agent_invalid_config', + + // API layer + API_VALIDATION_ERROR = 'agent_api_validation_error', + + // Runtime + STREAM_FAILED = 'agent_stream_failed', +} diff --git a/dexto/packages/core/src/agent/errors.ts b/dexto/packages/core/src/agent/errors.ts new file mode 100644 index 00000000..a0595489 --- /dev/null +++ b/dexto/packages/core/src/agent/errors.ts @@ -0,0 +1,122 @@ +import { DextoRuntimeError } from '@core/errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { AgentErrorCode } from './error-codes.js'; + +/** + * Agent-specific error factory + * Creates properly typed errors for Agent operations + * Note: Domain-specific errors (LLM, Session, MCP) have been moved to their respective modules + */ +export class AgentError { + /** + * Agent not started + */ + static notStarted() { + return new DextoRuntimeError( + AgentErrorCode.NOT_STARTED, + ErrorScope.AGENT, + ErrorType.USER, + 'Agent must be started before use', + undefined, + 'Call agent.start() before using other methods' + ); + } + + /** + * Agent already started + */ + static alreadyStarted() { + return new DextoRuntimeError( + AgentErrorCode.ALREADY_STARTED, + ErrorScope.AGENT, + ErrorType.USER, + 'Agent is already started', + undefined, + 'Call agent.stop() before starting again' + ); + } + + /** + * Agent stopped + */ + static stopped() { + return new DextoRuntimeError( + AgentErrorCode.STOPPED, + ErrorScope.AGENT, + ErrorType.USER, + 'Agent has been stopped and cannot be used', + undefined, + 'Create a new agent instance or restart this one' + ); + } + + /** + * Agent switch in progress + */ + static switchInProgress() { + return new DextoRuntimeError( + AgentErrorCode.SWITCH_IN_PROGRESS, + ErrorScope.AGENT, + ErrorType.CONFLICT, + 'Agent switch already in progress', + undefined, + 'Wait for the current switch operation to complete before starting a new one' + ); + } + + /** + * Agent initialization failed + */ + static initializationFailed(reason: string, details?: unknown) { + return new DextoRuntimeError( + AgentErrorCode.INITIALIZATION_FAILED, + ErrorScope.AGENT, + ErrorType.SYSTEM, + `Agent initialization failed: ${reason}`, + details, + 'Check logs for initialization errors' + ); + } + + /** + * No config path available + */ + static noConfigPath() { + return new DextoRuntimeError( + AgentErrorCode.NO_CONFIG_PATH, + ErrorScope.AGENT, + ErrorType.SYSTEM, + 'No configuration file path is available', + undefined, + 'Agent was created without a config file path, cannot perform file operations' + ); + } + + /** + * API validation error + */ + static apiValidationError(message: string, details?: unknown) { + return new DextoRuntimeError( + AgentErrorCode.API_VALIDATION_ERROR, + ErrorScope.AGENT, + ErrorType.USER, + message, + details, + 'Check the request parameters and try again' + ); + } + + /** + * Stream failed with unexpected error + */ + static streamFailed(message: string, details?: unknown) { + return new DextoRuntimeError( + AgentErrorCode.STREAM_FAILED, + ErrorScope.AGENT, + ErrorType.SYSTEM, + message, + details, + 'Check logs for details' + ); + } +} diff --git a/dexto/packages/core/src/agent/index.ts b/dexto/packages/core/src/agent/index.ts new file mode 100644 index 00000000..001ae30a --- /dev/null +++ b/dexto/packages/core/src/agent/index.ts @@ -0,0 +1,29 @@ +export { DextoAgent } from './DextoAgent.js'; +export { + AgentConfigSchema, + AgentCardSchema, + SecuritySchemeSchema, + type AgentCard, + type ValidatedAgentCard, +} from './schemas.js'; +export { + type ValidatedAgentConfig, + type AgentConfig, + type LLMValidationOptions, + createAgentConfigSchema, +} from './schemas.js'; +export { createAgentCard } from './agentCard.js'; +export * from './errors.js'; +export * from './error-codes.js'; + +// New generate/stream API types +export type { + ContentInput, + GenerateOptions, + GenerateResponse, + StreamOptions, + AgentToolCall, +} from './types.js'; + +// Stream events are now core AgentEvents (exported from events module) +export type { StreamingEvent, StreamingEventName, STREAMING_EVENTS } from '../events/index.js'; diff --git a/dexto/packages/core/src/agent/schemas.test.ts b/dexto/packages/core/src/agent/schemas.test.ts new file mode 100644 index 00000000..91a29803 --- /dev/null +++ b/dexto/packages/core/src/agent/schemas.test.ts @@ -0,0 +1,789 @@ +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { + AgentCardSchema, + AgentConfigSchema, + type AgentCard, + type ValidatedAgentCard, + type AgentConfig, +} from './schemas.js'; + +describe('AgentCardSchema', () => { + const validAgentCard: AgentCard = { + name: 'TestAgent', + description: 'A test agent for validation', + url: 'https://agent.example.com', + version: '1.0.0', + }; + + describe('Basic Structure Validation', () => { + it('should accept valid minimal config', () => { + const result = AgentCardSchema.parse(validAgentCard); + + expect(result.name).toBe('TestAgent'); + expect(result.url).toBe('https://agent.example.com'); + expect(result.version).toBe('1.0.0'); + }); + + it('should apply default values', () => { + const result = AgentCardSchema.parse(validAgentCard); + + expect(result.protocolVersion).toBe('0.3.0'); + expect(result.preferredTransport).toBe('JSONRPC'); + expect(result.description).toBe('A test agent for validation'); + expect(result.capabilities.streaming).toBe(true); + expect(result.capabilities.stateTransitionHistory).toBe(false); + expect(result.defaultInputModes).toEqual(['application/json', 'text/plain']); + expect(result.defaultOutputModes).toEqual([ + 'application/json', + 'text/event-stream', + 'text/plain', + ]); + expect(result.skills).toHaveLength(1); + expect(result.skills[0]!.id).toBe('chat_with_agent'); + }); + + it('should preserve explicit values', () => { + const config: AgentCard = { + ...validAgentCard, + description: 'Custom description', + capabilities: { + streaming: false, + pushNotifications: true, + stateTransitionHistory: true, + }, + metadata: { + dexto: { + authentication: { + schemes: ['bearer', 'api-key'], + credentials: 'optional-creds', + }, + }, + }, + defaultInputModes: ['text/plain'], + defaultOutputModes: ['application/json'], + skills: [ + { + id: 'custom-skill', + name: 'Custom Skill', + description: 'A custom skill', + tags: ['custom'], + inputModes: ['application/json'], + outputModes: ['text/plain'], + }, + ], + }; + + const result = AgentCardSchema.parse(config); + + expect(result.description).toBe('Custom description'); + expect(result.capabilities.streaming).toBe(false); + expect(result.capabilities.pushNotifications).toBe(true); + expect(result.capabilities.stateTransitionHistory).toBe(true); + expect(result.metadata?.dexto?.authentication?.schemes).toEqual(['bearer', 'api-key']); + expect(result.metadata?.dexto?.authentication?.credentials).toBe('optional-creds'); + expect(result.defaultInputModes).toEqual(['text/plain']); + expect(result.defaultOutputModes).toEqual(['application/json']); + expect(result.skills).toHaveLength(1); + expect(result.skills[0]!.id).toBe('custom-skill'); + }); + }); + + describe('Required Fields Validation', () => { + it('should require name field', () => { + const config = { ...validAgentCard }; + delete (config as any).name; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['name']); + }); + + it('should require url field', () => { + const config = { ...validAgentCard }; + delete (config as any).url; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['url']); + }); + + it('should require version field', () => { + const config = { ...validAgentCard }; + delete (config as any).version; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['version']); + }); + }); + + describe('URL Validation', () => { + it('should accept valid URLs', () => { + const validUrls = [ + 'https://example.com', + 'http://localhost:8080', + 'https://agent.company.com/v1', + ]; + + for (const url of validUrls) { + const config = { ...validAgentCard, url }; + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(true); + } + }); + + it('should reject invalid URLs', () => { + const invalidUrls = ['not-a-url', 'just-text', '']; + + for (const url of invalidUrls) { + const config = { ...validAgentCard, url }; + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + } + }); + + it('should validate provider.url when provider is specified', () => { + const config: AgentCard = { + ...validAgentCard, + provider: { + organization: 'Test Corp', + url: 'invalid-url', + }, + }; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['provider', 'url']); + }); + + it('should validate documentationUrl when specified', () => { + const config: AgentCard = { + ...validAgentCard, + documentationUrl: 'not-a-url', + }; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['documentationUrl']); + }); + }); + + describe('Skills Validation', () => { + it('should validate skill structure', () => { + const config: AgentCard = { + ...validAgentCard, + skills: [ + { + id: 'test-skill', + name: 'Test Skill', + description: 'A test skill', + tags: ['test', 'demo'], + }, + ], + }; + + const result = AgentCardSchema.parse(config); + expect(result.skills[0]!.inputModes).toEqual(['text/plain']); // default + expect(result.skills[0]!.outputModes).toEqual(['text/plain']); // default + }); + + it('should require skill fields', () => { + const config: AgentCard = { + ...validAgentCard, + skills: [ + { + id: 'test-skill', + name: 'Test Skill', + // Missing description and tags + } as any, + ], + }; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + }); + }); + + describe('Strict Validation', () => { + it('should reject unknown fields', () => { + const config: any = { + ...validAgentCard, + unknownField: 'should-fail', + }; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + + it('should reject unknown fields in nested objects', () => { + const config: any = { + ...validAgentCard, + capabilities: { + streaming: true, + unknownCapability: true, + }, + }; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + + describe('Type Safety', () => { + it('should handle input and output types correctly', () => { + const input: AgentCard = validAgentCard; + const result: ValidatedAgentCard = AgentCardSchema.parse(input); + + // Should have applied defaults + expect(result.description).toBeTruthy(); + expect(result.capabilities).toBeDefined(); + + // Should preserve input values + expect(result.name).toBe(input.name); + expect(result.url).toBe(input.url); + expect(result.version).toBe(input.version); + }); + }); + + describe('Security Schemes Validation', () => { + it('should validate apiKey security scheme', () => { + const config: AgentCard = { + ...validAgentCard, + securitySchemes: { + apiKey: { + type: 'apiKey', + name: 'X-API-Key', + in: 'header', + }, + }, + }; + + const result = AgentCardSchema.parse(config); + expect(result.securitySchemes?.apiKey).toBeDefined(); + if (result.securitySchemes?.apiKey) { + expect(result.securitySchemes.apiKey.type).toBe('apiKey'); + } + }); + + it('should require name and in for apiKey type', () => { + const config: any = { + ...validAgentCard, + securitySchemes: { + apiKey: { + type: 'apiKey', + // Missing name and in + }, + }, + }; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + }); + + it('should validate http security scheme', () => { + const config: AgentCard = { + ...validAgentCard, + securitySchemes: { + bearer: { + type: 'http', + scheme: 'bearer', + bearerFormat: 'JWT', + }, + }, + }; + + const result = AgentCardSchema.parse(config); + expect(result.securitySchemes?.bearer).toBeDefined(); + if (result.securitySchemes?.bearer) { + expect(result.securitySchemes.bearer.type).toBe('http'); + } + }); + + it('should require scheme for http type', () => { + const config: any = { + ...validAgentCard, + securitySchemes: { + http: { + type: 'http', + // Missing scheme + }, + }, + }; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + }); + + it('should validate oauth2 security scheme', () => { + const config: AgentCard = { + ...validAgentCard, + securitySchemes: { + oauth: { + type: 'oauth2', + flows: { + authorizationCode: { + authorizationUrl: 'https://auth.example.com/oauth/authorize', + tokenUrl: 'https://auth.example.com/oauth/token', + scopes: { + read: 'Read access', + write: 'Write access', + }, + }, + }, + }, + }, + }; + + const result = AgentCardSchema.parse(config); + expect(result.securitySchemes?.oauth).toBeDefined(); + if (result.securitySchemes?.oauth) { + expect(result.securitySchemes.oauth.type).toBe('oauth2'); + } + }); + + it('should validate openIdConnect security scheme', () => { + const config: AgentCard = { + ...validAgentCard, + securitySchemes: { + oidc: { + type: 'openIdConnect', + openIdConnectUrl: + 'https://accounts.google.com/.well-known/openid-configuration', + }, + }, + }; + + const result = AgentCardSchema.parse(config); + expect(result.securitySchemes?.oidc).toBeDefined(); + if (result.securitySchemes?.oidc) { + expect(result.securitySchemes.oidc.type).toBe('openIdConnect'); + } + }); + + it('should require openIdConnectUrl for openIdConnect type', () => { + const config: any = { + ...validAgentCard, + securitySchemes: { + oidc: { + type: 'openIdConnect', + // Missing openIdConnectUrl + }, + }, + }; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + }); + + it('should validate mutualTLS security scheme', () => { + const config: AgentCard = { + ...validAgentCard, + securitySchemes: { + mtls: { + type: 'mutualTLS', + }, + }, + }; + + const result = AgentCardSchema.parse(config); + expect(result.securitySchemes?.mtls).toBeDefined(); + if (result.securitySchemes?.mtls) { + expect(result.securitySchemes.mtls.type).toBe('mutualTLS'); + } + }); + }); + + describe('Metadata and Extensions', () => { + it('should support dexto metadata extensions', () => { + const config: AgentCard = { + ...validAgentCard, + metadata: { + dexto: { + delegation: { + protocol: 'a2a-jsonrpc', + endpoint: '/delegate', + supportsSession: true, + supportsStreaming: true, + }, + owner: { + userId: 'user123', + username: 'testuser', + email: 'test@example.com', + }, + }, + }, + }; + + const result = AgentCardSchema.parse(config); + expect(result.metadata?.dexto?.delegation?.protocol).toBe('a2a-jsonrpc'); + expect(result.metadata?.dexto?.owner?.userId).toBe('user123'); + }); + + it('should support custom metadata namespaces', () => { + const config: AgentCard = { + ...validAgentCard, + metadata: { + dexto: {}, + customExtension: { + foo: 'bar', + nested: { key: 'value' }, + }, + }, + }; + + const result = AgentCardSchema.parse(config); + expect(result.metadata?.customExtension).toBeDefined(); + }); + + it('should validate signatures field', () => { + const config: AgentCard = { + ...validAgentCard, + signatures: [ + { + protected: 'eyJhbGciOiJSUzI1NiJ9', + signature: + 'cC4hiUPoj9Eetdgtv3hF80EGrhuB__dzERat0XF9g2VtQgr9PJbu3XOiZj5RZmh7', + }, + ], + }; + + const result = AgentCardSchema.parse(config); + expect(result.signatures).toHaveLength(1); + expect(result.signatures![0]!.protected).toBe('eyJhbGciOiJSUzI1NiJ9'); + }); + }); + + describe('Required Description Field', () => { + it('should require description field', () => { + const config = { + name: 'TestAgent', + url: 'https://agent.example.com', + version: '1.0.0', + // Missing description + }; + + const result = AgentCardSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['description']); + }); + }); +}); + +describe('AgentConfigSchema', () => { + const validAgentConfig: AgentConfig = { + systemPrompt: 'You are a helpful assistant', + llm: { + provider: 'openai', + model: 'gpt-5', + apiKey: 'test-key', + }, + }; + + describe('Basic Structure Validation', () => { + it('should accept valid minimal config', () => { + const result = AgentConfigSchema.parse(validAgentConfig); + + expect(result.systemPrompt.contributors).toHaveLength(1); + expect(result.llm.provider).toBe('openai'); + expect(result.llm.model).toBe('gpt-5'); + expect(result.llm.apiKey).toBe('test-key'); + }); + + it('should apply default values', () => { + const result = AgentConfigSchema.parse(validAgentConfig); + + // Should apply defaults from composed schemas + expect(result.mcpServers).toEqual({}); + expect(result.internalTools).toEqual([]); + expect(result.storage).toBeDefined(); + expect(result.storage.cache.type).toBe('in-memory'); + expect(result.storage.database.type).toBe('in-memory'); + expect(result.sessions).toBeDefined(); + expect(result.toolConfirmation).toBeDefined(); + }); + + it('should preserve explicit values from all composed schemas', () => { + const config: AgentConfig = { + agentCard: { + name: 'TestAgent', + description: 'Test agent for validation', + url: 'https://agent.example.com', + version: '1.0.0', + }, + systemPrompt: { + contributors: [ + { + id: 'custom', + type: 'static', + content: 'Custom prompt', + priority: 0, + }, + ], + }, + mcpServers: { + testServer: { + type: 'stdio', + command: 'node', + args: ['server.js'], + }, + }, + internalTools: ['search_history'], + llm: { + provider: 'anthropic', + model: 'claude-haiku-4-5-20251001', + apiKey: 'test-anthropic-key', + maxIterations: 25, + }, + storage: { + cache: { type: 'redis', url: 'redis://localhost:6379' }, + database: { type: 'postgres', url: 'postgresql://localhost:5432/test' }, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }, + sessions: { + maxSessions: 5, + sessionTTL: 1800, + }, + toolConfirmation: { + mode: 'auto-approve', + timeout: 15000, + }, + }; + + const result = AgentConfigSchema.parse(config); + + expect(result.agentCard?.name).toBe('TestAgent'); + expect(result.systemPrompt.contributors[0]!.id).toBe('custom'); + expect(result.mcpServers.testServer).toBeDefined(); + expect(result.internalTools).toEqual(['search_history']); + expect(result.llm.provider).toBe('anthropic'); + expect(result.storage.cache.type).toBe('redis'); + expect(result.sessions.maxSessions).toBe(5); + expect(result.toolConfirmation.mode).toBe('auto-approve'); + }); + }); + + describe('Required Fields Validation', () => { + it('should require systemPrompt field', () => { + const config = { ...validAgentConfig }; + delete (config as any).systemPrompt; + + const result = AgentConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['systemPrompt']); + }); + + it('should require llm field', () => { + const config = { ...validAgentConfig }; + delete (config as any).llm; + + const result = AgentConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['llm']); + }); + }); + + describe('Validation Propagation', () => { + it('should propagate validation errors from nested schemas', () => { + // Test that validation failures in composed schemas bubble up correctly + // Detailed validation testing is done in individual schema test files + const configWithInvalidLLM: AgentConfig = { + ...validAgentConfig, + llm: { + provider: 'invalid-provider' as any, + model: 'test-model', + apiKey: 'test-key', + }, + }; + + const result = AgentConfigSchema.safeParse(configWithInvalidLLM); + expect(result.success).toBe(false); + // Verify error path points to the nested schema field + expect(result.error?.issues[0]?.path[0]).toBe('llm'); + }); + }); + + describe('Schema Composition Integration', () => { + it('should properly transform systemPrompt from string to object', () => { + const config: AgentConfig = { + ...validAgentConfig, + systemPrompt: 'Simple string prompt', + }; + + const result = AgentConfigSchema.parse(config); + + expect(result.systemPrompt.contributors).toHaveLength(1); + expect(result.systemPrompt.contributors[0]!.type).toBe('static'); + expect((result.systemPrompt.contributors[0] as any).content).toBe( + 'Simple string prompt' + ); + }); + + it('should apply defaults from all composed schemas', () => { + const result = AgentConfigSchema.parse(validAgentConfig); + + // Defaults from different schemas should all be applied + expect(result.llm.maxIterations).toBeUndefined(); // LLM schema default (unlimited) + expect(result.storage).toBeDefined(); + expect(result.storage.cache.type).toBe('in-memory'); // Storage schema default + expect(result.sessions.maxSessions).toBe(100); // Session schema default + expect(result.toolConfirmation.mode).toBe('auto-approve'); // Tool schema default + }); + }); + + describe('Strict Validation', () => { + it('should reject unknown fields', () => { + const config: any = { + ...validAgentConfig, + unknownField: 'should-fail', + }; + + const result = AgentConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + + describe('Type Safety', () => { + it('should handle input and output types correctly', () => { + const input: AgentConfig = validAgentConfig; + + const result = AgentConfigSchema.parse(input); + + // Should have applied defaults from all composed schemas + expect(result.mcpServers).toBeDefined(); + expect(result.internalTools).toBeDefined(); + expect(result.storage).toBeDefined(); + expect(result.sessions).toBeDefined(); + expect(result.toolConfirmation).toBeDefined(); + + // Should preserve input values + expect(result.llm.provider).toBe(input.llm.provider); + expect(result.llm.model).toBe(input.llm.model); + expect(result.llm.apiKey).toBe(input.llm.apiKey); + }); + + it('should maintain proper types for nested objects', () => { + const config = AgentConfigSchema.parse(validAgentConfig); + + // TypeScript should infer correct nested types + expect(typeof config.llm.provider).toBe('string'); + expect(typeof config.llm.model).toBe('string'); + expect(typeof config.storage.cache.type).toBe('string'); + expect(Array.isArray(config.internalTools)).toBe(true); + expect(typeof config.sessions.maxSessions).toBe('number'); + }); + }); + + describe('Real-world Scenarios', () => { + it('should handle complete production config', () => { + const prodConfig: AgentConfig = { + agentCard: { + name: 'Production Agent', + description: 'Production AI agent for customer support', + url: 'https://api.company.com/agent', + version: '2.1.0', + provider: { + organization: 'ACME Corp', + url: 'https://acme.com', + }, + documentationUrl: 'https://docs.acme.com/agent', + }, + systemPrompt: { + contributors: [ + { + id: 'main', + type: 'static', + content: 'You are a customer support agent.', + priority: 0, + }, + { + id: 'datetime', + type: 'dynamic', + source: 'date', + priority: 10, + }, + ], + }, + mcpServers: { + database: { + type: 'stdio', + command: 'python', + args: ['-m', 'db_server'], + env: { DB_URL: 'postgresql://prod:5432/db' }, + }, + search: { + type: 'http', + url: 'https://search.company.com/mcp', + headers: { Authorization: 'Bearer prod-token' }, + }, + }, + internalTools: ['search_history'], + llm: { + provider: 'openai', + model: 'gpt-5', + apiKey: 'sk-prod-key-123', + maxIterations: 30, + temperature: 0.3, + }, + storage: { + cache: { + type: 'redis', + url: 'redis://cache.company.com:6379', + }, + database: { + type: 'postgres', + url: 'postgresql://db.company.com:5432/agent_db', + }, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }, + sessions: { + maxSessions: 100, + sessionTTL: 7200, + }, + toolConfirmation: { + mode: 'manual', + timeout: 45000, + allowedToolsStorage: 'storage', + }, + }; + + const result = AgentConfigSchema.parse(prodConfig); + + expect(result.agentCard?.name).toBe('Production Agent'); + expect(result.systemPrompt.contributors).toHaveLength(2); + expect(Object.keys(result.mcpServers)).toHaveLength(2); + expect(result.internalTools).toEqual(['search_history']); + expect(result.llm.temperature).toBe(0.3); + expect(result.storage.cache.type).toBe('redis'); + expect(result.sessions.maxSessions).toBe(100); + expect(result.toolConfirmation.timeout).toBe(45000); + }); + + it('should handle minimal config with all defaults', () => { + const minimalConfig: AgentConfig = { + systemPrompt: 'You are helpful', + llm: { + provider: 'openai', + model: 'gpt-5-mini', + apiKey: 'sk-test', + }, + }; + + const result = AgentConfigSchema.parse(minimalConfig); + + // Should have all defaults applied + expect(result.mcpServers).toEqual({}); + expect(result.internalTools).toEqual([]); + expect(result.storage).toBeDefined(); + expect(result.storage.cache.type).toBe('in-memory'); + expect(result.storage.database.type).toBe('in-memory'); + expect(result.storage.blob.type).toBe('in-memory'); + expect(result.sessions).toBeDefined(); + expect(result.toolConfirmation.mode).toBe('auto-approve'); + expect(result.llm.maxIterations).toBeUndefined(); + }); + }); +}); diff --git a/dexto/packages/core/src/agent/schemas.ts b/dexto/packages/core/src/agent/schemas.ts new file mode 100644 index 00000000..47cf8c3a --- /dev/null +++ b/dexto/packages/core/src/agent/schemas.ts @@ -0,0 +1,466 @@ +/** + * Schema Defaults Conventions: + * – Field-level defaults live in the leaf schemas. + * – AgentConfig decides if a section is optional by adding `.default({})`. + * It never duplicates per-field literal defaults. + */ + +import { createLLMConfigSchema, type LLMValidationOptions } from '@core/llm/schemas.js'; +import { LoggerConfigSchema } from '@core/logger/index.js'; +import { ServerConfigsSchema as McpServersConfigSchema } from '@core/mcp/schemas.js'; +import { MemoriesConfigSchema } from '@core/memory/schemas.js'; +import { SessionConfigSchema } from '@core/session/schemas.js'; +import { StorageSchema } from '@core/storage/schemas.js'; +import { SystemPromptConfigSchema } from '@core/systemPrompt/schemas.js'; +import { + CompactionConfigSchema, + DEFAULT_COMPACTION_CONFIG, +} from '@core/context/compaction/schemas.js'; +import { + InternalToolsSchema, + CustomToolsSchema, + ToolConfirmationConfigSchema, + ElicitationConfigSchema, + ToolsConfigSchema, +} from '@core/tools/schemas.js'; +import { z } from 'zod'; +import { InternalResourcesSchema } from '@core/resources/schemas.js'; +import { PromptsSchema } from '@core/prompts/schemas.js'; +import { PluginsConfigSchema } from '@core/plugins/schemas.js'; +import { OtelConfigurationSchema } from '@core/telemetry/schemas.js'; + +// (agent card overrides are now represented as Partial and processed via AgentCardSchema) + +/** + * Security Scheme Schemas (A2A Protocol, based on OpenAPI 3.0 Security Scheme Object) + * Defines authentication mechanisms for the agent as a discriminated union + */ + +const ApiKeySecurityScheme = z + .object({ + type: z.literal('apiKey').describe('Security scheme type'), + name: z.string().describe('Name of the header/query/cookie parameter'), + in: z.enum(['query', 'header', 'cookie']).describe('Location of API key'), + description: z.string().optional().describe('Description of the security scheme'), + }) + .strict(); + +const HttpSecurityScheme = z + .object({ + type: z.literal('http').describe('Security scheme type'), + scheme: z.string().describe('HTTP authorization scheme (e.g., basic, bearer)'), + bearerFormat: z.string().optional().describe('Hint for bearer token format'), + description: z.string().optional().describe('Description of the security scheme'), + }) + .strict(); + +const OAuth2FlowSchema = z + .object({ + authorizationUrl: z.string().url().optional().describe('Authorization URL for the flow'), + tokenUrl: z.string().url().optional().describe('Token URL for the flow'), + refreshUrl: z.string().url().optional().describe('Refresh URL for the flow'), + scopes: z.record(z.string()).describe('Available scopes for the OAuth2 flow'), + }) + .strict(); + +const OAuth2SecurityScheme = z + .object({ + type: z.literal('oauth2').describe('Security scheme type'), + flows: z + .object({ + implicit: OAuth2FlowSchema.optional(), + password: OAuth2FlowSchema.optional(), + clientCredentials: OAuth2FlowSchema.optional(), + authorizationCode: OAuth2FlowSchema.optional(), + }) + .strict() + .describe('OAuth2 flow configurations'), + description: z.string().optional().describe('Description of the security scheme'), + }) + .strict(); + +const OpenIdConnectSecurityScheme = z + .object({ + type: z.literal('openIdConnect').describe('Security scheme type'), + openIdConnectUrl: z.string().url().describe('OpenID Connect discovery URL'), + description: z.string().optional().describe('Description of the security scheme'), + }) + .strict(); + +const MutualTLSSecurityScheme = z + .object({ + type: z.literal('mutualTLS').describe('Security scheme type'), + description: z.string().optional().describe('Description of the security scheme'), + }) + .strict(); + +export const SecuritySchemeSchema = z.discriminatedUnion('type', [ + ApiKeySecurityScheme, + HttpSecurityScheme, + OAuth2SecurityScheme, + OpenIdConnectSecurityScheme, + MutualTLSSecurityScheme, +]); + +/** + * Agent Card Signature Schema (A2A Protocol v0.3.0) + * JSON Web Signature for verifying AgentCard integrity + */ +const AgentCardSignatureSchema = z + .object({ + protected: z.string().describe('Base64url-encoded JWS Protected Header'), + signature: z.string().describe('Base64url-encoded JWS Signature'), + }) + .strict(); + +/** + * Dexto Extension Metadata Schema + * Namespace for Dexto-specific extension fields + */ +const DextoMetadataSchema = z + .object({ + authentication: z + .object({ + schemes: z + .array(z.string()) + .default([]) + .describe('Legacy authentication schemes (deprecated: use securitySchemes)'), + credentials: z.string().optional().describe('Credentials information'), + }) + .strict() + .optional() + .describe('Legacy authentication configuration'), + + delegation: z + .object({ + protocol: z + .enum(['dexto-v1', 'http-simple', 'a2a-jsonrpc', 'mcp-http']) + .describe('Delegation protocol version'), + endpoint: z.string().describe('Delegation endpoint (relative path or full URL)'), + supportsSession: z.boolean().describe('Whether agent supports stateful sessions'), + supportsStreaming: z + .boolean() + .optional() + .describe('Whether agent supports streaming responses'), + }) + .strict() + .optional() + .describe('Delegation protocol information for agent-to-agent communication'), + + owner: z + .object({ + userId: z.string().describe('Unique user identifier from auth system'), + username: z.string().describe('Display name'), + email: z + .string() + .email() + .max(254) + .optional() + .describe( + 'Optional user email (WARNING: publicly readable via .well-known/agent.json if provided)' + ), + }) + .strict() + .optional() + .describe('Agent owner information (for multi-tenant deployments)'), + }) + .strict(); + +/** + * Agent Card Schema (A2A Protocol v0.3.0 Compliant) + * Follows the A2A specification with extensions in the metadata field + */ +export const AgentCardSchema = z + .object({ + // ──────────────────────────────────────────────────────── + // A2A Protocol Required Fields + // ──────────────────────────────────────────────────────── + protocolVersion: z + .string() + .default('0.3.0') + .describe('A2A protocol version (e.g., "0.3.0")'), + + name: z.string().describe('Human-readable agent name'), + + description: z.string().describe('Detailed description of agent purpose and capabilities'), + + url: z.string().url().describe('Primary endpoint URL for the agent'), + + version: z.string().describe('Agent version (semantic versioning recommended)'), + + preferredTransport: z + .enum(['JSONRPC', 'GRPC', 'HTTP+JSON']) + .default('JSONRPC') + .describe('Primary transport protocol for communication'), + + defaultInputModes: z + .array(z.string()) + .default(['application/json', 'text/plain']) + .describe('Supported input MIME types'), + + defaultOutputModes: z + .array(z.string()) + .default(['application/json', 'text/event-stream', 'text/plain']) + .describe('Supported output MIME types'), + + skills: z + .array( + z + .object({ + id: z.string().describe('Unique skill identifier'), + name: z.string().describe('Human-readable skill name'), + description: z.string().describe('Detailed skill description'), + tags: z.array(z.string()).describe('Searchable tags for discovery'), + examples: z + .array(z.string()) + .optional() + .describe('Example use cases or queries'), + inputModes: z + .array(z.string()) + .optional() + .default(['text/plain']) + .describe('Skill-specific input MIME types'), + outputModes: z + .array(z.string()) + .optional() + .default(['text/plain']) + .describe('Skill-specific output MIME types'), + }) + .strict() + ) + .default([ + { + id: 'chat_with_agent', + name: 'chat_with_agent', + description: 'Allows you to chat with an AI agent. Send a message to interact.', + tags: ['chat', 'AI', 'assistant', 'mcp', 'natural language'], + examples: [ + `Send a JSON-RPC request to /mcp with method: "chat_with_agent" and params: {"message":"Your query..."}`, + 'Alternatively, use a compatible MCP client library.', + ], + }, + ]) + .describe('Agent capabilities/skills'), + + // ──────────────────────────────────────────────────────── + // A2A Protocol Optional Fields + // ──────────────────────────────────────────────────────── + provider: z + .object({ + organization: z.string().describe('Provider organization name'), + url: z.string().url().describe('Provider organization URL'), + }) + .strict() + .optional() + .describe('Agent provider information'), + + iconUrl: z.string().url().optional().describe('URL to agent icon/logo (for UI display)'), + + documentationUrl: z.string().url().optional().describe('URL to agent documentation'), + + additionalInterfaces: z + .array( + z + .object({ + url: z.string().url().describe('Endpoint URL'), + transport: z + .enum(['JSONRPC', 'GRPC', 'HTTP+JSON']) + .describe('Transport protocol'), + }) + .strict() + ) + .optional() + .describe('Additional interfaces/transports supported by the agent'), + + capabilities: z + .object({ + streaming: z + .boolean() + .optional() + .default(true) + .describe('Supports streaming responses'), + pushNotifications: z.boolean().optional().describe('Supports push notifications'), + stateTransitionHistory: z + .boolean() + .optional() + .default(false) + .describe('Provides state transition history'), + }) + .strict() + .default({}) + .describe('Agent capabilities and features'), + + securitySchemes: z + .record(SecuritySchemeSchema) + .optional() + .describe('Map of security scheme definitions (A2A format)'), + + security: z + .array(z.record(z.array(z.string()))) + .optional() + .describe( + 'Security requirements (array of security scheme references with required scopes)' + ), + + supportsAuthenticatedExtendedCard: z + .boolean() + .optional() + .describe('Whether extended card is available with authentication'), + + signatures: z + .array(AgentCardSignatureSchema) + .optional() + .describe('JSON Web Signatures for verifying AgentCard integrity'), + + metadata: z + .object({ + dexto: DextoMetadataSchema.optional().describe('Dexto-specific extension metadata'), + }) + .passthrough() + .optional() + .describe('Extension-specific metadata (namespaced by extension name)'), + }) + .strict(); +// Input type for user-facing API (pre-parsing) + +export type AgentCard = z.input; +// Validated type for internal use (post-parsing) +export type ValidatedAgentCard = z.output; + +/** + * Creates an agent config schema with configurable validation strictness. + * + * @param options.strict - When true (default), enforces API key and baseURL requirements. + * When false, allows missing credentials for interactive configuration. + */ +export function createAgentConfigSchema(options: LLMValidationOptions = {}) { + const llmSchema = createLLMConfigSchema(options); + + return z + .object({ + // ======================================== + // REQUIRED FIELDS (user must provide or schema validation fails) + // ======================================== + systemPrompt: SystemPromptConfigSchema.describe( + 'System prompt: string shorthand or structured config' + ), + + llm: llmSchema.describe('Core LLM configuration for the agent'), + + // ======================================== + // OPTIONAL FEATURES (undefined if not provided) + // ======================================== + agentCard: AgentCardSchema.describe('Configuration for the agent card').optional(), + + greeting: z + .string() + .max(500) + .describe('Default greeting text to show when a chat starts (for UI consumption)') + .optional(), + + telemetry: OtelConfigurationSchema.describe( + 'OpenTelemetry configuration for distributed tracing and observability' + ).optional(), + + memories: MemoriesConfigSchema.describe( + 'Memory configuration for system prompt inclusion (optional feature)' + ).optional(), + + image: z + .string() + .describe( + 'Image package that provides required providers (e.g., "@dexto/image-local"). Optional - platform can load images via CLI flag, environment variable, or static imports.' + ) + .optional(), + + // ======================================== + // FIELDS WITH DEFAULTS (always present after parsing) + // ======================================== + agentId: z + .string() + .describe( + 'Unique identifier for this agent instance - CLI enrichment derives from agentCard.name or filename' + ) + .default('coding-agent'), + + mcpServers: McpServersConfigSchema.describe( + 'Configurations for MCP (Model Context Protocol) servers used by the agent' + ).default({}), + + internalTools: InternalToolsSchema.describe( + 'Internal tools configuration (read-file, write-file, bash-exec, etc.)' + ).default([]), + + customTools: CustomToolsSchema.describe( + 'Custom tool provider configurations. Providers must be registered via customToolRegistry before loading agent config.' + ).default([]), + + tools: ToolsConfigSchema.describe( + 'Configuration for individual tools (limits, etc.)' + ).default({}), + + logger: LoggerConfigSchema.describe( + 'Logger configuration with multi-transport support (file, console, remote) - CLI enrichment adds per-agent file transport' + ).default({ + level: 'error', + transports: [{ type: 'console', colorize: true }], + }), + + storage: StorageSchema.describe( + 'Storage configuration for cache, database, and blob storage - defaults to in-memory, CLI enrichment provides filesystem paths' + ).default({ + cache: { type: 'in-memory' }, + database: { type: 'in-memory' }, + blob: { type: 'in-memory' }, + }), + + sessions: SessionConfigSchema.describe('Session management configuration').default({}), + + toolConfirmation: ToolConfirmationConfigSchema.describe( + 'Tool confirmation and approval configuration' + ).default({}), + + elicitation: ElicitationConfigSchema.default({}).describe( + 'Elicitation configuration for user input requests (ask_user tool and MCP server elicitations). Independent from toolConfirmation mode.' + ), + + internalResources: InternalResourcesSchema.describe( + 'Configuration for internal resources (filesystem, etc.)' + ).default([]), + + prompts: PromptsSchema.describe( + 'Agent prompts configuration - sample prompts which can be defined inline or referenced from file' + ).default([]), + + plugins: PluginsConfigSchema.describe( + 'Plugin system configuration for built-in and custom plugins' + ).default({}), + + compaction: CompactionConfigSchema.describe( + 'Context compaction configuration - custom providers can be registered via compactionRegistry' + ).default(DEFAULT_COMPACTION_CONFIG), + }) + .strict() + .describe('Main configuration for an agent, including its LLM and server connections') + .brand<'ValidatedAgentConfig'>(); +} + +/** + * Default agent config schema with strict validation (backwards compatible). + * Use createAgentConfigSchema({ strict: false }) for relaxed validation. + */ +export const AgentConfigSchema = createAgentConfigSchema({ strict: true }); + +/** + * Relaxed agent config schema that allows missing API keys and baseURLs. + * Use this for interactive modes (CLI, WebUI) where users can configure later. + */ +export const AgentConfigSchemaRelaxed = createAgentConfigSchema({ strict: false }); + +// Input type for user-facing API (pre-parsing) - makes fields with defaults optional +export type AgentConfig = z.input; +// Validated type for internal use (post-parsing) - all defaults applied +export type ValidatedAgentConfig = z.output; + +// Re-export validation options type for consumers +export type { LLMValidationOptions }; diff --git a/dexto/packages/core/src/agent/state-manager.test.ts b/dexto/packages/core/src/agent/state-manager.test.ts new file mode 100644 index 00000000..4eed4636 --- /dev/null +++ b/dexto/packages/core/src/agent/state-manager.test.ts @@ -0,0 +1,171 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { AgentStateManager } from './state-manager.js'; +import { AgentEventBus } from '../events/index.js'; +import { AgentConfigSchema } from '@core/agent/schemas.js'; +import { LLMConfigSchema } from '@core/llm/schemas.js'; +import { McpServerConfigSchema } from '@core/mcp/schemas.js'; +import type { AgentConfig, ValidatedAgentConfig } from '@core/agent/schemas.js'; + +describe('AgentStateManager Events', () => { + let stateManager: AgentStateManager; + let eventBus: AgentEventBus; + let mockConfig: AgentConfig; + let validatedConfig: ValidatedAgentConfig; + let mockLogger: any; + + beforeEach(() => { + eventBus = new AgentEventBus(); + mockLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), + } as any; + mockConfig = { + systemPrompt: 'You are a helpful assistant', + mcpServers: { + test: { + type: 'stdio', + command: 'test', + args: [], + env: {}, + timeout: 30000, + connectionMode: 'lenient', + }, + }, + llm: { + provider: 'openai', + model: 'gpt-5', + apiKey: 'test-key', + maxIterations: 50, + }, + internalTools: [], + sessions: { + maxSessions: 100, + sessionTTL: 3600000, + }, + toolConfirmation: { + mode: 'manual', + timeout: 30000, + allowedToolsStorage: 'storage', + }, + }; + // Parse through schema to validate and apply defaults, converting input to ValidatedAgentConfig + validatedConfig = AgentConfigSchema.parse(mockConfig); + stateManager = new AgentStateManager(validatedConfig, eventBus, mockLogger); + }); + + it('emits dexto:stateChanged when LLM config is updated', () => { + const eventSpy = vi.fn(); + eventBus.on('state:changed', eventSpy); + + const updatedConfig = LLMConfigSchema.parse({ + ...mockConfig.llm, + model: 'gpt-5-mini', + }); + stateManager.updateLLM(updatedConfig); + + expect(eventSpy).toHaveBeenCalledWith({ + field: 'llm', + oldValue: expect.objectContaining({ model: 'gpt-5' }), + newValue: expect.objectContaining({ model: 'gpt-5-mini' }), + sessionId: undefined, + }); + }); + + it('emits dexto:mcpServerAdded when adding a new MCP server', () => { + const eventSpy = vi.fn(); + eventBus.on('mcp:server-added', eventSpy); + + const newServerConfig = McpServerConfigSchema.parse({ + type: 'stdio' as const, + command: 'new-server', + args: [], + env: {}, + timeout: 30000, + connectionMode: 'lenient' as const, + }); + + stateManager.setMcpServer('new-server', newServerConfig); + + expect(eventSpy).toHaveBeenCalledWith({ + serverName: 'new-server', + config: newServerConfig, + }); + }); + + it('emits dexto:mcpServerRemoved when removing an MCP server', () => { + const eventSpy = vi.fn(); + eventBus.on('mcp:server-removed', eventSpy); + + stateManager.removeMcpServer('test'); + + expect(eventSpy).toHaveBeenCalledWith({ + serverName: 'test', + }); + }); + + it('emits dexto:sessionOverrideSet when setting session overrides', () => { + const eventSpy = vi.fn(); + eventBus.on('session:override-set', eventSpy); + + const sessionConfig = LLMConfigSchema.parse({ + ...mockConfig.llm, + model: 'gpt-5', + }); + stateManager.updateLLM(sessionConfig, 'session-123'); + + expect(eventSpy).toHaveBeenCalledWith({ + sessionId: 'session-123', + override: expect.objectContaining({ + llm: expect.objectContaining({ model: 'gpt-5' }), + }), + }); + }); + + it('emits dexto:sessionOverrideCleared when clearing session overrides', () => { + const eventSpy = vi.fn(); + eventBus.on('session:override-cleared', eventSpy); + + // First set an override + const sessionConfig = LLMConfigSchema.parse({ + ...mockConfig.llm, + model: 'gpt-5', + }); + stateManager.updateLLM(sessionConfig, 'session-123'); + + // Then clear it + stateManager.clearSessionOverride('session-123'); + + expect(eventSpy).toHaveBeenCalledWith({ + sessionId: 'session-123', + }); + }); + + it('emits dexto:stateReset when resetting to baseline', () => { + const eventSpy = vi.fn(); + eventBus.on('state:reset', eventSpy); + + stateManager.resetToBaseline(); + + expect(eventSpy).toHaveBeenCalledWith({ + toConfig: validatedConfig, + }); + }); + + it('emits dexto:stateExported when exporting state as config', () => { + const eventSpy = vi.fn(); + eventBus.on('state:exported', eventSpy); + + const exported = stateManager.exportAsConfig(); + + expect(eventSpy).toHaveBeenCalledWith({ + config: exported, + }); + }); +}); diff --git a/dexto/packages/core/src/agent/state-manager.ts b/dexto/packages/core/src/agent/state-manager.ts new file mode 100644 index 00000000..17391975 --- /dev/null +++ b/dexto/packages/core/src/agent/state-manager.ts @@ -0,0 +1,263 @@ +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import type { ValidatedAgentConfig } from '@core/agent/schemas.js'; +import type { ValidatedLLMConfig } from '@core/llm/schemas.js'; +import type { ValidatedMcpServerConfig } from '@core/mcp/schemas.js'; +import type { AgentEventBus } from '../events/index.js'; + +/** + * Session-specific overrides that can differ from the global configuration + */ +export interface SessionOverride { + /** Override LLM config for this session - must be a complete validated config */ + llm?: ValidatedLLMConfig; +} + +/** + * Manages the runtime configuration of the agent. + * + * This class handles dynamic configuration changes that occur during agent execution. + * + * Key responsibilities: + * 1. Track runtime changes separate from static config baseline + * 2. Support session-specific overrides for LLM settings + * 3. Dynamic MCP server management (add/remove servers at runtime) + * 4. Export modified state back to config format + * 5. Provide change tracking and validation capabilities + * 6. Maintain effective configuration for each session + */ +export class AgentStateManager { + private runtimeConfig: ValidatedAgentConfig; + private readonly baselineConfig: ValidatedAgentConfig; + private sessionOverrides: Map = new Map(); + private logger: IDextoLogger; + + /** + * Initialize AgentStateManager from a validated static configuration. + * + * @param staticConfig The validated configuration from DextoAgent + * @param agentEventBus The agent event bus for emitting state change events + * @param logger Logger instance for this agent + */ + constructor( + staticConfig: ValidatedAgentConfig, + private agentEventBus: AgentEventBus, + logger: IDextoLogger + ) { + this.baselineConfig = structuredClone(staticConfig); + this.runtimeConfig = structuredClone(staticConfig); + this.logger = logger.createChild(DextoLogComponent.AGENT); + + this.logger.debug('AgentStateManager initialized', { + staticConfigKeys: Object.keys(this.baselineConfig), + mcpServerCount: Object.keys(this.runtimeConfig.mcpServers).length, + }); + } + + // ============= GETTERS ============= + + /** + * Get runtime configuration for a session (includes session overrides if sessionId provided) + */ + public getRuntimeConfig(sessionId?: string): Readonly { + if (!sessionId) { + return structuredClone(this.runtimeConfig); + } + + const override = this.sessionOverrides.get(sessionId); + if (!override) { + return structuredClone(this.runtimeConfig); + } + + return { + ...this.runtimeConfig, + llm: { ...this.runtimeConfig.llm, ...override.llm }, + }; + } + + // ============= LLM CONFIGURATION ============= + + /** + * Update the LLM configuration (globally or for a specific session) + * + * This method is a pure state updater - it assumes the input has already been validated + * by the caller (typically DextoAgent.switchLLM). The ValidatedLLMConfig branded type + * ensures validation has occurred. + */ + public updateLLM(validatedConfig: ValidatedLLMConfig, sessionId?: string): void { + const oldValue = sessionId ? this.getRuntimeConfig(sessionId).llm : this.runtimeConfig.llm; + + if (sessionId) { + this.setSessionOverride(sessionId, { + llm: validatedConfig, + }); + } else { + this.runtimeConfig.llm = validatedConfig; + } + + this.agentEventBus.emit('state:changed', { + field: 'llm', + oldValue, + newValue: validatedConfig, + ...(sessionId && { sessionId }), + }); + + this.logger.info('LLM config updated', { + sessionId, + provider: validatedConfig.provider, + model: validatedConfig.model, + isSessionSpecific: !!sessionId, + }); + } + + // ============= MCP SERVER MANAGEMENT ============= + + /** + * Set an MCP server configuration at runtime (add or update). + * + * This method is a pure state updater - it assumes the input has already been validated + * by the caller (typically DextoAgent). The ValidatedMcpServerConfig branded type + * ensures validation has occurred. + */ + public setMcpServer(serverName: string, validatedConfig: ValidatedMcpServerConfig): void { + this.logger.debug(`Setting MCP server: ${serverName}`); + + // Update state + const isUpdate = serverName in this.runtimeConfig.mcpServers; + this.runtimeConfig.mcpServers[serverName] = validatedConfig; + + // Emit events + const eventName = isUpdate ? 'mcp:server-updated' : 'mcp:server-added'; + this.agentEventBus.emit(eventName, { serverName, config: validatedConfig }); + + this.agentEventBus.emit('state:changed', { + field: 'mcpServers', + oldValue: isUpdate ? 'updated' : 'added', + newValue: validatedConfig, + // sessionId omitted - MCP servers are global + }); + + this.logger.info( + `MCP server '${serverName}' ${isUpdate ? 'updated' : 'added'} successfully` + ); + } + + /** + * Remove an MCP server configuration at runtime. + */ + public removeMcpServer(serverName: string): void { + this.logger.debug(`Removing MCP server: ${serverName}`); + + if (serverName in this.runtimeConfig.mcpServers) { + delete this.runtimeConfig.mcpServers[serverName]; + + this.agentEventBus.emit('mcp:server-removed', { serverName }); + this.agentEventBus.emit('state:changed', { + field: 'mcpServers', + oldValue: 'removed', + newValue: undefined, + // sessionId omitted - MCP servers are global + }); + + this.logger.info(`MCP server '${serverName}' removed successfully`); + } else { + this.logger.warn(`MCP server '${serverName}' not found for removal`); + } + } + + // ============= SESSION MANAGEMENT ============= + + /** + * Set a session-specific override + */ + private setSessionOverride(sessionId: string, override: SessionOverride): void { + this.sessionOverrides.set(sessionId, override); + this.agentEventBus.emit('session:override-set', { + sessionId, + override: structuredClone(override), + }); + } + + /** + * Get a session override (internal helper) + */ + private getSessionOverride(sessionId: string): SessionOverride | undefined { + return this.sessionOverrides.get(sessionId); + } + + /** + * Clear session-specific overrides + */ + public clearSessionOverride(sessionId: string): void { + const hadOverride = this.sessionOverrides.has(sessionId); + this.sessionOverrides.delete(sessionId); + + if (hadOverride) { + this.agentEventBus.emit('session:override-cleared', { sessionId }); + this.logger.info('Session override cleared', { sessionId }); + } + } + + /** + * Clear all session overrides (private helper for resetToBaseline) + */ + private clearAllSessionOverrides(): void { + const sessionIds = Array.from(this.sessionOverrides.keys()); + this.sessionOverrides.clear(); + + sessionIds.forEach((sessionId) => { + this.agentEventBus.emit('session:override-cleared', { sessionId }); + }); + + if (sessionIds.length > 0) { + this.logger.info('All session overrides cleared', { clearedSessions: sessionIds }); + } + } + + // ============= CONFIG EXPORT ============= + + /** + * Export current runtime state as config. + * This allows users to save their runtime modifications as a new agent config. + */ + public exportAsConfig(): ValidatedAgentConfig { + const exportedConfig: ValidatedAgentConfig = { + ...this.baselineConfig, + llm: structuredClone(this.runtimeConfig.llm), + systemPrompt: this.runtimeConfig.systemPrompt, + mcpServers: structuredClone(this.runtimeConfig.mcpServers), + }; + + this.agentEventBus.emit('state:exported', { + config: exportedConfig, + }); + + this.logger.info('Runtime state exported as config', { + exportedConfig, + }); + + return exportedConfig; + } + + /** + * Reset runtime state back to baseline configuration + */ + public resetToBaseline(): void { + this.runtimeConfig = structuredClone(this.baselineConfig); + + this.clearAllSessionOverrides(); + this.agentEventBus.emit('state:reset', { toConfig: this.baselineConfig }); + + this.logger.info('Runtime state reset to baseline config'); + } + + // ============= CONVENIENCE GETTERS FOR USED FUNCTIONALITY ============= + + /** + * Get the current effective LLM configuration for a session. + * **Use this for session-specific LLM config** (includes session overrides). + */ + public getLLMConfig(sessionId?: string): Readonly { + return this.getRuntimeConfig(sessionId).llm; + } +} diff --git a/dexto/packages/core/src/agent/types.ts b/dexto/packages/core/src/agent/types.ts new file mode 100644 index 00000000..bb86272e --- /dev/null +++ b/dexto/packages/core/src/agent/types.ts @@ -0,0 +1,82 @@ +/** + * Type definitions for DextoAgent generate() and stream() APIs + * + * Re-uses existing types from context, llm/services, and events to avoid duplication. + */ + +import type { ContentPart } from '../context/types.js'; +import type { LLMTokenUsage } from '../llm/services/types.js'; + +/** + * Re-export content part types for API consumers + */ +export type { + ContentPart, + TextPart, + ImagePart, + FilePart, + ImageData, + FileData, +} from '../context/types.js'; +export type { LLMTokenUsage as TokenUsage } from '../llm/services/types.js'; + +/** + * Tool call information for agent streaming + * Simplified version of tools/types.ts ToolCall for streaming context + */ +export interface AgentToolCall { + toolName: string; + args: Record; + callId: string; + result?: + | { + success: boolean; + data: any; + } + | undefined; +} + +/** + * Content input for generate() and stream() methods. + * Can be a simple string (for text-only messages) or an array of ContentPart (for multimodal). + * + * @example + * ```typescript + * // Simple text + * agent.generate('What is 2+2?', sessionId); + * + * // Multimodal with image + * agent.generate([ + * { type: 'text', text: 'Describe this image' }, + * { type: 'image', image: base64Data, mimeType: 'image/png' } + * ], sessionId); + * ``` + */ +export type ContentInput = string | ContentPart[]; + +/** + * Options for generate() and stream() methods + */ +export interface GenerateOptions { + /** AbortSignal for cancellation */ + signal?: AbortSignal; +} + +/** + * Complete response from generate() method + */ +export interface GenerateResponse { + content: string; + reasoning?: string | undefined; // Extended thinking for o1/o3 models + usage: LLMTokenUsage; + toolCalls: AgentToolCall[]; + sessionId: string; +} + +/** + * Options for stream() method (same as generate) + * + * Note: stream() now returns core StreamingEvent types directly from the event system. + * See packages/core/src/events/index.ts for event definitions. + */ +export type StreamOptions = GenerateOptions; diff --git a/dexto/packages/core/src/approval/error-codes.ts b/dexto/packages/core/src/approval/error-codes.ts new file mode 100644 index 00000000..d7c8e9e0 --- /dev/null +++ b/dexto/packages/core/src/approval/error-codes.ts @@ -0,0 +1,31 @@ +/** + * Error codes for the approval system + * Covers validation, timeout, cancellation, and provider errors + */ +export enum ApprovalErrorCode { + // Validation errors + APPROVAL_INVALID_REQUEST = 'approval_invalid_request', + APPROVAL_INVALID_RESPONSE = 'approval_invalid_response', + APPROVAL_INVALID_METADATA = 'approval_invalid_metadata', + APPROVAL_INVALID_SCHEMA = 'approval_invalid_schema', + + // Timeout errors + APPROVAL_TIMEOUT = 'approval_timeout', + + // Cancellation errors + APPROVAL_CANCELLED = 'approval_cancelled', + APPROVAL_CANCELLED_ALL = 'approval_cancelled_all', + + // Provider errors + APPROVAL_PROVIDER_NOT_CONFIGURED = 'approval_provider_not_configured', + APPROVAL_PROVIDER_ERROR = 'approval_provider_error', + APPROVAL_NOT_FOUND = 'approval_not_found', + + // Type-specific errors + APPROVAL_TOOL_CONFIRMATION_DENIED = 'approval_tool_confirmation_denied', + APPROVAL_ELICITATION_DENIED = 'approval_elicitation_denied', + APPROVAL_ELICITATION_VALIDATION_FAILED = 'approval_elicitation_validation_failed', + + // Configuration errors + APPROVAL_CONFIG_INVALID = 'approval_config_invalid', +} diff --git a/dexto/packages/core/src/approval/errors.ts b/dexto/packages/core/src/approval/errors.ts new file mode 100644 index 00000000..c875e10f --- /dev/null +++ b/dexto/packages/core/src/approval/errors.ts @@ -0,0 +1,425 @@ +import { DextoRuntimeError, ErrorScope, ErrorType } from '../errors/index.js'; +import { ApprovalErrorCode } from './error-codes.js'; +import type { ApprovalType, DenialReason } from './types.js'; + +/** + * Context for approval validation errors + */ +export interface ApprovalValidationContext { + approvalId?: string; + type?: ApprovalType; + field?: string; + reason?: string; +} + +/** + * Context for approval timeout errors + */ +export interface ApprovalTimeoutContext { + approvalId: string; + type: ApprovalType; + timeout: number; + sessionId?: string; +} + +/** + * Context for approval cancellation errors + */ +export interface ApprovalCancellationContext { + approvalId?: string; + type?: ApprovalType; + reason?: string; +} + +/** + * Context for elicitation validation errors + */ +export interface ElicitationValidationContext { + approvalId: string; + serverName: string; + errors: string[]; +} + +/** + * Error factory for approval system errors + */ +export class ApprovalError { + /** + * Create an error for invalid approval request + */ + static invalidRequest( + reason: string, + context?: ApprovalValidationContext + ): DextoRuntimeError { + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_INVALID_REQUEST, + ErrorScope.TOOLS, // Approvals are part of tool execution flow + ErrorType.USER, + `Invalid approval request: ${reason}`, + context, + ['Check the approval request structure', 'Ensure all required fields are provided'] + ); + } + + /** + * Create an error for invalid approval response + */ + static invalidResponse( + reason: string, + context?: ApprovalValidationContext + ): DextoRuntimeError { + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_INVALID_RESPONSE, + ErrorScope.TOOLS, + ErrorType.USER, + `Invalid approval response: ${reason}`, + context, + [ + 'Check the approval response structure', + 'Ensure approvalId matches the request', + 'Verify status is valid', + ] + ); + } + + /** + * Create an error for invalid metadata + */ + static invalidMetadata( + type: ApprovalType, + reason: string + ): DextoRuntimeError { + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_INVALID_METADATA, + ErrorScope.TOOLS, + ErrorType.USER, + `Invalid metadata for ${type}: ${reason}`, + { type, reason }, + ['Check the metadata structure for this approval type'] + ); + } + + /** + * Create an error for invalid elicitation schema + */ + static invalidSchema(reason: string): DextoRuntimeError { + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_INVALID_SCHEMA, + ErrorScope.TOOLS, + ErrorType.USER, + `Invalid elicitation schema: ${reason}`, + { reason }, + ['Ensure the schema is a valid JSON Schema', 'Check MCP server implementation'] + ); + } + + /** + * Create an error for approval timeout + */ + static timeout( + approvalId: string, + type: ApprovalType, + timeout: number, + sessionId?: string + ): DextoRuntimeError { + const context: ApprovalTimeoutContext = { + approvalId, + type, + timeout, + }; + + if (sessionId !== undefined) { + context.sessionId = sessionId; + } + + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_TIMEOUT, + ErrorScope.TOOLS, + ErrorType.TIMEOUT, + `Approval request timed out after ${timeout}ms`, + context, + [ + 'Increase the timeout value', + 'Respond to approval requests more quickly', + 'Check if approval UI is functioning', + ] + ); + } + + /** + * Create an error for cancelled approval + */ + static cancelled( + approvalId: string, + type: ApprovalType, + reason?: string + ): DextoRuntimeError { + const message = reason + ? `Approval request cancelled: ${reason}` + : 'Approval request was cancelled'; + + const context: ApprovalCancellationContext = { + approvalId, + type, + }; + + if (reason !== undefined) { + context.reason = reason; + } + + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_CANCELLED, + ErrorScope.TOOLS, + ErrorType.USER, + message, + context + ); + } + + /** + * Create an error for all approvals cancelled + */ + static cancelledAll(reason?: string): DextoRuntimeError { + const message = reason + ? `All approval requests cancelled: ${reason}` + : 'All approval requests were cancelled'; + + const context: ApprovalCancellationContext = {}; + + if (reason !== undefined) { + context.reason = reason; + } + + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_CANCELLED_ALL, + ErrorScope.TOOLS, + ErrorType.USER, + message, + context + ); + } + + /** + * Create an error for approval provider not configured + */ + static providerNotConfigured(): DextoRuntimeError> { + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_PROVIDER_NOT_CONFIGURED, + ErrorScope.TOOLS, + ErrorType.SYSTEM, + 'Approval provider not configured', + {}, + [ + 'Configure an approval provider in your agent configuration', + 'Check approval.mode in agent.yml', + ] + ); + } + + /** + * Create an error for approval provider error + */ + static providerError(message: string, cause?: Error): DextoRuntimeError<{ cause?: string }> { + const context: { cause?: string } = {}; + + if (cause?.message !== undefined) { + context.cause = cause.message; + } + + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_PROVIDER_ERROR, + ErrorScope.TOOLS, + ErrorType.SYSTEM, + `Approval provider error: ${message}`, + context, + ['Check approval provider implementation', 'Review system logs for details'] + ); + } + + /** + * Create an error for approval not found + */ + static notFound(approvalId: string): DextoRuntimeError<{ approvalId: string }> { + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_NOT_FOUND, + ErrorScope.TOOLS, + ErrorType.NOT_FOUND, + `Approval request not found: ${approvalId}`, + { approvalId }, + [ + 'Verify the approvalId is correct', + 'Check if the approval has already been resolved or timed out', + ] + ); + } + + /** + * Create an error for tool confirmation denied + */ + static toolConfirmationDenied( + toolName: string, + reason?: DenialReason, + customMessage?: string, + sessionId?: string + ): DextoRuntimeError<{ toolName: string; reason?: DenialReason; sessionId?: string }> { + // Generate message based on reason + let message: string; + let suggestions: string[]; + + switch (reason) { + case 'user_denied': + message = customMessage ?? `Tool execution denied by user: ${toolName}`; + suggestions = ['Tool was denied by user']; + break; + case 'system_denied': + message = customMessage ?? `Tool execution denied by system policy: ${toolName}`; + suggestions = [ + 'Tool is in the alwaysDeny list', + 'Check toolConfirmation.toolPolicies in agent configuration', + ]; + break; + case 'timeout': + message = customMessage ?? `Tool confirmation timed out: ${toolName}`; + suggestions = [ + 'Increase the timeout value', + 'Respond to approval requests more quickly', + ]; + break; + default: + message = customMessage ?? `Tool execution denied: ${toolName}`; + suggestions = [ + 'Approve the tool in the confirmation dialog', + 'Check tool permissions', + ]; + } + + const context: { toolName: string; reason?: DenialReason; sessionId?: string } = { + toolName, + }; + if (reason) context.reason = reason; + if (sessionId) context.sessionId = sessionId; + + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_TOOL_CONFIRMATION_DENIED, + ErrorScope.TOOLS, + ErrorType.FORBIDDEN, + message, + context, + suggestions + ); + } + + /** + * Create an error for elicitation denied + */ + static elicitationDenied( + serverName: string, + reason?: DenialReason, + customMessage?: string, + sessionId?: string + ): DextoRuntimeError<{ serverName: string; reason?: DenialReason; sessionId?: string }> { + // Generate message based on reason + let message: string; + let suggestions: string[]; + + switch (reason) { + case 'user_denied': + message = + customMessage ?? + `Elicitation request denied by user from MCP server: ${serverName}`; + suggestions = [ + 'User clicked deny on the form', + 'The agent cannot proceed without this input', + ]; + break; + case 'user_cancelled': + message = + customMessage ?? + `Elicitation request cancelled by user from MCP server: ${serverName}`; + suggestions = [ + 'User cancelled the form', + 'The agent cannot proceed without this input', + ]; + break; + case 'system_cancelled': + message = + customMessage ?? `Elicitation request cancelled from MCP server: ${serverName}`; + suggestions = ['Session may have ended', 'Try again']; + break; + case 'timeout': + message = + customMessage ?? `Elicitation request timed out from MCP server: ${serverName}`; + suggestions = [ + 'Increase the timeout value', + 'Respond to elicitation requests more quickly', + ]; + break; + case 'elicitation_disabled': + message = + customMessage ?? + `Elicitation is disabled. Cannot request input from MCP server: ${serverName}`; + suggestions = [ + 'Enable elicitation in your agent configuration', + 'Set elicitation.enabled: true in agent.yml', + ]; + break; + case 'validation_failed': + message = + customMessage ?? + `Elicitation form validation failed from MCP server: ${serverName}`; + suggestions = ['Check the form inputs match the schema requirements']; + break; + default: + message = + customMessage ?? `Elicitation request denied from MCP server: ${serverName}`; + suggestions = ['Complete the requested form', 'Check MCP server requirements']; + } + + const context: { serverName: string; reason?: DenialReason; sessionId?: string } = { + serverName, + }; + if (reason) context.reason = reason; + if (sessionId) context.sessionId = sessionId; + + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_ELICITATION_DENIED, + ErrorScope.TOOLS, + ErrorType.FORBIDDEN, + message, + context, + suggestions + ); + } + + /** + * Create an error for elicitation validation failed + */ + static elicitationValidationFailed( + serverName: string, + errors: string[], + approvalId: string + ): DextoRuntimeError { + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_ELICITATION_VALIDATION_FAILED, + ErrorScope.TOOLS, + ErrorType.USER, + `Elicitation form validation failed: ${errors.join(', ')}`, + { approvalId, serverName, errors }, + ['Check the form inputs match the schema requirements', 'Review validation errors'] + ); + } + + /** + * Create an error for invalid approval configuration + */ + static invalidConfig(reason: string): DextoRuntimeError<{ reason: string }> { + return new DextoRuntimeError( + ApprovalErrorCode.APPROVAL_CONFIG_INVALID, + ErrorScope.TOOLS, + ErrorType.USER, + `Invalid approval configuration: ${reason}`, + { reason }, + ['Check approval configuration in agent.yml', 'Review approval.mode and related fields'] + ); + } +} diff --git a/dexto/packages/core/src/approval/factory.ts b/dexto/packages/core/src/approval/factory.ts new file mode 100644 index 00000000..d02ad32d --- /dev/null +++ b/dexto/packages/core/src/approval/factory.ts @@ -0,0 +1,22 @@ +import { randomUUID } from 'crypto'; +import type { ApprovalRequest, ApprovalRequestDetails } from './types.js'; + +/** + * Factory function to create an approval request with generated ID and timestamp. + * + * This is a generic helper used by ApprovalManager to create properly + * formatted approval requests from simplified details. + * + * @param details - Simplified approval request details without ID and timestamp + * @returns A complete ApprovalRequest with generated UUID and current timestamp + */ +export function createApprovalRequest(details: ApprovalRequestDetails): ApprovalRequest { + return { + approvalId: randomUUID(), + type: details.type, + sessionId: details.sessionId, + timeout: details.timeout, + timestamp: new Date(), + metadata: details.metadata, + } as ApprovalRequest; +} diff --git a/dexto/packages/core/src/approval/index.ts b/dexto/packages/core/src/approval/index.ts new file mode 100644 index 00000000..7dbeb87a --- /dev/null +++ b/dexto/packages/core/src/approval/index.ts @@ -0,0 +1,72 @@ +// ============================================================================ +// USER APPROVAL SYSTEM - Public API +// ============================================================================ + +// Types +export type { + ApprovalHandler, + ApprovalRequest, + ApprovalResponse, + ApprovalRequestDetails, + ElicitationMetadata, + ElicitationRequest, + ElicitationResponse, + ElicitationResponseData, + CustomApprovalMetadata, + CustomApprovalRequest, + CustomApprovalResponse, + CustomApprovalResponseData, + BaseApprovalRequest, + BaseApprovalResponse, +} from './types.js'; + +// Internal types - not exported to avoid naming conflicts with tools module +// ToolConfirmationMetadata, ToolConfirmationRequest, ToolConfirmationResponse, ToolConfirmationResponseData + +export { ApprovalType, ApprovalStatus, DenialReason } from './types.js'; + +// Schemas +export { + ApprovalTypeSchema, + ApprovalStatusSchema, + DenialReasonSchema, + ToolConfirmationMetadataSchema, + ElicitationMetadataSchema, + CustomApprovalMetadataSchema, + BaseApprovalRequestSchema, + ToolConfirmationRequestSchema, + ElicitationRequestSchema, + CustomApprovalRequestSchema, + ApprovalRequestSchema, + ToolConfirmationResponseDataSchema, + ElicitationResponseDataSchema, + CustomApprovalResponseDataSchema, + BaseApprovalResponseSchema, + ToolConfirmationResponseSchema, + ElicitationResponseSchema, + CustomApprovalResponseSchema, + ApprovalResponseSchema, + ApprovalRequestDetailsSchema, +} from './schemas.js'; + +export type { + ValidatedApprovalRequest, + ValidatedApprovalResponse, + ValidatedToolConfirmationRequest, + ValidatedElicitationRequest, + ValidatedCustomApprovalRequest, +} from './schemas.js'; + +// Error codes and errors +export { ApprovalErrorCode } from './error-codes.js'; +export { ApprovalError } from './errors.js'; +export type { + ApprovalValidationContext, + ApprovalTimeoutContext, + ApprovalCancellationContext, + ElicitationValidationContext, +} from './errors.js'; + +// Manager +export { ApprovalManager } from './manager.js'; +export type { ApprovalManagerConfig } from './manager.js'; diff --git a/dexto/packages/core/src/approval/manager.test.ts b/dexto/packages/core/src/approval/manager.test.ts new file mode 100644 index 00000000..e8320772 --- /dev/null +++ b/dexto/packages/core/src/approval/manager.test.ts @@ -0,0 +1,957 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { ApprovalManager } from './manager.js'; +import { ApprovalStatus, DenialReason } from './types.js'; +import { AgentEventBus } from '../events/index.js'; +import { DextoRuntimeError } from '../errors/index.js'; +import { ApprovalErrorCode } from './error-codes.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; + +describe('ApprovalManager', () => { + let agentEventBus: AgentEventBus; + const mockLogger = createMockLogger(); + + beforeEach(() => { + agentEventBus = new AgentEventBus(); + }); + + describe('Configuration - Separate tool and elicitation control', () => { + it('should allow auto-approve for tools while elicitation is enabled', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + // Tool confirmation should be auto-approved + const toolResponse = await manager.requestToolConfirmation({ + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: { foo: 'bar' }, + }); + + expect(toolResponse.status).toBe(ApprovalStatus.APPROVED); + }); + + it('should reject elicitation when disabled, even if tools are auto-approved', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: false, + timeout: 120000, + }, + }, + mockLogger + ); + + // Elicitation should throw error when disabled + await expect( + manager.requestElicitation({ + schema: { + type: 'object' as const, + properties: { + name: { type: 'string' as const }, + }, + }, + prompt: 'Enter your name', + serverName: 'Test Server', + }) + ).rejects.toThrow(DextoRuntimeError); + + await expect( + manager.requestElicitation({ + schema: { + type: 'object' as const, + properties: { + name: { type: 'string' as const }, + }, + }, + prompt: 'Enter your name', + serverName: 'Test Server', + }) + ).rejects.toThrow(/Elicitation is disabled/); + }); + + it('should auto-deny tools while elicitation is enabled', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-deny', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + // Tool confirmation should be auto-denied + const toolResponse = await manager.requestToolConfirmation({ + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: { foo: 'bar' }, + }); + + expect(toolResponse.status).toBe(ApprovalStatus.DENIED); + }); + + it('should use separate timeouts for tools and elicitation', () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + timeout: 60000, + }, + elicitation: { + enabled: true, + timeout: 180000, + }, + }, + mockLogger + ); + + const config = manager.getConfig(); + expect(config.toolConfirmation.timeout).toBe(60000); + expect(config.elicitation.timeout).toBe(180000); + }); + }); + + describe('Approval routing by type', () => { + it('should route tool confirmations to tool provider', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + const response = await manager.requestToolConfirmation({ + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }); + + expect(response.status).toBe(ApprovalStatus.APPROVED); + }); + + it('should route command confirmations to tool provider', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + const response = await manager.requestCommandConfirmation({ + toolName: 'bash_exec', + command: 'rm -rf /', + originalCommand: 'rm -rf /', + }); + + expect(response.status).toBe(ApprovalStatus.APPROVED); + }); + + it('should route elicitation to elicitation provider when enabled', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-deny', // Different mode for tools + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + // Elicitation should not be auto-denied (uses manual handler) + // We'll timeout immediately to avoid hanging tests + await expect( + manager.requestElicitation({ + schema: { + type: 'object' as const, + properties: { + name: { type: 'string' as const }, + }, + }, + prompt: 'Enter your name', + serverName: 'Test Server', + timeout: 1, // 1ms timeout to fail fast + }) + ).rejects.toThrow(); // Should timeout, not be auto-denied + }); + }); + + describe('Pending approvals tracking', () => { + it('should track pending approvals across both providers', () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + // Initially no pending approvals + expect(manager.getPendingApprovals()).toEqual([]); + + // Auto-approve mode would not create pending approvals + // Event-based mode would, but we don't want hanging requests in tests + }); + + it('should cancel approvals in both providers', () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + // Should not throw when cancelling (even if approval doesn't exist) + expect(() => manager.cancelApproval('test-id')).not.toThrow(); + expect(() => manager.cancelAllApprovals()).not.toThrow(); + }); + }); + + describe('Error handling', () => { + it('should throw clear error when elicitation is disabled', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: false, + timeout: 120000, + }, + }, + mockLogger + ); + + await expect( + manager.getElicitationData({ + schema: { + type: 'object' as const, + properties: { + name: { type: 'string' as const }, + }, + }, + prompt: 'Enter your name', + serverName: 'Test Server', + }) + ).rejects.toThrow(/Elicitation is disabled/); + }); + + it('should provide helpful error message about enabling elicitation', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: false, + timeout: 120000, + }, + }, + mockLogger + ); + + try { + await manager.requestElicitation({ + schema: { + type: 'object' as const, + properties: { + name: { type: 'string' as const }, + }, + }, + prompt: 'Enter your name', + serverName: 'Test Server', + }); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as Error).message).toContain('Enable elicitation'); + expect((error as Error).message).toContain('agent configuration'); + } + }); + }); + + describe('Timeout Configuration', () => { + it('should allow undefined timeout (infinite wait) for tool confirmation', () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + // No timeout specified - should wait indefinitely + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + const config = manager.getConfig(); + expect(config.toolConfirmation.timeout).toBeUndefined(); + }); + + it('should allow undefined timeout (infinite wait) for elicitation', () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + timeout: 60000, + }, + elicitation: { + enabled: true, + // No timeout specified - should wait indefinitely + }, + }, + mockLogger + ); + + const config = manager.getConfig(); + expect(config.elicitation.timeout).toBeUndefined(); + }); + + it('should allow both timeouts to be undefined (infinite wait for all approvals)', () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + // No timeout + }, + elicitation: { + enabled: true, + // No timeout + }, + }, + mockLogger + ); + + const config = manager.getConfig(); + expect(config.toolConfirmation.timeout).toBeUndefined(); + expect(config.elicitation.timeout).toBeUndefined(); + }); + + it('should use per-request timeout override when provided', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', // Auto-approve so we can test immediately + timeout: 60000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + // The per-request timeout should override the config timeout + // This is tested implicitly through the factory flow + const response = await manager.requestToolConfirmation({ + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: { foo: 'bar' }, + timeout: 30000, // Per-request override + }); + + expect(response.status).toBe(ApprovalStatus.APPROVED); + }); + + it('should not timeout when timeout is undefined in auto-approve mode', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + // No timeout - should not cause any issues with auto-approve + }, + elicitation: { + enabled: false, + }, + }, + mockLogger + ); + + const response = await manager.requestToolConfirmation({ + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }); + + expect(response.status).toBe(ApprovalStatus.APPROVED); + }); + + it('should not timeout when timeout is undefined in auto-deny mode', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-deny', + // No timeout - should not cause any issues with auto-deny + }, + elicitation: { + enabled: false, + }, + }, + mockLogger + ); + + const response = await manager.requestToolConfirmation({ + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }); + + expect(response.status).toBe(ApprovalStatus.DENIED); + expect(response.reason).toBe(DenialReason.SYSTEM_DENIED); + }); + }); + + describe('Backward compatibility', () => { + it('should work with manual mode for both tools and elicitation', () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + expect(manager.getConfig()).toEqual({ + toolConfirmation: { + mode: 'manual', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }); + }); + + it('should respect explicitly set elicitation enabled value', () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + expect(manager.getConfig().elicitation.enabled).toBe(true); + }); + }); + + describe('Denial Reasons', () => { + it('should include system_denied reason in auto-deny mode', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-deny', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + const response = await manager.requestToolConfirmation({ + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }); + + expect(response.status).toBe(ApprovalStatus.DENIED); + expect(response.reason).toBe(DenialReason.SYSTEM_DENIED); + expect(response.message).toContain('system policy'); + }); + + it('should throw error with specific reason when tool is denied', async () => { + const manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-deny', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + try { + await manager.checkToolConfirmation({ + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe( + ApprovalErrorCode.APPROVAL_TOOL_CONFIRMATION_DENIED + ); + expect((error as DextoRuntimeError).message).toContain('system policy'); + expect((error as any).context.reason).toBe(DenialReason.SYSTEM_DENIED); + } + }); + + it('should handle user_denied reason in error message', async () => { + const _manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + timeout: 1, // Quick timeout for test + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + // Simulate user denying via event + setTimeout(() => { + agentEventBus.emit('approval:response', { + approvalId: expect.any(String), + status: ApprovalStatus.DENIED, + reason: DenialReason.USER_DENIED, + message: 'User clicked deny', + } as any); + }, 50); + + // This will be challenging to test properly without mocking more, + // so let's just ensure the type system accepts it + expect(DenialReason.USER_DENIED).toBe('user_denied'); + expect(DenialReason.TIMEOUT).toBe('timeout'); + }); + + it('should include reason in response schema', () => { + // Verify the type system allows reason and message + const response: { reason?: DenialReason; message?: string } = { + reason: DenialReason.USER_DENIED, + message: 'You denied this request', + }; + + expect(response.reason).toBe(DenialReason.USER_DENIED); + expect(response.message).toBe('You denied this request'); + }); + + it('should support all denial reason types', () => { + const reasons: DenialReason[] = [ + DenialReason.USER_DENIED, + DenialReason.SYSTEM_DENIED, + DenialReason.TIMEOUT, + DenialReason.USER_CANCELLED, + DenialReason.SYSTEM_CANCELLED, + DenialReason.VALIDATION_FAILED, + DenialReason.ELICITATION_DISABLED, + ]; + + expect(reasons.length).toBe(7); + reasons.forEach((reason) => { + expect(typeof reason).toBe('string'); + }); + }); + }); + + describe('Bash Pattern Approval', () => { + let manager: ApprovalManager; + + beforeEach(() => { + manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + timeout: 120000, + }, + elicitation: { + enabled: false, + }, + }, + mockLogger + ); + }); + + describe('addBashPattern', () => { + it('should add a pattern to the approved list', () => { + manager.addBashPattern('git *'); + expect(manager.getBashPatterns().has('git *')).toBe(true); + }); + + it('should add multiple patterns', () => { + manager.addBashPattern('git *'); + manager.addBashPattern('npm *'); + manager.addBashPattern('ls *'); + + const patterns = manager.getBashPatterns(); + expect(patterns.size).toBe(3); + expect(patterns.has('git *')).toBe(true); + expect(patterns.has('npm *')).toBe(true); + expect(patterns.has('ls *')).toBe(true); + }); + + it('should not duplicate patterns', () => { + manager.addBashPattern('git *'); + manager.addBashPattern('git *'); + + expect(manager.getBashPatterns().size).toBe(1); + }); + }); + + describe('matchesBashPattern (pattern-to-pattern covering)', () => { + // Note: matchesBashPattern expects pattern keys (e.g., "git push *"), + // not raw commands. ToolManager generates pattern keys from commands. + + it('should match exact pattern against exact stored pattern', () => { + manager.addBashPattern('git status *'); + expect(manager.matchesBashPattern('git status *')).toBe(true); + expect(manager.matchesBashPattern('git push *')).toBe(false); + }); + + it('should cover narrower pattern with broader pattern', () => { + // "git *" is broader and should cover "git push *", "git status *", etc. + manager.addBashPattern('git *'); + expect(manager.matchesBashPattern('git *')).toBe(true); + expect(manager.matchesBashPattern('git push *')).toBe(true); + expect(manager.matchesBashPattern('git status *')).toBe(true); + expect(manager.matchesBashPattern('npm *')).toBe(false); + }); + + it('should not let narrower pattern cover broader pattern', () => { + // "git push *" should NOT cover "git *" + manager.addBashPattern('git push *'); + expect(manager.matchesBashPattern('git push *')).toBe(true); + expect(manager.matchesBashPattern('git *')).toBe(false); + expect(manager.matchesBashPattern('git status *')).toBe(false); + }); + + it('should match against multiple patterns', () => { + manager.addBashPattern('git *'); + manager.addBashPattern('npm install *'); + + expect(manager.matchesBashPattern('git status *')).toBe(true); + expect(manager.matchesBashPattern('npm install *')).toBe(true); + // npm * is not covered, only npm install * specifically + expect(manager.matchesBashPattern('npm run *')).toBe(false); + }); + + it('should return false when no patterns are set', () => { + expect(manager.matchesBashPattern('git status *')).toBe(false); + }); + + it('should not cross-match unrelated commands', () => { + manager.addBashPattern('npm *'); + // "npx" starts with "np" but is not "npm " + something + expect(manager.matchesBashPattern('npx *')).toBe(false); + }); + + it('should handle multi-level subcommands', () => { + manager.addBashPattern('docker compose *'); + expect(manager.matchesBashPattern('docker compose *')).toBe(true); + expect(manager.matchesBashPattern('docker compose up *')).toBe(true); + expect(manager.matchesBashPattern('docker *')).toBe(false); + }); + }); + + describe('clearBashPatterns', () => { + it('should clear all patterns', () => { + manager.addBashPattern('git *'); + manager.addBashPattern('npm *'); + expect(manager.getBashPatterns().size).toBe(2); + + manager.clearBashPatterns(); + expect(manager.getBashPatterns().size).toBe(0); + }); + + it('should allow adding patterns after clearing', () => { + manager.addBashPattern('git *'); + manager.clearBashPatterns(); + manager.addBashPattern('npm *'); + + expect(manager.getBashPatterns().size).toBe(1); + expect(manager.getBashPatterns().has('npm *')).toBe(true); + }); + }); + + describe('getBashPatterns', () => { + it('should return empty set initially', () => { + expect(manager.getBashPatterns().size).toBe(0); + }); + + it('should return a copy that reflects current patterns', () => { + manager.addBashPattern('git *'); + const patterns = manager.getBashPatterns(); + expect(patterns.has('git *')).toBe(true); + + // Note: ReadonlySet is a TypeScript type constraint, not runtime protection + // The returned set IS the internal set, so modifying it would affect the manager + // This is acceptable for our use case (debugging/display) + }); + }); + }); + + describe('Directory Access Approval', () => { + let manager: ApprovalManager; + + beforeEach(() => { + manager = new ApprovalManager( + { + toolConfirmation: { + mode: 'manual', + timeout: 120000, + }, + elicitation: { + enabled: false, + }, + }, + mockLogger + ); + }); + + describe('initializeWorkingDirectory', () => { + it('should add working directory as session-approved', () => { + manager.initializeWorkingDirectory('/home/user/project'); + expect(manager.isDirectorySessionApproved('/home/user/project/src/file.ts')).toBe( + true + ); + }); + + it('should normalize the path before adding', () => { + manager.initializeWorkingDirectory('/home/user/../user/project'); + expect(manager.isDirectorySessionApproved('/home/user/project/file.ts')).toBe(true); + }); + }); + + describe('addApprovedDirectory', () => { + it('should add directory with session type by default', () => { + manager.addApprovedDirectory('/external/project'); + expect(manager.isDirectorySessionApproved('/external/project/file.ts')).toBe(true); + }); + + it('should add directory with explicit session type', () => { + manager.addApprovedDirectory('/external/project', 'session'); + expect(manager.isDirectorySessionApproved('/external/project/file.ts')).toBe(true); + }); + + it('should add directory with once type', () => { + manager.addApprovedDirectory('/external/project', 'once'); + // 'once' type should NOT be session-approved (requires prompt each time) + expect(manager.isDirectorySessionApproved('/external/project/file.ts')).toBe(false); + // But should be generally approved for execution + expect(manager.isDirectoryApproved('/external/project/file.ts')).toBe(true); + }); + + it('should not downgrade from session to once', () => { + manager.addApprovedDirectory('/external/project', 'session'); + manager.addApprovedDirectory('/external/project', 'once'); + // Should still be session-approved + expect(manager.isDirectorySessionApproved('/external/project/file.ts')).toBe(true); + }); + + it('should upgrade from once to session', () => { + manager.addApprovedDirectory('/external/project', 'once'); + expect(manager.isDirectorySessionApproved('/external/project/file.ts')).toBe(false); + + manager.addApprovedDirectory('/external/project', 'session'); + expect(manager.isDirectorySessionApproved('/external/project/file.ts')).toBe(true); + }); + + it('should normalize paths before adding', () => { + manager.addApprovedDirectory('/external/../external/project'); + expect(manager.isDirectoryApproved('/external/project/file.ts')).toBe(true); + }); + }); + + describe('isDirectorySessionApproved', () => { + it('should return true for files within session-approved directory', () => { + manager.addApprovedDirectory('/external/project', 'session'); + expect(manager.isDirectorySessionApproved('/external/project/file.ts')).toBe(true); + expect( + manager.isDirectorySessionApproved('/external/project/src/deep/file.ts') + ).toBe(true); + }); + + it('should return false for files within once-approved directory', () => { + manager.addApprovedDirectory('/external/project', 'once'); + expect(manager.isDirectorySessionApproved('/external/project/file.ts')).toBe(false); + }); + + it('should return false for files outside approved directories', () => { + manager.addApprovedDirectory('/external/project', 'session'); + expect(manager.isDirectorySessionApproved('/other/file.ts')).toBe(false); + }); + + it('should handle path containment correctly', () => { + manager.addApprovedDirectory('/external', 'session'); + // Approving /external should cover /external/sub/file.ts + expect(manager.isDirectorySessionApproved('/external/sub/file.ts')).toBe(true); + // But not /external-other/file.ts (different directory) + expect(manager.isDirectorySessionApproved('/external-other/file.ts')).toBe(false); + }); + + it('should return true when working directory is initialized', () => { + manager.initializeWorkingDirectory('/home/user/project'); + expect(manager.isDirectorySessionApproved('/home/user/project/any/file.ts')).toBe( + true + ); + }); + }); + + describe('isDirectoryApproved', () => { + it('should return true for files within session-approved directory', () => { + manager.addApprovedDirectory('/external/project', 'session'); + expect(manager.isDirectoryApproved('/external/project/file.ts')).toBe(true); + }); + + it('should return true for files within once-approved directory', () => { + manager.addApprovedDirectory('/external/project', 'once'); + expect(manager.isDirectoryApproved('/external/project/file.ts')).toBe(true); + }); + + it('should return false for files outside approved directories', () => { + manager.addApprovedDirectory('/external/project', 'session'); + expect(manager.isDirectoryApproved('/other/file.ts')).toBe(false); + }); + + it('should handle multiple approved directories', () => { + manager.addApprovedDirectory('/external/project1', 'session'); + manager.addApprovedDirectory('/external/project2', 'once'); + + expect(manager.isDirectoryApproved('/external/project1/file.ts')).toBe(true); + expect(manager.isDirectoryApproved('/external/project2/file.ts')).toBe(true); + expect(manager.isDirectoryApproved('/external/project3/file.ts')).toBe(false); + }); + + it('should handle nested directory approvals', () => { + manager.addApprovedDirectory('/external', 'session'); + // Approving /external should cover all subdirectories + expect(manager.isDirectoryApproved('/external/sub/deep/file.ts')).toBe(true); + }); + }); + + describe('getApprovedDirectories', () => { + it('should return empty map initially', () => { + expect(manager.getApprovedDirectories().size).toBe(0); + }); + + it('should return map with type information', () => { + manager.addApprovedDirectory('/external/project1', 'session'); + manager.addApprovedDirectory('/external/project2', 'once'); + + const dirs = manager.getApprovedDirectories(); + expect(dirs.size).toBe(2); + // Check that paths are normalized (absolute) + const keys = Array.from(dirs.keys()); + expect(keys.some((k) => k.includes('project1'))).toBe(true); + expect(keys.some((k) => k.includes('project2'))).toBe(true); + }); + + it('should include working directory after initialization', () => { + manager.initializeWorkingDirectory('/home/user/project'); + const dirs = manager.getApprovedDirectories(); + expect(dirs.size).toBe(1); + // Check that working directory is session type + const entries = Array.from(dirs.entries()); + expect(entries[0]![1]).toBe('session'); + }); + }); + + describe('Session vs Once Prompting Behavior', () => { + // These tests verify the expected prompting flow + + it('working directory should not require prompt (session-approved)', () => { + manager.initializeWorkingDirectory('/home/user/project'); + // isDirectorySessionApproved returns true → no directory prompt needed + expect(manager.isDirectorySessionApproved('/home/user/project/src/file.ts')).toBe( + true + ); + }); + + it('external dir after session approval should not require prompt', () => { + manager.addApprovedDirectory('/external', 'session'); + // isDirectorySessionApproved returns true → no directory prompt needed + expect(manager.isDirectorySessionApproved('/external/file.ts')).toBe(true); + }); + + it('external dir after once approval should require prompt each time', () => { + manager.addApprovedDirectory('/external', 'once'); + // isDirectorySessionApproved returns false → directory prompt needed + expect(manager.isDirectorySessionApproved('/external/file.ts')).toBe(false); + // But isDirectoryApproved returns true → execution allowed + expect(manager.isDirectoryApproved('/external/file.ts')).toBe(true); + }); + + it('unapproved external dir should require prompt', () => { + // No directories approved + expect(manager.isDirectorySessionApproved('/external/file.ts')).toBe(false); + expect(manager.isDirectoryApproved('/external/file.ts')).toBe(false); + }); + }); + }); +}); diff --git a/dexto/packages/core/src/approval/manager.ts b/dexto/packages/core/src/approval/manager.ts new file mode 100644 index 00000000..2287a9ce --- /dev/null +++ b/dexto/packages/core/src/approval/manager.ts @@ -0,0 +1,661 @@ +import path from 'node:path'; +import type { + ApprovalHandler, + ApprovalRequest, + ApprovalResponse, + ApprovalRequestDetails, + ToolConfirmationMetadata, + CommandConfirmationMetadata, + ElicitationMetadata, + DirectoryAccessMetadata, +} from './types.js'; +import { ApprovalType, ApprovalStatus, DenialReason } from './types.js'; +import { createApprovalRequest } from './factory.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import { ApprovalError } from './errors.js'; +import { patternCovers } from '../tools/bash-pattern-utils.js'; + +/** + * Configuration for the approval manager + */ +export interface ApprovalManagerConfig { + toolConfirmation: { + mode: 'manual' | 'auto-approve' | 'auto-deny'; + timeout?: number; // Optional - no timeout if not specified + }; + elicitation: { + enabled: boolean; + timeout?: number; // Optional - no timeout if not specified + }; +} + +/** + * ApprovalManager orchestrates all user approval flows in Dexto. + * + * It provides a unified interface for requesting user approvals across different + * types (tool confirmation, MCP elicitation, custom approvals) and manages the + * underlying approval provider based on configuration. + * + * Key responsibilities: + * - Create and submit approval requests + * - Route approvals to appropriate providers + * - Provide convenience methods for specific approval types + * - Handle approval responses and errors + * - Support multiple approval modes (manual, auto-approve, auto-deny) + * + * @example + * ```typescript + * const manager = new ApprovalManager( + * { toolConfirmation: { mode: 'manual', timeout: 60000 }, elicitation: { enabled: true, timeout: 60000 } }, + * logger + * ); + * + * // Request tool confirmation + * const response = await manager.requestToolConfirmation({ + * toolName: 'git_commit', + * args: { message: 'feat: add feature' }, + * sessionId: 'session-123' + * }); + * + * if (response.status === 'approved') { + * // Execute tool + * } + * ``` + */ +export class ApprovalManager { + private handler: ApprovalHandler | undefined; + private config: ApprovalManagerConfig; + private logger: IDextoLogger; + + /** + * Bash command patterns approved for the current session. + * Patterns use simple glob syntax (e.g., "git *", "npm install *"). + * Cleared when session ends. + */ + private bashPatterns: Set = new Set(); + + /** + * Directories approved for file access for the current session. + * Stores normalized absolute paths mapped to their approval type: + * - 'session': No directory prompt, follows tool config (working dir + user session-approved) + * - 'once': Prompts each time, but tool can execute + * Cleared when session ends. + */ + private approvedDirectories: Map = new Map(); + + constructor(config: ApprovalManagerConfig, logger: IDextoLogger) { + this.config = config; + this.logger = logger.createChild(DextoLogComponent.APPROVAL); + + this.logger.debug( + `ApprovalManager initialized with toolConfirmation.mode: ${config.toolConfirmation.mode}, elicitation.enabled: ${config.elicitation.enabled}` + ); + } + + // ==================== Bash Pattern Methods ==================== + + /** + * Add a bash command pattern to the approved list for this session. + * Patterns use simple glob syntax with * as wildcard. + * + * @example + * ```typescript + * manager.addBashPattern("git *"); // Approves all git commands + * manager.addBashPattern("npm install *"); // Approves npm install with any package + * ``` + */ + addBashPattern(pattern: string): void { + this.bashPatterns.add(pattern); + this.logger.debug(`Added bash pattern: "${pattern}"`); + } + + /** + * Check if a bash pattern key is covered by any approved pattern. + * Uses pattern-to-pattern covering for broader pattern support. + * + * @param patternKey The pattern key generated from the command (e.g., "git push *") + * @returns true if the pattern key is covered by an approved pattern + */ + matchesBashPattern(patternKey: string): boolean { + for (const storedPattern of this.bashPatterns) { + if (patternCovers(storedPattern, patternKey)) { + this.logger.debug( + `Pattern key "${patternKey}" is covered by approved pattern "${storedPattern}"` + ); + return true; + } + } + return false; + } + + /** + * Clear all approved bash patterns. + * Should be called when session ends. + */ + clearBashPatterns(): void { + const count = this.bashPatterns.size; + this.bashPatterns.clear(); + if (count > 0) { + this.logger.debug(`Cleared ${count} bash patterns`); + } + } + + /** + * Get the current set of approved bash patterns (for debugging/display). + */ + getBashPatterns(): ReadonlySet { + return this.bashPatterns; + } + + // ==================== Directory Access Methods ==================== + + /** + * Initialize the working directory as a session-approved directory. + * This should be called once during setup to ensure the working directory + * never triggers directory access prompts. + * + * @param workingDir The working directory path + */ + initializeWorkingDirectory(workingDir: string): void { + const normalized = path.resolve(workingDir); + this.approvedDirectories.set(normalized, 'session'); + this.logger.debug(`Initialized working directory as session-approved: "${normalized}"`); + } + + /** + * Add a directory to the approved list for this session. + * Files within this directory (including subdirectories) will be allowed. + * + * @param directory Absolute path to the directory to approve + * @param type The approval type: + * - 'session': No directory prompt on future accesses, follows tool config + * - 'once': Will prompt again on future accesses, but tool can execute this time + * @example + * ```typescript + * manager.addApprovedDirectory("/external/project", 'session'); + * // Now /external/project/src/file.ts is accessible without directory prompt + * + * manager.addApprovedDirectory("/tmp/files", 'once'); + * // Tool can access, but will prompt again next time + * ``` + */ + addApprovedDirectory(directory: string, type: 'session' | 'once' = 'session'): void { + const normalized = path.resolve(directory); + const existing = this.approvedDirectories.get(normalized); + + // Don't downgrade from 'session' to 'once' + if (existing === 'session') { + this.logger.debug( + `Directory "${normalized}" already approved as 'session', not downgrading to '${type}'` + ); + return; + } + + this.approvedDirectories.set(normalized, type); + this.logger.debug(`Added approved directory: "${normalized}" (type: ${type})`); + } + + /** + * Check if a file path is within any session-approved directory. + * This is used for PROMPTING decisions - only 'session' type directories count. + * Working directory and user session-approved directories return true. + * + * @param filePath The file path to check (can be relative or absolute) + * @returns true if the path is within a session-approved directory + */ + isDirectorySessionApproved(filePath: string): boolean { + const normalized = path.resolve(filePath); + + for (const [approvedDir, type] of this.approvedDirectories) { + // Only check 'session' type directories for prompting decisions + if (type !== 'session') continue; + + const relative = path.relative(approvedDir, normalized); + if (!relative.startsWith('..') && !path.isAbsolute(relative)) { + this.logger.debug( + `Path "${normalized}" is within session-approved directory "${approvedDir}"` + ); + return true; + } + } + return false; + } + + /** + * Check if a file path is within any approved directory (session OR once). + * This is used for EXECUTION decisions - both 'session' and 'once' types count. + * PathValidator uses this to determine if a tool can access the path. + * + * @param filePath The file path to check (can be relative or absolute) + * @returns true if the path is within any approved directory + */ + isDirectoryApproved(filePath: string): boolean { + const normalized = path.resolve(filePath); + + for (const [approvedDir] of this.approvedDirectories) { + const relative = path.relative(approvedDir, normalized); + if (!relative.startsWith('..') && !path.isAbsolute(relative)) { + this.logger.debug( + `Path "${normalized}" is within approved directory "${approvedDir}"` + ); + return true; + } + } + return false; + } + + /** + * Clear all approved directories. + * Should be called when session ends. + */ + clearApprovedDirectories(): void { + const count = this.approvedDirectories.size; + this.approvedDirectories.clear(); + if (count > 0) { + this.logger.debug(`Cleared ${count} approved directories`); + } + } + + /** + * Get the current map of approved directories with their types (for debugging/display). + */ + getApprovedDirectories(): ReadonlyMap { + return this.approvedDirectories; + } + + /** + * Get just the directory paths that are approved (for debugging/display). + */ + getApprovedDirectoryPaths(): string[] { + return Array.from(this.approvedDirectories.keys()); + } + + /** + * Clear all session-scoped approvals (bash patterns and directories). + * Convenience method for clearing all session state at once. + */ + clearSessionApprovals(): void { + this.clearBashPatterns(); + this.clearApprovedDirectories(); + this.logger.debug('Cleared all session approvals'); + } + + /** + * Request directory access approval. + * Convenience method for directory access requests. + * + * @example + * ```typescript + * const response = await manager.requestDirectoryAccess({ + * path: '/external/project/src/file.ts', + * parentDir: '/external/project', + * operation: 'write', + * toolName: 'write_file', + * sessionId: 'session-123' + * }); + * ``` + */ + async requestDirectoryAccess( + metadata: DirectoryAccessMetadata & { sessionId?: string; timeout?: number } + ): Promise { + const { sessionId, timeout, ...directoryMetadata } = metadata; + + const details: ApprovalRequestDetails = { + type: ApprovalType.DIRECTORY_ACCESS, + // Use provided timeout, fallback to config timeout, or undefined (no timeout) + timeout: timeout !== undefined ? timeout : this.config.toolConfirmation.timeout, + metadata: directoryMetadata, + }; + + if (sessionId !== undefined) { + details.sessionId = sessionId; + } + + return this.requestApproval(details); + } + + /** + * Request a generic approval + */ + async requestApproval(details: ApprovalRequestDetails): Promise { + const request = createApprovalRequest(details); + + // Check elicitation config if this is an elicitation request + if (request.type === ApprovalType.ELICITATION && !this.config.elicitation.enabled) { + throw ApprovalError.invalidConfig( + 'Elicitation is disabled. Enable elicitation in your agent configuration to use the ask_user tool or MCP server elicitations.' + ); + } + + // Handle all approval types uniformly + return this.handleApproval(request); + } + + /** + * Handle approval requests (tool confirmation, elicitation, command confirmation, directory access, custom) + * @private + */ + private async handleApproval(request: ApprovalRequest): Promise { + // Elicitation always uses manual mode (requires handler) + if (request.type === ApprovalType.ELICITATION) { + const handler = this.ensureHandler(); + this.logger.info( + `Elicitation requested, approvalId: ${request.approvalId}, sessionId: ${request.sessionId ?? 'global'}` + ); + return handler(request); + } + + // Tool/command/directory-access/custom confirmations respect the configured mode + const mode = this.config.toolConfirmation.mode; + + // Auto-approve mode + if (mode === 'auto-approve') { + this.logger.info( + `Auto-approve approval '${request.type}', approvalId: ${request.approvalId}` + ); + const response: ApprovalResponse = { + approvalId: request.approvalId, + status: ApprovalStatus.APPROVED, + }; + if (request.sessionId !== undefined) { + response.sessionId = request.sessionId; + } + return response; + } + + // Auto-deny mode + if (mode === 'auto-deny') { + this.logger.info( + `Auto-deny approval '${request.type}', approvalId: ${request.approvalId}` + ); + const response: ApprovalResponse = { + approvalId: request.approvalId, + status: ApprovalStatus.DENIED, + reason: DenialReason.SYSTEM_DENIED, + message: `Approval automatically denied by system policy (auto-deny mode)`, + }; + if (request.sessionId !== undefined) { + response.sessionId = request.sessionId; + } + return response; + } + + // Manual mode - delegate to handler + const handler = this.ensureHandler(); + this.logger.info( + `Manual approval '${request.type}' requested, approvalId: ${request.approvalId}, sessionId: ${request.sessionId ?? 'global'}` + ); + return handler(request); + } + + /** + * Request tool confirmation approval + * Convenience method for tool execution confirmation + * + * TODO: Make sessionId required once all callers are updated to pass it + * Tool confirmations always happen in session context during LLM execution + */ + async requestToolConfirmation( + metadata: ToolConfirmationMetadata & { sessionId?: string; timeout?: number } + ): Promise { + const { sessionId, timeout, ...toolMetadata } = metadata; + + const details: ApprovalRequestDetails = { + type: ApprovalType.TOOL_CONFIRMATION, + // Use provided timeout, fallback to config timeout, or undefined (no timeout) + timeout: timeout !== undefined ? timeout : this.config.toolConfirmation.timeout, + metadata: toolMetadata, + }; + + if (sessionId !== undefined) { + details.sessionId = sessionId; + } + + return this.requestApproval(details); + } + + /** + * Request command confirmation approval + * Convenience method for dangerous command execution within an already-approved tool + * + * This is different from tool confirmation - it's for per-command approval + * of dangerous operations (like rm, git push) within tools that are already approved. + * + * TODO: Make sessionId required once all callers are updated to pass it + * Command confirmations always happen during tool execution which has session context + * + * @example + * ```typescript + * // bash_exec tool is approved, but dangerous commands still require approval + * const response = await manager.requestCommandConfirmation({ + * toolName: 'bash_exec', + * command: 'rm -rf /important', + * originalCommand: 'rm -rf /important', + * sessionId: 'session-123' + * }); + * ``` + */ + async requestCommandConfirmation( + metadata: CommandConfirmationMetadata & { sessionId?: string; timeout?: number } + ): Promise { + const { sessionId, timeout, ...commandMetadata } = metadata; + + const details: ApprovalRequestDetails = { + type: ApprovalType.COMMAND_CONFIRMATION, + // Use provided timeout, fallback to config timeout, or undefined (no timeout) + timeout: timeout !== undefined ? timeout : this.config.toolConfirmation.timeout, + metadata: commandMetadata, + }; + + if (sessionId !== undefined) { + details.sessionId = sessionId; + } + + return this.requestApproval(details); + } + + /** + * Request elicitation from MCP server + * Convenience method for MCP elicitation requests + * + * Note: sessionId is optional because MCP servers are shared across sessions + * and the MCP protocol doesn't include session context in elicitation requests. + */ + async requestElicitation( + metadata: ElicitationMetadata & { sessionId?: string; timeout?: number } + ): Promise { + const { sessionId, timeout, ...elicitationMetadata } = metadata; + + const details: ApprovalRequestDetails = { + type: ApprovalType.ELICITATION, + // Use provided timeout, fallback to config timeout, or undefined (no timeout) + timeout: timeout !== undefined ? timeout : this.config.elicitation.timeout, + metadata: elicitationMetadata, + }; + + if (sessionId !== undefined) { + details.sessionId = sessionId; + } + + return this.requestApproval(details); + } + + /** + * Check if tool confirmation was approved + * Throws appropriate error if denied + */ + async checkToolConfirmation( + metadata: ToolConfirmationMetadata & { sessionId?: string; timeout?: number } + ): Promise { + const response = await this.requestToolConfirmation(metadata); + + if (response.status === ApprovalStatus.APPROVED) { + return true; + } else if (response.status === ApprovalStatus.DENIED) { + throw ApprovalError.toolConfirmationDenied( + metadata.toolName, + response.reason, + response.message, + metadata.sessionId + ); + } else { + throw ApprovalError.cancelled( + response.approvalId, + ApprovalType.TOOL_CONFIRMATION, + response.message ?? response.reason + ); + } + } + + /** + * Get elicitation form data + * Throws appropriate error if denied or cancelled + */ + async getElicitationData( + metadata: ElicitationMetadata & { sessionId?: string; timeout?: number } + ): Promise> { + const response = await this.requestElicitation(metadata); + + if (response.status === ApprovalStatus.APPROVED) { + // Extract formData from response (handler always provides formData for elicitation) + if ( + response.data && + typeof response.data === 'object' && + 'formData' in response.data && + typeof (response.data as { formData: unknown }).formData === 'object' && + (response.data as { formData: unknown }).formData !== null + ) { + return (response.data as { formData: Record }).formData; + } + // Fallback to empty form if data is missing (edge case) + return {}; + } else if (response.status === ApprovalStatus.DENIED) { + throw ApprovalError.elicitationDenied( + metadata.serverName, + response.reason, + response.message, + metadata.sessionId + ); + } else { + throw ApprovalError.cancelled( + response.approvalId, + ApprovalType.ELICITATION, + response.message ?? response.reason + ); + } + } + + /** + * Cancel a specific approval request + */ + cancelApproval(approvalId: string): void { + this.handler?.cancel?.(approvalId); + } + + /** + * Cancel all pending approval requests + */ + cancelAllApprovals(): void { + this.handler?.cancelAll?.(); + } + + /** + * Get list of pending approval IDs + */ + getPendingApprovals(): string[] { + return this.handler?.getPending?.() ?? []; + } + + /** + * Get full pending approval requests + */ + getPendingApprovalRequests(): ApprovalRequest[] { + return this.handler?.getPendingRequests?.() ?? []; + } + + /** + * Auto-approve pending requests that match a predicate. + * Used when a pattern is remembered to auto-approve other parallel requests + * that would now match the same pattern. + * + * @param predicate Function that returns true for requests that should be auto-approved + * @param responseData Optional data to include in the auto-approval response + * @returns Number of requests that were auto-approved + */ + autoApprovePendingRequests( + predicate: (request: ApprovalRequest) => boolean, + responseData?: Record + ): number { + const count = this.handler?.autoApprovePending?.(predicate, responseData) ?? 0; + if (count > 0) { + this.logger.info(`Auto-approved ${count} pending request(s) due to matching pattern`); + } + return count; + } + + /** + * Get current configuration + */ + getConfig(): ApprovalManagerConfig { + return { ...this.config }; + } + + /** + * Set the approval handler for manual approval mode. + * + * The handler will be called for: + * - Tool confirmation requests when toolConfirmation.mode is 'manual' + * - All elicitation requests (when elicitation is enabled, regardless of toolConfirmation.mode) + * + * A handler must be set before processing requests if: + * - toolConfirmation.mode is 'manual', or + * - elicitation is enabled (elicitation.enabled is true) + * + * @param handler The approval handler function, or null to clear + */ + setHandler(handler: ApprovalHandler | null): void { + if (handler === null) { + this.handler = undefined; + } else { + this.handler = handler; + } + this.logger.debug(`Approval handler ${handler ? 'registered' : 'cleared'}`); + } + + /** + * Clear the current approval handler + */ + clearHandler(): void { + this.handler = undefined; + this.logger.debug('Approval handler cleared'); + } + + /** + * Check if an approval handler is registered + */ + public hasHandler(): boolean { + return this.handler !== undefined; + } + + /** + * Get the approval handler, throwing if not set + * @private + */ + private ensureHandler(): ApprovalHandler { + if (!this.handler) { + // TODO: add an example for usage here for users + throw ApprovalError.invalidConfig( + 'An approval handler is required but not configured.\n' + + 'Handlers are required for:\n' + + ' • manual tool confirmation mode\n' + + ' • all elicitation requests (when elicitation is enabled)\n' + + 'Either:\n' + + ' • set toolConfirmation.mode to "auto-approve" or "auto-deny", or\n' + + ' • disable elicitation (set elicitation.enabled: false), or\n' + + ' • call agent.setApprovalHandler(...) before processing requests.' + ); + } + return this.handler; + } +} diff --git a/dexto/packages/core/src/approval/schemas.ts b/dexto/packages/core/src/approval/schemas.ts new file mode 100644 index 00000000..6aabba35 --- /dev/null +++ b/dexto/packages/core/src/approval/schemas.ts @@ -0,0 +1,386 @@ +// ============================================================================ +// USER APPROVAL SCHEMAS - Zod validation schemas for approval requests/responses +// ============================================================================ + +import { z } from 'zod'; +import type { JSONSchema7 } from 'json-schema'; +import { ApprovalType, ApprovalStatus, DenialReason } from './types.js'; +import type { ToolDisplayData } from '../tools/display-types.js'; +import { isValidDisplayData } from '../tools/display-types.js'; + +// Zod schema that validates as object but types as JSONSchema7 +const JsonSchema7Schema = z.record(z.unknown()) as z.ZodType; + +/** + * Schema for approval types + */ +export const ApprovalTypeSchema = z.nativeEnum(ApprovalType); + +/** + * Schema for approval status + */ +export const ApprovalStatusSchema = z.nativeEnum(ApprovalStatus); + +/** + * Schema for denial/cancellation reasons + */ +export const DenialReasonSchema = z.nativeEnum(DenialReason); + +// Custom Zod schema for ToolDisplayData validation +const ToolDisplayDataSchema = z.custom((val) => isValidDisplayData(val), { + message: 'Invalid ToolDisplayData', +}); + +/** + * Tool confirmation metadata schema + */ +export const ToolConfirmationMetadataSchema = z + .object({ + toolName: z.string().describe('Name of the tool to confirm'), + toolCallId: z.string().describe('Unique tool call ID for tracking parallel tool calls'), + args: z.record(z.unknown()).describe('Arguments for the tool'), + description: z.string().optional().describe('Description of the tool'), + displayPreview: ToolDisplayDataSchema.optional().describe( + 'Preview display data for approval UI (e.g., diff preview)' + ), + suggestedPatterns: z + .array(z.string()) + .optional() + .describe( + 'Suggested patterns for session approval (for bash commands). ' + + 'E.g., ["git push *", "git *"] for command "git push origin main"' + ), + }) + .strict() + .describe('Tool confirmation metadata'); + +/** + * Command confirmation metadata schema + * TODO: Consider combining this with regular tools schemas for consistency + */ +export const CommandConfirmationMetadataSchema = z + .object({ + toolName: z.string().describe('Name of the tool executing the command'), + command: z.string().describe('The normalized command to execute'), + originalCommand: z + .string() + .optional() + .describe('The original command before normalization'), + }) + .strict() + .describe('Command confirmation metadata'); + +/** + * Elicitation metadata schema + */ +export const ElicitationMetadataSchema = z + .object({ + schema: JsonSchema7Schema.describe('JSON Schema for the form'), + prompt: z.string().describe('Prompt to show the user'), + serverName: z.string().describe('MCP server requesting input'), + context: z.record(z.unknown()).optional().describe('Additional context'), + }) + .strict() + .describe('Elicitation metadata'); + +/** + * Custom approval metadata schema - flexible + */ +export const CustomApprovalMetadataSchema = z.record(z.unknown()).describe('Custom metadata'); + +/** + * Directory access metadata schema + * Used when a tool tries to access files outside the working directory + */ +export const DirectoryAccessMetadataSchema = z + .object({ + path: z.string().describe('Full path being accessed'), + parentDir: z.string().describe('Parent directory (what gets approved for session)'), + operation: z.enum(['read', 'write', 'edit']).describe('Type of file operation'), + toolName: z.string().describe('Name of the tool requesting access'), + }) + .strict() + .describe('Directory access metadata'); + +/** + * Base approval request schema + */ +export const BaseApprovalRequestSchema = z + .object({ + approvalId: z.string().uuid().describe('Unique approval identifier'), + type: ApprovalTypeSchema.describe('Type of approval'), + sessionId: z.string().optional().describe('Session identifier'), + timeout: z + .number() + .int() + .positive() + .optional() + .describe('Timeout in milliseconds (optional - no timeout if not specified)'), + timestamp: z.date().describe('When the request was created'), + }) + .describe('Base approval request'); + +/** + * Tool confirmation request schema + */ +export const ToolConfirmationRequestSchema = BaseApprovalRequestSchema.extend({ + type: z.literal(ApprovalType.TOOL_CONFIRMATION), + metadata: ToolConfirmationMetadataSchema, +}).strict(); + +/** + * Command confirmation request schema + */ +export const CommandConfirmationRequestSchema = BaseApprovalRequestSchema.extend({ + type: z.literal(ApprovalType.COMMAND_CONFIRMATION), + metadata: CommandConfirmationMetadataSchema, +}).strict(); + +/** + * Elicitation request schema + */ +export const ElicitationRequestSchema = BaseApprovalRequestSchema.extend({ + type: z.literal(ApprovalType.ELICITATION), + metadata: ElicitationMetadataSchema, +}).strict(); + +/** + * Custom approval request schema + */ +export const CustomApprovalRequestSchema = BaseApprovalRequestSchema.extend({ + type: z.literal(ApprovalType.CUSTOM), + metadata: CustomApprovalMetadataSchema, +}).strict(); + +/** + * Directory access request schema + */ +export const DirectoryAccessRequestSchema = BaseApprovalRequestSchema.extend({ + type: z.literal(ApprovalType.DIRECTORY_ACCESS), + metadata: DirectoryAccessMetadataSchema, +}).strict(); + +/** + * Discriminated union for all approval requests + */ +export const ApprovalRequestSchema = z.discriminatedUnion('type', [ + ToolConfirmationRequestSchema, + CommandConfirmationRequestSchema, + ElicitationRequestSchema, + CustomApprovalRequestSchema, + DirectoryAccessRequestSchema, +]); + +/** + * Tool confirmation response data schema + */ +export const ToolConfirmationResponseDataSchema = z + .object({ + rememberChoice: z + .boolean() + .optional() + .describe('Remember this tool for the session (approves ALL uses of this tool)'), + rememberPattern: z + .string() + .optional() + .describe( + 'Remember a command pattern for bash commands (e.g., "git *"). ' + + 'Only applicable for bash_exec tool approvals.' + ), + }) + .strict() + .describe('Tool confirmation response data'); + +/** + * Command confirmation response data schema + */ +export const CommandConfirmationResponseDataSchema = z + .object({ + // Command confirmations don't have remember choice - they're per-command + // Could add command pattern remembering in future (e.g., "remember git push *") + }) + .strict() + .describe('Command confirmation response data'); + +/** + * Elicitation response data schema + */ +export const ElicitationResponseDataSchema = z + .object({ + formData: z.record(z.unknown()).describe('Form data matching schema'), + }) + .strict() + .describe('Elicitation response data'); + +/** + * Custom approval response data schema + */ +export const CustomApprovalResponseDataSchema = z + .record(z.unknown()) + .describe('Custom response data'); + +/** + * Directory access response data schema + */ +export const DirectoryAccessResponseDataSchema = z + .object({ + rememberDirectory: z + .boolean() + .optional() + .describe('Remember this directory for the session (allows all file access within it)'), + }) + .strict() + .describe('Directory access response data'); + +/** + * Base approval response schema + */ +export const BaseApprovalResponseSchema = z + .object({ + approvalId: z.string().uuid().describe('Must match request approvalId'), + status: ApprovalStatusSchema.describe('Approval status'), + sessionId: z.string().optional().describe('Session identifier'), + reason: DenialReasonSchema.optional().describe( + 'Reason for denial/cancellation (only present when status is denied or cancelled)' + ), + message: z + .string() + .optional() + .describe('Human-readable message explaining the denial/cancellation'), + timeoutMs: z + .number() + .int() + .positive() + .optional() + .describe('Timeout duration in milliseconds (present for timeout events)'), + }) + .describe('Base approval response'); + +/** + * Tool confirmation response schema + */ +export const ToolConfirmationResponseSchema = BaseApprovalResponseSchema.extend({ + data: ToolConfirmationResponseDataSchema.optional(), +}).strict(); + +/** + * Command confirmation response schema + */ +export const CommandConfirmationResponseSchema = BaseApprovalResponseSchema.extend({ + data: CommandConfirmationResponseDataSchema.optional(), +}).strict(); + +/** + * Elicitation response schema + */ +export const ElicitationResponseSchema = BaseApprovalResponseSchema.extend({ + data: ElicitationResponseDataSchema.optional(), +}).strict(); + +/** + * Custom approval response schema + */ +export const CustomApprovalResponseSchema = BaseApprovalResponseSchema.extend({ + data: CustomApprovalResponseDataSchema.optional(), +}).strict(); + +/** + * Directory access response schema + */ +export const DirectoryAccessResponseSchema = BaseApprovalResponseSchema.extend({ + data: DirectoryAccessResponseDataSchema.optional(), +}).strict(); + +/** + * Union of all approval responses + */ +export const ApprovalResponseSchema = z.union([ + ToolConfirmationResponseSchema, + CommandConfirmationResponseSchema, + ElicitationResponseSchema, + CustomApprovalResponseSchema, + DirectoryAccessResponseSchema, +]); + +/** + * Approval request details schema for creating requests + */ +export const ApprovalRequestDetailsSchema = z + .object({ + type: ApprovalTypeSchema, + sessionId: z.string().optional(), + timeout: z + .number() + .int() + .positive() + .optional() + .describe('Timeout in milliseconds (optional - no timeout if not specified)'), + metadata: z.union([ + ToolConfirmationMetadataSchema, + CommandConfirmationMetadataSchema, + ElicitationMetadataSchema, + CustomApprovalMetadataSchema, + DirectoryAccessMetadataSchema, + ]), + }) + .superRefine((data, ctx) => { + // Validate metadata matches type + if (data.type === ApprovalType.TOOL_CONFIRMATION) { + const result = ToolConfirmationMetadataSchema.safeParse(data.metadata); + if (!result.success) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + 'Metadata must match ToolConfirmationMetadataSchema for TOOL_CONFIRMATION type', + path: ['metadata'], + }); + } + } else if (data.type === ApprovalType.COMMAND_CONFIRMATION) { + const result = CommandConfirmationMetadataSchema.safeParse(data.metadata); + if (!result.success) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + 'Metadata must match CommandConfirmationMetadataSchema for COMMAND_CONFIRMATION type', + path: ['metadata'], + }); + } + } else if (data.type === ApprovalType.ELICITATION) { + const result = ElicitationMetadataSchema.safeParse(data.metadata); + if (!result.success) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Metadata must match ElicitationMetadataSchema for ELICITATION type', + path: ['metadata'], + }); + } + } else if (data.type === ApprovalType.DIRECTORY_ACCESS) { + const result = DirectoryAccessMetadataSchema.safeParse(data.metadata); + if (!result.success) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + 'Metadata must match DirectoryAccessMetadataSchema for DIRECTORY_ACCESS type', + path: ['metadata'], + }); + } + } else if (data.type === ApprovalType.CUSTOM) { + const result = CustomApprovalMetadataSchema.safeParse(data.metadata); + if (!result.success) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Metadata must match CustomApprovalMetadataSchema for CUSTOM type', + path: ['metadata'], + }); + } + } + }); + +/** + * Type inference for validated schemas + */ +export type ValidatedApprovalRequest = z.output; +export type ValidatedApprovalResponse = z.output; +export type ValidatedToolConfirmationRequest = z.output; +export type ValidatedElicitationRequest = z.output; +export type ValidatedCustomApprovalRequest = z.output; diff --git a/dexto/packages/core/src/approval/types.ts b/dexto/packages/core/src/approval/types.ts new file mode 100644 index 00000000..54ba0f09 --- /dev/null +++ b/dexto/packages/core/src/approval/types.ts @@ -0,0 +1,352 @@ +// ============================================================================ +// USER APPROVAL TYPES - Generalized approval and user input system +// ============================================================================ + +import type { z } from 'zod'; +import type { + ToolConfirmationMetadataSchema, + CommandConfirmationMetadataSchema, + ElicitationMetadataSchema, + CustomApprovalMetadataSchema, + DirectoryAccessMetadataSchema, + BaseApprovalRequestSchema, + ToolConfirmationRequestSchema, + CommandConfirmationRequestSchema, + ElicitationRequestSchema, + CustomApprovalRequestSchema, + DirectoryAccessRequestSchema, + ApprovalRequestSchema, + ApprovalRequestDetailsSchema, + ToolConfirmationResponseDataSchema, + CommandConfirmationResponseDataSchema, + ElicitationResponseDataSchema, + CustomApprovalResponseDataSchema, + DirectoryAccessResponseDataSchema, + BaseApprovalResponseSchema, + ToolConfirmationResponseSchema, + CommandConfirmationResponseSchema, + ElicitationResponseSchema, + CustomApprovalResponseSchema, + DirectoryAccessResponseSchema, + ApprovalResponseSchema, +} from './schemas.js'; + +/** + * Types of approval requests supported by the system + */ +export enum ApprovalType { + /** + * Binary approval for tool execution + * Metadata contains: toolName, args, description + */ + TOOL_CONFIRMATION = 'tool_confirmation', + + /** + * Binary approval for dangerous commands within an already-approved tool + * Metadata contains: toolName, command, originalCommand + * (sessionId is provided at the request level, not in metadata) + */ + COMMAND_CONFIRMATION = 'command_confirmation', + + /** + * Schema-based form input from MCP servers + * Metadata contains: schema, prompt, serverName, context + */ + ELICITATION = 'elicitation', + + /** + * Approval for accessing files outside the working directory + * Metadata contains: path, parentDir, operation, toolName + */ + DIRECTORY_ACCESS = 'directory_access', + + /** + * Custom approval types for extensibility + * Metadata format defined by consumer + */ + CUSTOM = 'custom', +} + +/** + * Status of an approval response + */ +export enum ApprovalStatus { + APPROVED = 'approved', + DENIED = 'denied', + CANCELLED = 'cancelled', +} + +/** + * Reason for denial or cancellation + * Provides context about why an approval was not granted + */ +export enum DenialReason { + /** User explicitly clicked deny/reject */ + USER_DENIED = 'user_denied', + /** System denied due to policy (auto-deny mode, alwaysDeny list) */ + SYSTEM_DENIED = 'system_denied', + /** Request timed out waiting for user response */ + TIMEOUT = 'timeout', + /** User cancelled the request */ + USER_CANCELLED = 'user_cancelled', + /** System cancelled (session ended, agent stopped) */ + SYSTEM_CANCELLED = 'system_cancelled', + /** Validation failed (form validation, schema mismatch) */ + VALIDATION_FAILED = 'validation_failed', + /** Elicitation disabled in configuration */ + ELICITATION_DISABLED = 'elicitation_disabled', +} + +// ============================================================================ +// Metadata Types - Derived from Zod schemas +// ============================================================================ + +/** + * Tool confirmation specific metadata + * Derived from ToolConfirmationMetadataSchema + */ +export type ToolConfirmationMetadata = z.output; + +/** + * Command confirmation specific metadata + * Derived from CommandConfirmationMetadataSchema + */ +export type CommandConfirmationMetadata = z.output; + +/** + * Elicitation specific metadata (MCP) + * Derived from ElicitationMetadataSchema + */ +export type ElicitationMetadata = z.output; + +/** + * Custom approval metadata - flexible structure + * Derived from CustomApprovalMetadataSchema + */ +export type CustomApprovalMetadata = z.output; + +/** + * Directory access metadata + * Derived from DirectoryAccessMetadataSchema + */ +export type DirectoryAccessMetadata = z.output; + +// ============================================================================ +// Request Types - Derived from Zod schemas +// ============================================================================ + +/** + * Base approval request that all approvals extend + * Derived from BaseApprovalRequestSchema + */ +export type BaseApprovalRequest<_TMetadata = unknown> = z.output; + +/** + * Tool confirmation request + * Derived from ToolConfirmationRequestSchema + */ +export type ToolConfirmationRequest = z.output; + +/** + * Command confirmation request + * Derived from CommandConfirmationRequestSchema + */ +export type CommandConfirmationRequest = z.output; + +/** + * Elicitation request from MCP server + * Derived from ElicitationRequestSchema + */ +export type ElicitationRequest = z.output; + +/** + * Custom approval request + * Derived from CustomApprovalRequestSchema + */ +export type CustomApprovalRequest = z.output; + +/** + * Directory access request + * Derived from DirectoryAccessRequestSchema + */ +export type DirectoryAccessRequest = z.output; + +/** + * Union of all approval request types + * Derived from ApprovalRequestSchema + */ +export type ApprovalRequest = z.output; + +// ============================================================================ +// Response Data Types - Derived from Zod schemas +// ============================================================================ + +/** + * Tool confirmation response data + * Derived from ToolConfirmationResponseDataSchema + */ +export type ToolConfirmationResponseData = z.output; + +/** + * Command confirmation response data + * Derived from CommandConfirmationResponseDataSchema + */ +export type CommandConfirmationResponseData = z.output< + typeof CommandConfirmationResponseDataSchema +>; + +/** + * Elicitation response data - validated form inputs + * Derived from ElicitationResponseDataSchema + */ +export type ElicitationResponseData = z.output; + +/** + * Custom approval response data + * Derived from CustomApprovalResponseDataSchema + */ +export type CustomApprovalResponseData = z.output; + +/** + * Directory access response data + * Derived from DirectoryAccessResponseDataSchema + */ +export type DirectoryAccessResponseData = z.output; + +// ============================================================================ +// Response Types - Derived from Zod schemas +// ============================================================================ + +/** + * Base approval response + * Derived from BaseApprovalResponseSchema + */ +export type BaseApprovalResponse<_TData = unknown> = z.output; + +/** + * Tool confirmation response + * Derived from ToolConfirmationResponseSchema + */ +export type ToolConfirmationResponse = z.output; + +/** + * Command confirmation response + * Derived from CommandConfirmationResponseSchema + */ +export type CommandConfirmationResponse = z.output; + +/** + * Elicitation response + * Derived from ElicitationResponseSchema + */ +export type ElicitationResponse = z.output; + +/** + * Custom approval response + * Derived from CustomApprovalResponseSchema + */ +export type CustomApprovalResponse = z.output; + +/** + * Directory access response + * Derived from DirectoryAccessResponseSchema + */ +export type DirectoryAccessResponse = z.output; + +/** + * Union of all approval response types + * Derived from ApprovalResponseSchema + */ +export type ApprovalResponse = z.output; + +// ============================================================================ +// Helper Types +// ============================================================================ + +/** + * Details for creating an approval request + * Derived from ApprovalRequestDetailsSchema + */ +export type ApprovalRequestDetails = z.output; + +/** + * Handler interface for processing approval requests. + * + * This is the core abstraction for approval handling in Dexto. When tool confirmation + * mode is 'manual', a handler must be provided to process approval requests. + * + * The handler is a callable interface that: + * - Processes approval requests and returns responses + * - Manages pending approval state (for cancellation) + * - Provides lifecycle management methods + * + * @example + * ```typescript + * const handler: ApprovalHandler = Object.assign( + * async (request: ApprovalRequest) => { + * console.log(`Approve tool: ${request.metadata.toolName}?`); + * // In real implementation, wait for user input + * return { + * approvalId: request.approvalId, + * status: ApprovalStatus.APPROVED, + * sessionId: request.sessionId, + * }; + * }, + * { + * cancel: (id: string) => { }, + * cancelAll: () => { }, + * getPending: () => [] as string[], + * } + * ); + * ``` + */ +export interface ApprovalHandler { + /** + * Process an approval request + * @param request The approval request to handle + * @returns Promise resolving to the approval response + */ + (request: ApprovalRequest): Promise; + + /** + * Cancel a specific pending approval request (optional) + * @param approvalId The ID of the approval to cancel + * @remarks Not all handlers support cancellation (e.g., auto-approve handlers) + */ + cancel?(approvalId: string): void; + + /** + * Cancel all pending approval requests (optional) + * @remarks Not all handlers support cancellation (e.g., auto-approve handlers) + */ + cancelAll?(): void; + + /** + * Get list of pending approval request IDs (optional) + * @returns Array of approval IDs currently pending + * @remarks Not all handlers track pending requests (e.g., auto-approve handlers) + */ + getPending?(): string[]; + + /** + * Get full pending approval requests (optional) + * @returns Array of pending approval requests + * @remarks Not all handlers track pending requests (e.g., auto-approve handlers) + */ + getPendingRequests?(): ApprovalRequest[]; + + /** + * Auto-approve pending requests that match a predicate (optional) + * Used when a pattern is remembered to auto-approve other parallel requests + * that would now match the same pattern. + * + * @param predicate Function that returns true for requests that should be auto-approved + * @param responseData Optional data to include in the auto-approval response + * @returns Number of requests that were auto-approved + * @remarks Not all handlers support this (e.g., auto-approve handlers don't need it) + */ + autoApprovePending?( + predicate: (request: ApprovalRequest) => boolean, + responseData?: Record + ): number; +} diff --git a/dexto/packages/core/src/context/compaction/compaction.integration.test.ts b/dexto/packages/core/src/context/compaction/compaction.integration.test.ts new file mode 100644 index 00000000..4d486990 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/compaction.integration.test.ts @@ -0,0 +1,405 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { ContextManager } from '../manager.js'; +import { filterCompacted } from '../utils.js'; +import { ReactiveOverflowStrategy } from './strategies/reactive-overflow.js'; +import { VercelMessageFormatter } from '../../llm/formatters/vercel.js'; +import { SystemPromptManager } from '../../systemPrompt/manager.js'; +import { SystemPromptConfigSchema } from '../../systemPrompt/schemas.js'; +import { MemoryHistoryProvider } from '../../session/history/memory.js'; +import { ResourceManager } from '../../resources/index.js'; +import { MCPManager } from '../../mcp/manager.js'; +import { MemoryManager } from '../../memory/index.js'; +import { createStorageManager, StorageManager } from '../../storage/storage-manager.js'; +import { createLogger } from '../../logger/factory.js'; +import type { ModelMessage } from 'ai'; +import type { LanguageModel } from 'ai'; +import type { ValidatedLLMConfig } from '../../llm/schemas.js'; +import type { ValidatedStorageConfig } from '../../storage/schemas.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import type { InternalMessage } from '../types.js'; + +// Only mock the AI SDK's generateText - everything else is real +vi.mock('ai', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + generateText: vi.fn(), + }; +}); + +import { generateText } from 'ai'; + +const mockGenerateText = vi.mocked(generateText); + +function createMockModel(): LanguageModel { + return { + modelId: 'test-model', + provider: 'test-provider', + specificationVersion: 'v1', + doStream: vi.fn(), + doGenerate: vi.fn(), + } as unknown as LanguageModel; +} + +/** + * Integration tests for context compaction. + * + * These tests use real components (ContextManager, ReactiveOverflowStrategy, filterCompacted) + * and only mock the LLM calls. This ensures the full compaction flow works correctly, + * including the interaction between compaction and filterCompacted. + */ +describe('Context Compaction Integration Tests', () => { + let contextManager: ContextManager; + let compactionStrategy: ReactiveOverflowStrategy; + let logger: IDextoLogger; + let historyProvider: MemoryHistoryProvider; + let storageManager: StorageManager; + let mcpManager: MCPManager; + let resourceManager: ResourceManager; + + const sessionId = 'compaction-test-session'; + + beforeEach(async () => { + vi.clearAllMocks(); + + // Create real logger (quiet for tests) + logger = createLogger({ + config: { + level: 'warn', + transports: [{ type: 'console', colorize: false }], + }, + agentId: 'test-agent', + }); + + // Create real storage manager with in-memory backends + const storageConfig = { + cache: { type: 'in-memory' }, + database: { type: 'in-memory' }, + blob: { + type: 'in-memory', + maxBlobSize: 10 * 1024 * 1024, + maxTotalSize: 100 * 1024 * 1024, + }, + } as unknown as ValidatedStorageConfig; + storageManager = await createStorageManager(storageConfig, logger); + + // Create real MCP and resource managers + mcpManager = new MCPManager(logger); + resourceManager = new ResourceManager( + mcpManager, + { + internalResourcesConfig: { enabled: false, resources: [] }, + blobStore: storageManager.getBlobStore(), + }, + logger + ); + await resourceManager.initialize(); + + // Create real history provider + historyProvider = new MemoryHistoryProvider(logger); + + // Create real memory and system prompt managers + const memoryManager = new MemoryManager(storageManager.getDatabase(), logger); + const systemPromptConfig = SystemPromptConfigSchema.parse('You are a helpful assistant.'); + const systemPromptManager = new SystemPromptManager( + systemPromptConfig, + '/tmp', + memoryManager, + undefined, + logger + ); + + // Create real context manager + const formatter = new VercelMessageFormatter(logger); + const llmConfig = { + provider: 'openai', + model: 'gpt-4', + apiKey: 'test-api-key', + maxInputTokens: 100000, + maxOutputTokens: 4096, + } as unknown as ValidatedLLMConfig; + + contextManager = new ContextManager( + llmConfig, + formatter, + systemPromptManager, + 100000, + historyProvider, + sessionId, + resourceManager, + logger + ); + + // Create real compaction strategy + compactionStrategy = new ReactiveOverflowStrategy(createMockModel(), {}, logger); + + // Default mock for generateText (compaction summary) + mockGenerateText.mockResolvedValue({ + text: 'Summary of conversation', + } as Awaited>); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + logger.destroy(); + }); + + /** + * Helper to add a batch of messages to the context + */ + async function addMessages(count: number): Promise { + for (let i = 0; i < count; i++) { + await contextManager.addUserMessage([{ type: 'text', text: `Question ${i}` }]); + await contextManager.addAssistantMessage(`Answer ${i}`); + } + } + + /** + * Helper to run compaction and add result to history + */ + async function runCompaction(): Promise { + const history = await contextManager.getHistory(); + const summaryMessages = await compactionStrategy.compact(history); + + if (summaryMessages.length === 0) { + return null; + } + + const summary = summaryMessages[0]!; + await contextManager.addMessage(summary); + return summary; + } + + describe('Single Compaction', () => { + it('should compact history and filterCompacted should return correct messages', async () => { + // Add 20 messages (10 turns) + await addMessages(10); + + const historyBefore = await contextManager.getHistory(); + expect(historyBefore).toHaveLength(20); + + // Run compaction + const summary = await runCompaction(); + expect(summary).not.toBeNull(); + expect(summary?.metadata?.isSummary).toBe(true); + + // Verify history grew by 1 (summary added) + const historyAfter = await contextManager.getHistory(); + expect(historyAfter).toHaveLength(21); + + // filterCompacted should return much fewer messages + const filtered = filterCompacted(historyAfter); + expect(filtered.length).toBeLessThan(historyAfter.length); + expect(filtered[0]?.metadata?.isSummary).toBe(true); + + // Preserved messages should be non-summary messages + const nonSummaryMessages = filtered.filter((m) => !m.metadata?.isSummary); + expect(nonSummaryMessages.length).toBeGreaterThan(0); + expect(nonSummaryMessages.length).toBeLessThan(10); // Some were summarized + }); + }); + + describe('Multiple Sequential Compactions', () => { + it('should handle two compactions correctly', async () => { + // === FIRST COMPACTION === + await addMessages(10); + const summary1 = await runCompaction(); + expect(summary1).not.toBeNull(); + expect(summary1?.metadata?.isRecompaction).toBeUndefined(); + + const historyAfter1 = await contextManager.getHistory(); + // Verify first compaction produced fewer filtered messages + const filtered1 = filterCompacted(historyAfter1); + expect(filtered1.length).toBeLessThan(historyAfter1.length); + + // === ADD MORE MESSAGES === + await addMessages(10); + + const historyBefore2 = await contextManager.getHistory(); + // 21 (after first compaction) + 20 new = 41 + expect(historyBefore2).toHaveLength(41); + + // === SECOND COMPACTION === + const summary2 = await runCompaction(); + expect(summary2).not.toBeNull(); + expect(summary2?.metadata?.isRecompaction).toBe(true); + + const historyAfter2 = await contextManager.getHistory(); + expect(historyAfter2).toHaveLength(42); + + const filtered2 = filterCompacted(historyAfter2); + + // Critical: second compaction should result in FEWER filtered messages + // (or at least not significantly more) + expect(filtered2.length).toBeLessThan(30); + + // Only the most recent summary should be in filtered result + const summariesInFiltered = filtered2.filter((m) => m.metadata?.isSummary); + expect(summariesInFiltered).toHaveLength(1); + expect(summariesInFiltered[0]?.metadata?.isRecompaction).toBe(true); + + // The first summary should NOT be in filtered result + expect(filtered2).not.toContain(summary1); + }); + + it('should handle three compactions correctly', async () => { + // === FIRST COMPACTION === + await addMessages(10); + const summary1 = await runCompaction(); + expect(summary1).not.toBeNull(); + + // === SECOND COMPACTION === + await addMessages(10); + const summary2 = await runCompaction(); + expect(summary2).not.toBeNull(); + expect(summary2?.metadata?.isRecompaction).toBe(true); + + // === THIRD COMPACTION === + await addMessages(10); + const summary3 = await runCompaction(); + expect(summary3).not.toBeNull(); + expect(summary3?.metadata?.isRecompaction).toBe(true); + + // Verify final state + const historyFinal = await contextManager.getHistory(); + // 20 + 1 + 20 + 1 + 20 + 1 = 63 + expect(historyFinal).toHaveLength(63); + + const filteredFinal = filterCompacted(historyFinal); + + // Critical assertions: + // 1. Only the most recent summary should be visible + const summariesInFiltered = filteredFinal.filter((m) => m.metadata?.isSummary); + expect(summariesInFiltered).toHaveLength(1); + expect(summariesInFiltered[0]).toBe(summary3); + + // 2. Neither summary1 nor summary2 should be in the result + expect(filteredFinal).not.toContain(summary1); + expect(filteredFinal).not.toContain(summary2); + + // 3. Filtered result should be much smaller than full history + expect(filteredFinal.length).toBeLessThan(20); + + // 4. Preserved messages should exist and be reasonable count + const nonSummaryMessages = filteredFinal.filter((m) => !m.metadata?.isSummary); + expect(nonSummaryMessages.length).toBeGreaterThan(0); + expect(nonSummaryMessages.length).toBeLessThan(15); + }); + + it('should correctly calculate originalMessageCount for each compaction', async () => { + // === FIRST COMPACTION === + await addMessages(10); + const summary1 = await runCompaction(); + expect(summary1).not.toBeNull(); + + // First compaction: originalMessageCount should be the number of summarized messages + const originalCount1 = summary1?.metadata?.originalMessageCount; + expect(typeof originalCount1).toBe('number'); + expect(originalCount1).toBeLessThan(20); // Less than total, some were preserved + + // === SECOND COMPACTION === + await addMessages(10); + const historyBefore2 = await contextManager.getHistory(); + const summary1Index = historyBefore2.findIndex((m) => m === summary1); + + const summary2 = await runCompaction(); + expect(summary2).not.toBeNull(); + + // Second compaction: originalMessageCount should be ABSOLUTE + // It should be > summary1Index (pointing past the first summary) + const originalCount2 = summary2?.metadata?.originalMessageCount; + expect(typeof originalCount2).toBe('number'); + expect(originalCount2).toBeGreaterThan(summary1Index); + + // Verify filterCompacted works with this absolute count + const historyAfter2 = await contextManager.getHistory(); + const filtered2 = filterCompacted(historyAfter2); + + // The filtered result should NOT include summary1 + expect(filtered2).not.toContain(summary1); + // Preserved messages should exist + const preserved = filtered2.filter((m) => !m.metadata?.isSummary); + expect(preserved.length).toBeGreaterThan(0); + }); + }); + + describe('Edge Cases', () => { + it('should not compact if history is too short', async () => { + await addMessages(1); // Only 2 messages + + const summary = await runCompaction(); + expect(summary).toBeNull(); + }); + + it('should not re-compact if few messages after existing summary', async () => { + // First compaction + await addMessages(10); + await runCompaction(); + + // Add only 2 messages (4 messages = 2 turns, below threshold) + await addMessages(2); + + // Should skip re-compaction + const summary2 = await runCompaction(); + expect(summary2).toBeNull(); + }); + + it('should handle compaction through prepareHistory flow', async () => { + // This tests the real integration with ContextManager.prepareHistory() + // which is what's used when formatting messages for LLM + + await addMessages(10); + await runCompaction(); + await addMessages(10); + await runCompaction(); + + // prepareHistory uses filterCompacted internally + const { preparedHistory, stats } = await contextManager.prepareHistory(); + + // Stats should reflect the filtered counts + expect(stats.filteredCount).toBeLessThan(stats.originalCount); + + // preparedHistory should only contain filtered messages + const summaries = preparedHistory.filter((m) => m.metadata?.isSummary); + expect(summaries).toHaveLength(1); + }); + }); + + describe('Token Estimation After Compaction', () => { + it('should provide accurate token estimates after compaction', async () => { + await addMessages(10); + + // Get estimate before compaction + const estimateBefore = await contextManager.getContextTokenEstimate({ mcpManager }, {}); + const messagesBefore = estimateBefore.stats.filteredMessageCount; + + // Run compaction + await runCompaction(); + contextManager.resetActualTokenTracking(); + + // Get estimate after compaction + const estimateAfter = await contextManager.getContextTokenEstimate({ mcpManager }, {}); + const messagesAfter = estimateAfter.stats.filteredMessageCount; + + // After compaction, should have fewer messages + expect(messagesAfter).toBeLessThan(messagesBefore); + }); + + it('should maintain consistency between /context and compaction stats', async () => { + await addMessages(10); + await runCompaction(); + await addMessages(10); + await runCompaction(); + + // This is what /context command uses + const estimate = await contextManager.getContextTokenEstimate({ mcpManager }, {}); + + // The filteredMessageCount should match what filterCompacted returns + const history = await contextManager.getHistory(); + const filtered = filterCompacted(history); + + expect(estimate.stats.filteredMessageCount).toBe(filtered.length); + expect(estimate.stats.originalMessageCount).toBe(history.length); + }); + }); +}); diff --git a/dexto/packages/core/src/context/compaction/factory.ts b/dexto/packages/core/src/context/compaction/factory.ts new file mode 100644 index 00000000..23a6611c --- /dev/null +++ b/dexto/packages/core/src/context/compaction/factory.ts @@ -0,0 +1,60 @@ +import { z } from 'zod'; +import type { ICompactionStrategy } from './types.js'; +import type { CompactionContext, CompactionConfig } from './provider.js'; +import { compactionRegistry } from './registry.js'; +import { ContextError } from '../errors.js'; + +/** + * Create a compaction strategy from configuration. + * + * Follows the same pattern as blob storage and tools: + * - Validates provider exists + * - Validates configuration with Zod schema + * - Checks LLM requirements + * - Creates strategy instance + * + * @param config - Compaction configuration from agent config + * @param context - Context with logger and optional LanguageModel + * @returns Strategy instance or null if disabled + */ +export async function createCompactionStrategy( + config: CompactionConfig, + context: CompactionContext +): Promise { + // If disabled, return null + if (config.enabled === false) { + context.logger.info(`Compaction provider '${config.type}' is disabled`); + return null; + } + + // Get provider + const provider = compactionRegistry.get(config.type); + if (!provider) { + const available = compactionRegistry.getTypes(); + throw ContextError.compactionInvalidType(config.type, available); + } + + // Validate configuration + try { + const validatedConfig = provider.configSchema.parse(config); + + // Check if LLM is required but not provided + if (provider.metadata?.requiresLLM && !context.model) { + throw ContextError.compactionMissingLLM(config.type); + } + + // Create strategy instance + const strategy = await provider.create(validatedConfig, context); + + context.logger.info( + `Created compaction strategy: ${provider.metadata?.displayName || config.type}` + ); + + return strategy; + } catch (error) { + if (error instanceof z.ZodError) { + throw ContextError.compactionValidation(config.type, error.errors); + } + throw error; + } +} diff --git a/dexto/packages/core/src/context/compaction/index.ts b/dexto/packages/core/src/context/compaction/index.ts new file mode 100644 index 00000000..cbcb8cf4 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/index.ts @@ -0,0 +1,31 @@ +// Core types and interfaces +export * from './types.js'; +export * from './provider.js'; +export * from './registry.js'; +export * from './factory.js'; +export * from './schemas.js'; + +// Strategies +export * from './strategies/reactive-overflow.js'; +export * from './strategies/noop.js'; + +// Providers +export * from './providers/reactive-overflow-provider.js'; +export * from './providers/noop-provider.js'; + +// Utilities +export * from './overflow.js'; + +// Register built-in providers +import { compactionRegistry } from './registry.js'; +import { reactiveOverflowProvider } from './providers/reactive-overflow-provider.js'; +import { noopProvider } from './providers/noop-provider.js'; + +// Auto-register built-in providers when module is imported +// Guard against duplicate registration when module is imported multiple times +if (!compactionRegistry.has('reactive-overflow')) { + compactionRegistry.register(reactiveOverflowProvider); +} +if (!compactionRegistry.has('noop')) { + compactionRegistry.register(noopProvider); +} diff --git a/dexto/packages/core/src/context/compaction/overflow.test.ts b/dexto/packages/core/src/context/compaction/overflow.test.ts new file mode 100644 index 00000000..8d368bcf --- /dev/null +++ b/dexto/packages/core/src/context/compaction/overflow.test.ts @@ -0,0 +1,234 @@ +import { describe, it, expect } from 'vitest'; +import { isOverflow, getCompactionTarget, type ModelLimits } from './overflow.js'; +import type { TokenUsage } from '../../llm/types.js'; + +describe('isOverflow', () => { + describe('basic overflow detection', () => { + it('should return false when input tokens are well below limit', () => { + const tokens: TokenUsage = { + inputTokens: 50000, + }; + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + const result = isOverflow(tokens, modelLimits); + + expect(result).toBe(false); + }); + + it('should return false when input tokens are just below context window (with 100% threshold)', () => { + const tokens: TokenUsage = { + inputTokens: 199999, + }; + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + // Explicitly use 1.0 threshold to test full capacity boundary + const result = isOverflow(tokens, modelLimits, 1.0); + + expect(result).toBe(false); + }); + + it('should return true when input tokens exceed context window', () => { + const tokens: TokenUsage = { + inputTokens: 200001, + }; + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + const result = isOverflow(tokens, modelLimits); + + expect(result).toBe(true); + }); + + it('should return false when input tokens exactly equal context window (with 100% threshold)', () => { + // Edge case: exactly at the limit should NOT trigger overflow + // (inputTokens > effectiveLimit, not >=) + const tokens: TokenUsage = { + inputTokens: 200000, + }; + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + // Explicitly use 1.0 threshold to test full capacity boundary + const result = isOverflow(tokens, modelLimits, 1.0); + + expect(result).toBe(false); + }); + }); + + describe('handling missing inputTokens', () => { + it('should default to 0 when inputTokens is undefined', () => { + const tokens: TokenUsage = { + // inputTokens is undefined + }; + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + const result = isOverflow(tokens, modelLimits); + + expect(result).toBe(false); + }); + }); + + describe('small context windows', () => { + it('should correctly detect overflow for small context windows', () => { + const tokens: TokenUsage = { + inputTokens: 8193, + }; + const modelLimits: ModelLimits = { + contextWindow: 8192, + }; + + const result = isOverflow(tokens, modelLimits); + + expect(result).toBe(true); + }); + }); + + describe('configurable context window (via maxContextTokens override)', () => { + it('should work with reduced context window from config', () => { + // User configured maxContextTokens: 50000 + // Even though model supports 200K, we treat it as 50K + const tokens: TokenUsage = { + inputTokens: 50001, + }; + // The effective context window passed would be 50000 + const modelLimits: ModelLimits = { + contextWindow: 50000, + }; + + const result = isOverflow(tokens, modelLimits); + + expect(result).toBe(true); + }); + }); + + describe('thresholdPercent parameter', () => { + it('should trigger overflow earlier when thresholdPercent is less than 1.0', () => { + // contextWindow: 200000 + // With threshold 0.9: effectiveLimit = floor(200000 * 0.9) = 180000 + const tokens: TokenUsage = { + inputTokens: 180001, // Just over 90% threshold + }; + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + // Without threshold (or threshold=1.0), this would NOT overflow + expect(isOverflow(tokens, modelLimits, 1.0)).toBe(false); + // With threshold=0.9, this SHOULD overflow + expect(isOverflow(tokens, modelLimits, 0.9)).toBe(true); + }); + + it('should use default threshold of 0.9 when not specified', () => { + const tokens: TokenUsage = { + inputTokens: 180000, // 90% of context window + }; + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + // Default should be same as explicit 0.9 + expect(isOverflow(tokens, modelLimits)).toBe(isOverflow(tokens, modelLimits, 0.9)); + }); + + it('should handle threshold of 0.5 (50%)', () => { + // contextWindow: 200000 + // With threshold 0.5: effectiveLimit = floor(200000 * 0.5) = 100000 + const tokens: TokenUsage = { + inputTokens: 100001, + }; + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + expect(isOverflow(tokens, modelLimits, 0.5)).toBe(true); + expect(isOverflow(tokens, modelLimits, 1.0)).toBe(false); + }); + + it('should floor the effective limit', () => { + // contextWindow: 100, thresholdPercent: 0.9 + // effectiveLimit = floor(100 * 0.9) = 90 + const modelLimits: ModelLimits = { + contextWindow: 100, + }; + + // At exactly 90 tokens, should NOT overflow + expect(isOverflow({ inputTokens: 90 }, modelLimits, 0.9)).toBe(false); + // At 91 tokens, SHOULD overflow + expect(isOverflow({ inputTokens: 91 }, modelLimits, 0.9)).toBe(true); + }); + }); +}); + +describe('getCompactionTarget', () => { + describe('default target percentage (70%)', () => { + it('should return 70% of context window by default', () => { + // target = floor(200000 * 0.7) = 140000 + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + const target = getCompactionTarget(modelLimits); + + expect(target).toBe(140000); + }); + }); + + describe('custom target percentage', () => { + it('should return correct target for 50% percentage', () => { + // target = floor(200000 * 0.5) = 100000 + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + const target = getCompactionTarget(modelLimits, 0.5); + + expect(target).toBe(100000); + }); + + it('should return correct target for 90% percentage', () => { + // target = floor(200000 * 0.9) = 180000 + const modelLimits: ModelLimits = { + contextWindow: 200000, + }; + + const target = getCompactionTarget(modelLimits, 0.9); + + expect(target).toBe(180000); + }); + }); + + describe('floor behavior', () => { + it('should floor the result to avoid fractional tokens', () => { + // target = floor(100000 * 0.33) = 33000 + const modelLimits: ModelLimits = { + contextWindow: 100000, + }; + + const target = getCompactionTarget(modelLimits, 0.33); + + expect(Number.isInteger(target)).toBe(true); + expect(target).toBe(33000); + }); + }); + + describe('small context windows', () => { + it('should work correctly with small context windows', () => { + // target = floor(8192 * 0.7) = 5734 + const modelLimits: ModelLimits = { + contextWindow: 8192, + }; + + const target = getCompactionTarget(modelLimits); + + expect(target).toBe(5734); + }); + }); +}); diff --git a/dexto/packages/core/src/context/compaction/overflow.ts b/dexto/packages/core/src/context/compaction/overflow.ts new file mode 100644 index 00000000..74ccb9e3 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/overflow.ts @@ -0,0 +1,59 @@ +import type { TokenUsage } from '../../llm/types.js'; + +/** + * Model limits configuration for overflow detection. + * These limits define the context window boundaries. + */ +export interface ModelLimits { + /** Maximum context window size in tokens (the model's input limit) */ + contextWindow: number; +} + +/** + * Determines if the context has overflowed based on token usage. + * + * Overflow is detected when: + * inputTokens > contextWindow * thresholdPercent + * + * The thresholdPercent allows triggering compaction before hitting 100% (e.g., at 90%). + * This provides a safety margin for estimation errors and prevents hitting hard limits. + * + * Note: We don't reserve space for "output" because input and output have separate limits + * in LLM APIs. The model's output doesn't consume from the input context window. + * + * @param tokens The token usage (actual from API or estimated) + * @param modelLimits The model's context window limit + * @param thresholdPercent Percentage of context window at which to trigger (default 0.9 = 90%) + * @returns true if context has overflowed and compaction is needed + */ +export function isOverflow( + tokens: TokenUsage, + modelLimits: ModelLimits, + thresholdPercent: number = 0.9 +): boolean { + const { contextWindow } = modelLimits; + + // Apply threshold - trigger compaction at thresholdPercent of context window + const effectiveLimit = Math.floor(contextWindow * thresholdPercent); + + // Calculate used tokens - inputTokens is the main metric + const inputTokens = tokens.inputTokens ?? 0; + + // Check if we've exceeded the effective limit + return inputTokens > effectiveLimit; +} + +/** + * Calculate the compaction target - how many tokens we need to reduce to. + * + * @param modelLimits The model's context window limit + * @param targetPercentage What percentage of context to target (default 70%) + * @returns The target token count after compaction + */ +export function getCompactionTarget( + modelLimits: ModelLimits, + targetPercentage: number = 0.7 +): number { + const { contextWindow } = modelLimits; + return Math.floor(contextWindow * targetPercentage); +} diff --git a/dexto/packages/core/src/context/compaction/provider.ts b/dexto/packages/core/src/context/compaction/provider.ts new file mode 100644 index 00000000..0205b87b --- /dev/null +++ b/dexto/packages/core/src/context/compaction/provider.ts @@ -0,0 +1,59 @@ +import { z } from 'zod'; +import type { LanguageModel } from 'ai'; +import type { ICompactionStrategy } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; + +/** + * Context provided to compaction strategy creation + */ +export interface CompactionContext { + logger: IDextoLogger; + model?: LanguageModel; // Optional - some strategies may not need LLM +} + +/** + * Provider interface for compaction strategies. + * + * Follows the same pattern as blob storage and tools providers: + * - Type discriminator for config validation + * - Zod schema for runtime validation + * - Factory function to create instances + * - Metadata for discovery and UI + * + * TConfig should be the output type (z.output) with defaults applied + */ +export interface CompactionProvider< + TType extends string = string, + TConfig extends CompactionConfig = CompactionConfig, +> { + /** Unique identifier for this strategy type */ + type: TType; + + /** Zod schema for validating configuration - accepts input, produces TConfig output */ + configSchema: z.ZodType; + + /** Metadata for discovery and UI */ + metadata?: { + displayName: string; + description: string; + requiresLLM: boolean; // Does it need LLM access? + isProactive: boolean; // Proactive vs reactive? + }; + + /** + * Create a compaction strategy instance + * @param config - Validated configuration with defaults applied (output type) + */ + create( + config: TConfig, + context: CompactionContext + ): ICompactionStrategy | Promise; +} + +/** + * Base configuration for all compaction strategies + */ +export interface CompactionConfig { + type: string; + enabled?: boolean; // Allow disabling without removing config +} diff --git a/dexto/packages/core/src/context/compaction/providers/noop-provider.ts b/dexto/packages/core/src/context/compaction/providers/noop-provider.ts new file mode 100644 index 00000000..b8fbaa24 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/providers/noop-provider.ts @@ -0,0 +1,36 @@ +import { z } from 'zod'; +import type { CompactionProvider } from '../provider.js'; +import { NoOpCompactionStrategy } from '../strategies/noop.js'; + +/** + * Configuration schema for no-op compaction + */ +export const NoOpConfigSchema = z + .object({ + type: z.literal('noop'), + enabled: z.boolean().default(true).describe('Enable or disable compaction'), + }) + .strict(); + +export type NoOpConfig = z.output; + +/** + * Provider for no-op compaction strategy. + * + * This strategy disables compaction entirely, keeping full conversation history. + * Useful for testing, debugging, or contexts where full history is required. + */ +export const noopProvider: CompactionProvider<'noop', NoOpConfig> = { + type: 'noop', + configSchema: NoOpConfigSchema, + metadata: { + displayName: 'No Compaction', + description: 'Disables compaction entirely, keeping full conversation history', + requiresLLM: false, + isProactive: false, + }, + + create(_config, _context) { + return new NoOpCompactionStrategy(); + }, +}; diff --git a/dexto/packages/core/src/context/compaction/providers/reactive-overflow-provider.ts b/dexto/packages/core/src/context/compaction/providers/reactive-overflow-provider.ts new file mode 100644 index 00000000..96ad7994 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/providers/reactive-overflow-provider.ts @@ -0,0 +1,95 @@ +import { z } from 'zod'; +import type { CompactionProvider } from '../provider.js'; +import { ReactiveOverflowStrategy } from '../strategies/reactive-overflow.js'; + +/** + * Configuration schema for reactive overflow compaction + */ +export const ReactiveOverflowConfigSchema = z + .object({ + type: z.literal('reactive-overflow'), + enabled: z.boolean().default(true).describe('Enable or disable compaction'), + /** + * Maximum context tokens before compaction triggers. + * When set, overrides the model's context window for compaction threshold. + * Useful for capping context size below the model's maximum limit. + */ + maxContextTokens: z + .number() + .positive() + .optional() + .describe( + 'Maximum context tokens before compaction triggers. Overrides model context window when set.' + ), + /** + * Percentage of context window that triggers compaction (0.1 to 1.0). + * Default is 1.0 (100%), meaning compaction triggers when context is full. + */ + thresholdPercent: z + .number() + .min(0.1) + .max(1.0) + .default(1.0) + .describe( + 'Percentage of context window that triggers compaction (0.1 to 1.0, default 1.0)' + ), + preserveLastNTurns: z + .number() + .int() + .positive() + .default(2) + .describe('Number of recent turns (user+assistant pairs) to preserve'), + maxSummaryTokens: z + .number() + .int() + .positive() + .default(2000) + .describe('Maximum tokens for the summary output'), + summaryPrompt: z + .string() + .optional() + .describe('Custom summary prompt template. Use {conversation} as placeholder'), + }) + .strict(); + +export type ReactiveOverflowConfig = z.output; + +/** + * Provider for reactive overflow compaction strategy. + * + * This strategy triggers compaction when context window overflow is detected: + * - Generates LLM-powered summaries of older messages + * - Preserves recent turns for context continuity + * - Falls back to simple text summary if LLM call fails + * - Adds summary message to history (read-time filtering excludes old messages) + */ +export const reactiveOverflowProvider: CompactionProvider< + 'reactive-overflow', + ReactiveOverflowConfig +> = { + type: 'reactive-overflow', + configSchema: ReactiveOverflowConfigSchema, + metadata: { + displayName: 'Reactive Overflow Compaction', + description: 'Generates summaries when context window overflows, preserving recent turns', + requiresLLM: true, + isProactive: false, + }, + + create(config, context) { + if (!context.model) { + throw new Error('ReactiveOverflowStrategy requires LanguageModel'); + } + + const options: import('../strategies/reactive-overflow.js').ReactiveOverflowOptions = { + preserveLastNTurns: config.preserveLastNTurns, + maxSummaryTokens: config.maxSummaryTokens, + }; + + if (config.summaryPrompt !== undefined) { + options.summaryPrompt = config.summaryPrompt; + } + + return new ReactiveOverflowStrategy(context.model, options, context.logger); + }, +}; diff --git a/dexto/packages/core/src/context/compaction/registry.test.ts b/dexto/packages/core/src/context/compaction/registry.test.ts new file mode 100644 index 00000000..844c32af --- /dev/null +++ b/dexto/packages/core/src/context/compaction/registry.test.ts @@ -0,0 +1,537 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { z } from 'zod'; +import { compactionRegistry } from './registry.js'; +import type { CompactionProvider, CompactionConfig, CompactionContext } from './provider.js'; +import type { ICompactionStrategy } from './types.js'; +import type { InternalMessage } from '../types.js'; + +// Mock compaction config types +interface MockCompactionConfig extends CompactionConfig { + type: 'mock'; + enabled?: boolean; + maxTokens?: number; +} + +interface AnotherMockConfig extends CompactionConfig { + type: 'another-mock'; + enabled?: boolean; + threshold?: number; +} + +// Mock compaction strategy implementation +class MockCompressionStrategy implements ICompactionStrategy { + readonly name = 'mock-compaction'; + + constructor(private config: MockCompactionConfig) {} + + async compact(history: readonly InternalMessage[]): Promise { + return history.slice(0, this.config.maxTokens || 100) as InternalMessage[]; + } +} + +class AnotherMockStrategy implements ICompactionStrategy { + readonly name = 'another-mock-compaction'; + + constructor(private config: AnotherMockConfig) {} + + async compact(history: readonly InternalMessage[]): Promise { + return history.slice(0, this.config.threshold || 50) as InternalMessage[]; + } +} + +// Mock compaction providers +const mockProvider: CompactionProvider<'mock', MockCompactionConfig> = { + type: 'mock', + configSchema: z.object({ + type: z.literal('mock'), + enabled: z.boolean().default(true), + maxTokens: z.number().default(100), + }), + metadata: { + displayName: 'Mock Compaction', + description: 'A mock compaction strategy for testing', + requiresLLM: false, + isProactive: true, + }, + create(config: MockCompactionConfig, _context: CompactionContext): ICompactionStrategy { + return new MockCompressionStrategy(config); + }, +}; + +const anotherMockProvider: CompactionProvider<'another-mock', AnotherMockConfig> = { + type: 'another-mock', + configSchema: z.object({ + type: z.literal('another-mock'), + enabled: z.boolean().default(true), + threshold: z.number().default(50), + }), + metadata: { + displayName: 'Another Mock Compaction', + description: 'Another mock compaction strategy for testing', + requiresLLM: true, + isProactive: false, + }, + create(config: AnotherMockConfig, _context: CompactionContext): ICompactionStrategy { + return new AnotherMockStrategy(config); + }, +}; + +const minimalProvider: CompactionProvider<'minimal', CompactionConfig> = { + type: 'minimal', + configSchema: z.object({ + type: z.literal('minimal'), + enabled: z.boolean().default(true), + }), + create(_config: CompactionConfig, _context: CompactionContext): ICompactionStrategy { + return { + name: 'minimal-compaction', + compact: async (history: readonly InternalMessage[]) => + history.slice() as InternalMessage[], + }; + }, +}; + +describe('CompactionRegistry', () => { + beforeEach(() => { + // Clear registry before each test to ensure isolation + compactionRegistry.clear(); + }); + + describe('register()', () => { + it('successfully registers a provider', () => { + expect(() => compactionRegistry.register(mockProvider)).not.toThrow(); + expect(compactionRegistry.has('mock')).toBe(true); + }); + + it('successfully registers multiple providers', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + expect(compactionRegistry.has('mock')).toBe(true); + expect(compactionRegistry.has('another-mock')).toBe(true); + }); + + it('throws error when registering duplicate provider', () => { + compactionRegistry.register(mockProvider); + + expect(() => compactionRegistry.register(mockProvider)).toThrow( + "Compaction provider 'mock' is already registered" + ); + }); + + it('allows re-registration after unregistering', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.unregister('mock'); + + expect(() => compactionRegistry.register(mockProvider)).not.toThrow(); + expect(compactionRegistry.has('mock')).toBe(true); + }); + + it('registers provider with minimal metadata', () => { + compactionRegistry.register(minimalProvider); + + const provider = compactionRegistry.get('minimal'); + expect(provider).toBeDefined(); + expect(provider?.type).toBe('minimal'); + expect(provider?.metadata).toBeUndefined(); + }); + }); + + describe('unregister()', () => { + it('successfully unregisters an existing provider', () => { + compactionRegistry.register(mockProvider); + + const result = compactionRegistry.unregister('mock'); + + expect(result).toBe(true); + expect(compactionRegistry.has('mock')).toBe(false); + }); + + it('returns false when unregistering non-existent provider', () => { + const result = compactionRegistry.unregister('non-existent'); + + expect(result).toBe(false); + }); + + it('returns false when unregistering from empty registry', () => { + const result = compactionRegistry.unregister('mock'); + + expect(result).toBe(false); + }); + + it('can unregister one provider while keeping others', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + const result = compactionRegistry.unregister('mock'); + + expect(result).toBe(true); + expect(compactionRegistry.has('mock')).toBe(false); + expect(compactionRegistry.has('another-mock')).toBe(true); + }); + }); + + describe('get()', () => { + it('returns registered provider', () => { + compactionRegistry.register(mockProvider); + + const provider = compactionRegistry.get('mock'); + + expect(provider).toBeDefined(); + expect(provider?.type).toBe('mock'); + expect(provider?.metadata?.displayName).toBe('Mock Compaction'); + }); + + it('returns undefined for non-existent provider', () => { + const provider = compactionRegistry.get('non-existent'); + + expect(provider).toBeUndefined(); + }); + + it('returns undefined from empty registry', () => { + const provider = compactionRegistry.get('mock'); + + expect(provider).toBeUndefined(); + }); + + it('returns correct provider when multiple are registered', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + const provider1 = compactionRegistry.get('mock'); + const provider2 = compactionRegistry.get('another-mock'); + + expect(provider1?.type).toBe('mock'); + expect(provider2?.type).toBe('another-mock'); + }); + + it('returns full provider interface including create function', () => { + compactionRegistry.register(mockProvider); + + const provider = compactionRegistry.get('mock'); + + expect(provider).toBeDefined(); + expect(typeof provider?.create).toBe('function'); + expect(provider?.configSchema).toBeDefined(); + }); + }); + + describe('has()', () => { + it('returns true for registered provider', () => { + compactionRegistry.register(mockProvider); + + expect(compactionRegistry.has('mock')).toBe(true); + }); + + it('returns false for non-existent provider', () => { + expect(compactionRegistry.has('non-existent')).toBe(false); + }); + + it('returns false from empty registry', () => { + expect(compactionRegistry.has('mock')).toBe(false); + }); + + it('returns false after unregistering', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.unregister('mock'); + + expect(compactionRegistry.has('mock')).toBe(false); + }); + + it('correctly identifies multiple registered providers', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + expect(compactionRegistry.has('mock')).toBe(true); + expect(compactionRegistry.has('another-mock')).toBe(true); + expect(compactionRegistry.has('non-existent')).toBe(false); + }); + }); + + describe('getTypes()', () => { + it('returns all registered provider types', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + const types = compactionRegistry.getTypes(); + + expect(types).toHaveLength(2); + expect(types).toContain('mock'); + expect(types).toContain('another-mock'); + }); + + it('returns empty array for empty registry', () => { + const types = compactionRegistry.getTypes(); + + expect(types).toEqual([]); + }); + + it('returns single type when only one provider is registered', () => { + compactionRegistry.register(mockProvider); + + const types = compactionRegistry.getTypes(); + + expect(types).toEqual(['mock']); + }); + + it('updates after unregistering a provider', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + compactionRegistry.unregister('mock'); + + const types = compactionRegistry.getTypes(); + + expect(types).toHaveLength(1); + expect(types).toContain('another-mock'); + expect(types).not.toContain('mock'); + }); + + it('returns array that can be iterated', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + const types = compactionRegistry.getTypes(); + const typeArray: string[] = []; + + types.forEach((type) => { + expect(typeof type).toBe('string'); + typeArray.push(type); + }); + + expect(typeArray.length).toBe(2); + }); + }); + + describe('getAll()', () => { + it('returns all registered providers', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + const providers = compactionRegistry.getAll(); + + expect(providers).toHaveLength(2); + expect(providers[0]!.type).toBe('mock'); + expect(providers[1]!.type).toBe('another-mock'); + }); + + it('returns empty array for empty registry', () => { + const providers = compactionRegistry.getAll(); + + expect(providers).toEqual([]); + }); + + it('returns single provider when only one is registered', () => { + compactionRegistry.register(mockProvider); + + const providers = compactionRegistry.getAll(); + + expect(providers).toHaveLength(1); + expect(providers[0]!.type).toBe('mock'); + }); + + it('updates after unregistering a provider', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + compactionRegistry.unregister('mock'); + + const providers = compactionRegistry.getAll(); + + expect(providers).toHaveLength(1); + expect(providers[0]!.type).toBe('another-mock'); + }); + + it('returns providers with full interface including metadata', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + const providers = compactionRegistry.getAll(); + + expect(providers[0]!.metadata).toBeDefined(); + expect(providers[0]!.metadata?.displayName).toBe('Mock Compaction'); + expect(providers[1]!.metadata).toBeDefined(); + expect(providers[1]!.metadata?.requiresLLM).toBe(true); + }); + + it('returns array that can be filtered and mapped', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + compactionRegistry.register(minimalProvider); + + const providers = compactionRegistry.getAll(); + const providersWithLLM = providers.filter((p) => p.metadata?.requiresLLM === true); + const providerTypes = providers.map((p) => p.type); + + expect(providersWithLLM).toHaveLength(1); + expect(providersWithLLM[0]!.type).toBe('another-mock'); + expect(providerTypes).toEqual(['mock', 'another-mock', 'minimal']); + }); + }); + + describe('clear()', () => { + it('clears all registered providers', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + compactionRegistry.clear(); + + expect(compactionRegistry.getTypes()).toEqual([]); + expect(compactionRegistry.getAll()).toEqual([]); + expect(compactionRegistry.has('mock')).toBe(false); + expect(compactionRegistry.has('another-mock')).toBe(false); + }); + + it('can clear empty registry without errors', () => { + expect(() => compactionRegistry.clear()).not.toThrow(); + + expect(compactionRegistry.getTypes()).toEqual([]); + }); + + it('allows re-registration after clearing', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.clear(); + + expect(() => compactionRegistry.register(mockProvider)).not.toThrow(); + expect(compactionRegistry.has('mock')).toBe(true); + }); + + it('truly removes all providers including their state', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + compactionRegistry.register(minimalProvider); + + compactionRegistry.clear(); + + expect(compactionRegistry.get('mock')).toBeUndefined(); + expect(compactionRegistry.get('another-mock')).toBeUndefined(); + expect(compactionRegistry.get('minimal')).toBeUndefined(); + expect(compactionRegistry.getAll().length).toBe(0); + }); + }); + + describe('Integration scenarios', () => { + it('supports complete provider lifecycle', () => { + // Register + compactionRegistry.register(mockProvider); + expect(compactionRegistry.has('mock')).toBe(true); + + // Get and verify + const provider = compactionRegistry.get('mock'); + expect(provider?.type).toBe('mock'); + + // Use provider + expect(typeof provider?.create).toBe('function'); + + // Unregister + const unregistered = compactionRegistry.unregister('mock'); + expect(unregistered).toBe(true); + expect(compactionRegistry.has('mock')).toBe(false); + }); + + it('handles multiple provider types with different configurations', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + compactionRegistry.register(minimalProvider); + + const types = compactionRegistry.getTypes(); + expect(types).toHaveLength(3); + + const withMetadata = compactionRegistry + .getAll() + .filter((p) => p.metadata !== undefined); + expect(withMetadata).toHaveLength(2); + + const requiresLLM = compactionRegistry + .getAll() + .filter((p) => p.metadata?.requiresLLM === true); + expect(requiresLLM).toHaveLength(1); + expect(requiresLLM[0]!.type).toBe('another-mock'); + }); + + it('maintains provider isolation between operations', () => { + compactionRegistry.register(mockProvider); + + const provider1 = compactionRegistry.get('mock'); + const provider2 = compactionRegistry.get('mock'); + + // Both should return the same provider instance + expect(provider1).toBe(provider2); + + // Unregistering should affect all references + compactionRegistry.unregister('mock'); + expect(compactionRegistry.get('mock')).toBeUndefined(); + }); + + it('supports provider discovery pattern', () => { + compactionRegistry.register(mockProvider); + compactionRegistry.register(anotherMockProvider); + + // Discover all available providers + const allProviders = compactionRegistry.getAll(); + + // Filter by capability + const proactiveProviders = allProviders.filter((p) => p.metadata?.isProactive === true); + const llmProviders = allProviders.filter((p) => p.metadata?.requiresLLM === true); + + expect(proactiveProviders).toHaveLength(1); + expect(proactiveProviders[0]!.type).toBe('mock'); + expect(llmProviders).toHaveLength(1); + expect(llmProviders[0]!.type).toBe('another-mock'); + }); + }); + + describe('Edge cases and error handling', () => { + it('handles provider types with special characters', () => { + const specialProvider: CompactionProvider = { + type: 'special-provider_v2', + configSchema: z.object({ + type: z.literal('special-provider_v2'), + }), + create: () => ({ + name: 'special-provider', + compact: async (history: readonly InternalMessage[]) => + history.slice() as InternalMessage[], + }), + }; + + compactionRegistry.register(specialProvider); + + expect(compactionRegistry.has('special-provider_v2')).toBe(true); + expect(compactionRegistry.get('special-provider_v2')?.type).toBe('special-provider_v2'); + }); + + it('preserves provider metadata exactly as provided', () => { + compactionRegistry.register(mockProvider); + + const retrieved = compactionRegistry.get('mock'); + + expect(retrieved?.metadata).toEqual(mockProvider.metadata); + expect(retrieved?.metadata?.displayName).toBe(mockProvider.metadata?.displayName); + expect(retrieved?.metadata?.description).toBe(mockProvider.metadata?.description); + expect(retrieved?.metadata?.requiresLLM).toBe(mockProvider.metadata?.requiresLLM); + expect(retrieved?.metadata?.isProactive).toBe(mockProvider.metadata?.isProactive); + }); + + it('handles providers without optional metadata gracefully', () => { + compactionRegistry.register(minimalProvider); + + const provider = compactionRegistry.get('minimal'); + + expect(provider).toBeDefined(); + expect(provider?.metadata).toBeUndefined(); + expect(provider?.type).toBe('minimal'); + }); + + it('maintains type safety for provider retrieval', () => { + compactionRegistry.register(mockProvider); + + const provider = compactionRegistry.get('mock'); + + // TypeScript should know this is CompactionProvider + if (provider) { + expect(provider.type).toBeDefined(); + expect(provider.configSchema).toBeDefined(); + expect(provider.create).toBeDefined(); + } + }); + }); +}); diff --git a/dexto/packages/core/src/context/compaction/registry.ts b/dexto/packages/core/src/context/compaction/registry.ts new file mode 100644 index 00000000..c7df96e2 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/registry.ts @@ -0,0 +1,32 @@ +import type { CompactionProvider } from './provider.js'; +import { ContextError } from '../errors.js'; +import { BaseRegistry, type RegistryErrorFactory } from '../../providers/base-registry.js'; + +/** + * Error factory for compaction registry errors. + * Uses ContextError for consistent error handling. + */ +const compactionErrorFactory: RegistryErrorFactory = { + alreadyRegistered: (type: string) => ContextError.compactionProviderAlreadyRegistered(type), + notFound: (type: string, availableTypes: string[]) => + ContextError.compactionInvalidType(type, availableTypes), +}; + +/** + * Global registry for compaction providers. + * + * Follows the same pattern as blob storage and tools registries: + * - Singleton instance exported + * - Registration before agent initialization + * - Type-safe provider lookup + * + * Extends BaseRegistry for common registry functionality. + */ +class CompactionRegistry extends BaseRegistry> { + constructor() { + super(compactionErrorFactory); + } +} + +/** Global singleton instance */ +export const compactionRegistry = new CompactionRegistry(); diff --git a/dexto/packages/core/src/context/compaction/schemas.test.ts b/dexto/packages/core/src/context/compaction/schemas.test.ts new file mode 100644 index 00000000..b95d2c34 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/schemas.test.ts @@ -0,0 +1,265 @@ +import { describe, it, expect } from 'vitest'; +import { + CompactionConfigSchema, + DEFAULT_COMPACTION_CONFIG, + type CompactionConfigInput, +} from './schemas.js'; + +describe('CompactionConfigSchema', () => { + describe('basic validation', () => { + it('should accept valid minimal config', () => { + const input = { + type: 'reactive-overflow', + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.type).toBe('reactive-overflow'); + expect(result.data.enabled).toBe(true); + expect(result.data.thresholdPercent).toBe(0.9); + } + }); + + it('should accept config with enabled explicitly set', () => { + const input = { + type: 'reactive-overflow', + enabled: false, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.enabled).toBe(false); + } + }); + + it('should reject config without type', () => { + const input = { + enabled: true, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(false); + }); + }); + + describe('maxContextTokens', () => { + it('should accept positive maxContextTokens', () => { + const input = { + type: 'reactive-overflow', + maxContextTokens: 50000, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.maxContextTokens).toBe(50000); + } + }); + + it('should reject zero maxContextTokens', () => { + const input = { + type: 'reactive-overflow', + maxContextTokens: 0, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(false); + }); + + it('should reject negative maxContextTokens', () => { + const input = { + type: 'reactive-overflow', + maxContextTokens: -1000, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(false); + }); + + it('should allow omitting maxContextTokens', () => { + const input = { + type: 'reactive-overflow', + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.maxContextTokens).toBeUndefined(); + } + }); + }); + + describe('thresholdPercent', () => { + it('should default thresholdPercent to 0.9', () => { + const input = { + type: 'reactive-overflow', + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.thresholdPercent).toBe(0.9); + } + }); + + it('should accept thresholdPercent of 0.8 (80%)', () => { + const input = { + type: 'reactive-overflow', + thresholdPercent: 0.8, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.thresholdPercent).toBe(0.8); + } + }); + + it('should accept thresholdPercent of 0.1 (10% - minimum)', () => { + const input = { + type: 'reactive-overflow', + thresholdPercent: 0.1, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.thresholdPercent).toBe(0.1); + } + }); + + it('should accept thresholdPercent of 1.0 (100% - maximum)', () => { + const input = { + type: 'reactive-overflow', + thresholdPercent: 1.0, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.thresholdPercent).toBe(1.0); + } + }); + + it('should reject thresholdPercent below 0.1', () => { + const input = { + type: 'reactive-overflow', + thresholdPercent: 0.05, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(false); + }); + + it('should reject thresholdPercent above 1.0', () => { + const input = { + type: 'reactive-overflow', + thresholdPercent: 1.5, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(false); + }); + + it('should reject thresholdPercent of 0', () => { + const input = { + type: 'reactive-overflow', + thresholdPercent: 0, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(false); + }); + }); + + describe('combined configuration', () => { + it('should accept full config with all fields', () => { + const input = { + type: 'reactive-overflow', + enabled: true, + maxContextTokens: 100000, + thresholdPercent: 0.75, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.type).toBe('reactive-overflow'); + expect(result.data.enabled).toBe(true); + expect(result.data.maxContextTokens).toBe(100000); + expect(result.data.thresholdPercent).toBe(0.75); + } + }); + + it('should allow additional passthrough fields for provider-specific config', () => { + const input = { + type: 'reactive-overflow', + enabled: true, + maxSummaryTokens: 2000, + preserveLastNTurns: 3, + }; + + const result = CompactionConfigSchema.safeParse(input); + + expect(result.success).toBe(true); + if (result.success) { + // Passthrough fields should be preserved + expect((result.data as Record).maxSummaryTokens).toBe(2000); + expect((result.data as Record).preserveLastNTurns).toBe(3); + } + }); + }); + + describe('DEFAULT_COMPACTION_CONFIG', () => { + it('should have expected default values', () => { + expect(DEFAULT_COMPACTION_CONFIG.type).toBe('reactive-overflow'); + expect(DEFAULT_COMPACTION_CONFIG.enabled).toBe(true); + expect(DEFAULT_COMPACTION_CONFIG.thresholdPercent).toBe(0.9); + }); + + it('should validate successfully', () => { + const result = CompactionConfigSchema.safeParse(DEFAULT_COMPACTION_CONFIG); + + expect(result.success).toBe(true); + }); + }); + + describe('type inference', () => { + it('should produce correct output type', () => { + const config: CompactionConfigInput = { + type: 'reactive-overflow', + enabled: true, + maxContextTokens: 50000, + thresholdPercent: 0.9, + }; + + // Type checking - these should compile without errors + const type: string = config.type; + const enabled: boolean = config.enabled; + const maxTokens: number | undefined = config.maxContextTokens; + const threshold: number = config.thresholdPercent; + + expect(type).toBe('reactive-overflow'); + expect(enabled).toBe(true); + expect(maxTokens).toBe(50000); + expect(threshold).toBe(0.9); + }); + }); +}); diff --git a/dexto/packages/core/src/context/compaction/schemas.ts b/dexto/packages/core/src/context/compaction/schemas.ts new file mode 100644 index 00000000..f9650c97 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/schemas.ts @@ -0,0 +1,55 @@ +import { z } from 'zod'; + +/** + * Base compaction configuration schema. + * Uses discriminated union to support different provider types. + * + * Each provider registers its own schema with specific validation rules. + * This schema accepts any configuration with a 'type' field. + */ +export const CompactionConfigSchema = z + .object({ + type: z.string().describe('Compaction provider type'), + enabled: z.boolean().default(true).describe('Enable or disable compaction'), + /** + * Maximum context tokens before compaction triggers. + * When set, overrides the model's context window for compaction threshold. + * Useful for capping context size below the model's maximum limit. + * Example: Set to 50000 to trigger compaction at 50K tokens even if + * the model supports 200K tokens. + */ + maxContextTokens: z + .number() + .positive() + .optional() + .describe( + 'Maximum context tokens before compaction triggers. Overrides model context window when set.' + ), + /** + * Percentage of context window that triggers compaction (0.0 to 1.0). + * Default is 0.9 (90%), leaving a 10% buffer to avoid context degradation. + * Set lower values to trigger compaction earlier. + * Example: 0.8 triggers compaction when 80% of context is used. + */ + thresholdPercent: z + .number() + .min(0.1) + .max(1.0) + .default(0.9) + .describe( + 'Percentage of context window that triggers compaction (0.1 to 1.0, default 0.9)' + ), + }) + .passthrough() // Allow additional fields that will be validated by provider schemas + .describe('Context compaction configuration'); + +export type CompactionConfigInput = z.output; + +/** + * Default compaction configuration - uses reactive-overflow strategy + */ +export const DEFAULT_COMPACTION_CONFIG: CompactionConfigInput = { + type: 'reactive-overflow', + enabled: true, + thresholdPercent: 0.9, +}; diff --git a/dexto/packages/core/src/context/compaction/strategies/noop.ts b/dexto/packages/core/src/context/compaction/strategies/noop.ts new file mode 100644 index 00000000..b7d6258d --- /dev/null +++ b/dexto/packages/core/src/context/compaction/strategies/noop.ts @@ -0,0 +1,21 @@ +import type { ICompactionStrategy } from '../types.js'; +import type { InternalMessage } from '../../types.js'; + +/** + * No-op compaction strategy that doesn't perform any compaction. + * + * Useful for: + * - Testing without compaction overhead + * - Disabling compaction temporarily + * - Contexts where full history is required + */ +export class NoOpCompactionStrategy implements ICompactionStrategy { + readonly name = 'noop'; + + /** + * Does nothing - returns empty array (no summary needed) + */ + async compact(_history: readonly InternalMessage[]): Promise { + return []; + } +} diff --git a/dexto/packages/core/src/context/compaction/strategies/reactive-overflow.test.ts b/dexto/packages/core/src/context/compaction/strategies/reactive-overflow.test.ts new file mode 100644 index 00000000..56271f6e --- /dev/null +++ b/dexto/packages/core/src/context/compaction/strategies/reactive-overflow.test.ts @@ -0,0 +1,703 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { ReactiveOverflowStrategy } from './reactive-overflow.js'; +import type { InternalMessage } from '../../types.js'; +import type { LanguageModel } from 'ai'; +import { createMockLogger } from '../../../logger/v2/test-utils.js'; +import { filterCompacted } from '../../utils.js'; + +// Mock the ai module +vi.mock('ai', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + generateText: vi.fn(), + }; +}); + +import { generateText } from 'ai'; + +const mockGenerateText = vi.mocked(generateText); + +/** + * Helper to create a mock LanguageModel + */ +function createMockModel(): LanguageModel { + return { + modelId: 'test-model', + provider: 'test-provider', + specificationVersion: 'v1', + doStream: vi.fn(), + doGenerate: vi.fn(), + } as unknown as LanguageModel; +} + +/** + * Helper to create test messages + */ +function createUserMessage(text: string, timestamp?: number): InternalMessage { + return { + role: 'user', + content: [{ type: 'text', text }], + timestamp: timestamp ?? Date.now(), + }; +} + +function createAssistantMessage(text: string, timestamp?: number): InternalMessage { + return { + role: 'assistant', + content: [{ type: 'text', text }], + timestamp: timestamp ?? Date.now(), + }; +} + +function createSummaryMessage( + text: string, + originalMessageCount: number, + timestamp?: number +): InternalMessage { + return { + role: 'assistant', + content: [{ type: 'text', text }], + timestamp: timestamp ?? Date.now(), + metadata: { + isSummary: true, + summarizedAt: Date.now(), + originalMessageCount, + }, + }; +} + +describe('ReactiveOverflowStrategy', () => { + const logger = createMockLogger(); + let strategy: ReactiveOverflowStrategy; + + beforeEach(() => { + vi.clearAllMocks(); + strategy = new ReactiveOverflowStrategy(createMockModel(), {}, logger); + }); + + describe('compact() - short history guard', () => { + it('should return empty array when history has 2 or fewer messages', async () => { + const history: InternalMessage[] = [ + createUserMessage('Hello'), + createAssistantMessage('Hi there!'), + ]; + + const result = await strategy.compact(history); + + expect(result).toEqual([]); + expect(mockGenerateText).not.toHaveBeenCalled(); + }); + + it('should return empty array for empty history', async () => { + const result = await strategy.compact([]); + + expect(result).toEqual([]); + expect(mockGenerateText).not.toHaveBeenCalled(); + }); + + it('should return empty array for single message', async () => { + const history: InternalMessage[] = [createUserMessage('Hello')]; + + const result = await strategy.compact(history); + + expect(result).toEqual([]); + }); + }); + + describe('compact() - summary message metadata', () => { + it('should return summary with isSummary=true metadata', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Test summary', + } as Awaited>); + + // Create enough messages to trigger compaction + // preserveLastNTurns=2 by default, so we need more than 2 turns + const history: InternalMessage[] = [ + createUserMessage('First question', 1000), + createAssistantMessage('First answer', 1001), + createUserMessage('Second question', 1002), + createAssistantMessage('Second answer', 1003), + createUserMessage('Third question', 1004), + createAssistantMessage('Third answer', 1005), + ]; + + const result = await strategy.compact(history); + + expect(result).toHaveLength(1); + expect(result[0]?.metadata?.isSummary).toBe(true); + }); + + it('should set originalMessageCount to number of summarized messages', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Test summary', + } as Awaited>); + + // 6 messages total, preserveLastNTurns=2 means last 4 messages are kept + // (2 turns = 2 user + 2 assistant messages in the last 2 turns) + const history: InternalMessage[] = [ + createUserMessage('Old question 1', 1000), + createAssistantMessage('Old answer 1', 1001), + createUserMessage('Recent question 1', 1002), + createAssistantMessage('Recent answer 1', 1003), + createUserMessage('Recent question 2', 1004), + createAssistantMessage('Recent answer 2', 1005), + ]; + + const result = await strategy.compact(history); + + expect(result).toHaveLength(1); + // First 2 messages (1 turn) should be summarized + expect(result[0]?.metadata?.originalMessageCount).toBe(2); + }); + + it('should include summarizedAt timestamp in metadata', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Test summary', + } as Awaited>); + + const history: InternalMessage[] = [ + createUserMessage('Question 1', 1000), + createAssistantMessage('Answer 1', 1001), + createUserMessage('Question 2', 1002), + createAssistantMessage('Answer 2', 1003), + createUserMessage('Question 3', 1004), + createAssistantMessage('Answer 3', 1005), + ]; + + const beforeTime = Date.now(); + const result = await strategy.compact(history); + const afterTime = Date.now(); + + expect(result[0]?.metadata?.summarizedAt).toBeGreaterThanOrEqual(beforeTime); + expect(result[0]?.metadata?.summarizedAt).toBeLessThanOrEqual(afterTime); + }); + + it('should include original timestamps in metadata', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Test summary', + } as Awaited>); + + const history: InternalMessage[] = [ + createUserMessage('Old question', 1000), + createAssistantMessage('Old answer', 2000), + createUserMessage('Recent question 1', 3000), + createAssistantMessage('Recent answer 1', 4000), + createUserMessage('Recent question 2', 5000), + createAssistantMessage('Recent answer 2', 6000), + ]; + + const result = await strategy.compact(history); + + expect(result[0]?.metadata?.originalFirstTimestamp).toBe(1000); + expect(result[0]?.metadata?.originalLastTimestamp).toBe(2000); + }); + }); + + describe('compact() - re-compaction with existing summary', () => { + it('should detect existing summary and only summarize messages after it', async () => { + mockGenerateText.mockResolvedValue({ + text: 'New summary', + } as Awaited>); + + // History with existing summary + const history: InternalMessage[] = [ + createUserMessage('Very old question', 1000), + createAssistantMessage('Very old answer', 1001), + createSummaryMessage('Previous summary', 2, 1002), + // Messages after the summary + createUserMessage('Question after summary 1', 2000), + createAssistantMessage('Answer after summary 1', 2001), + createUserMessage('Question after summary 2', 2002), + createAssistantMessage('Answer after summary 2', 2003), + createUserMessage('Question after summary 3', 2004), + createAssistantMessage('Answer after summary 3', 2005), + ]; + + const result = await strategy.compact(history); + + expect(result).toHaveLength(1); + // Should mark as re-compaction + expect(result[0]?.metadata?.isRecompaction).toBe(true); + }); + + it('should skip re-compaction if few messages after existing summary', async () => { + // History with summary and only 3 messages after (threshold is 4) + const history: InternalMessage[] = [ + createUserMessage('Old question', 1000), + createAssistantMessage('Old answer', 1001), + createSummaryMessage('Existing summary', 2, 1002), + createUserMessage('New question', 2000), + createAssistantMessage('New answer', 2001), + createUserMessage('Another question', 2002), + ]; + + const result = await strategy.compact(history); + + expect(result).toEqual([]); + expect(mockGenerateText).not.toHaveBeenCalled(); + }); + + it('should find most recent summary when multiple exist', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Newest summary', + } as Awaited>); + + // History with two summaries - should use the most recent one + const history: InternalMessage[] = [ + createUserMessage('Ancient question', 100), + createSummaryMessage('First summary', 1, 200), + createUserMessage('Old question', 300), + createAssistantMessage('Old answer', 301), + createSummaryMessage('Second summary', 2, 400), + // Messages after second summary + createUserMessage('Q1', 500), + createAssistantMessage('A1', 501), + createUserMessage('Q2', 502), + createAssistantMessage('A2', 503), + createUserMessage('Q3', 504), + createAssistantMessage('A3', 505), + ]; + + const result = await strategy.compact(history); + + // Should have re-compaction metadata + expect(result).toHaveLength(1); + expect(result[0]?.metadata?.isRecompaction).toBe(true); + }); + + it('should set originalMessageCount as absolute index for filterCompacted compatibility', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Re-compacted summary', + } as Awaited>); + + // History with existing summary at index 2 + // - Indices 0-1: old messages (summarized by old summary) + // - Index 2: old summary with originalMessageCount=2 + // - Indices 3-8: 6 messages after old summary + const history: InternalMessage[] = [ + createUserMessage('Very old question', 1000), + createAssistantMessage('Very old answer', 1001), + createSummaryMessage('Previous summary', 2, 1002), + // 6 messages after the summary + createUserMessage('Q1', 2000), + createAssistantMessage('A1', 2001), + createUserMessage('Q2', 2002), + createAssistantMessage('A2', 2003), + createUserMessage('Q3', 2004), + createAssistantMessage('A3', 2005), + ]; + + // Run re-compaction + const result = await strategy.compact(history); + expect(result).toHaveLength(1); + + const newSummary = result[0]!; + expect(newSummary.metadata?.isRecompaction).toBe(true); + + // The existing summary is at index 2, and messagesAfterSummary has 6 messages + // With default preserveLastNTurns=2, we split: toSummarize=2, toKeep=4 + // So originalMessageCount should be: (2 + 1) + 2 = 5 (absolute index) + // NOT 2 (relative count of summarized messages) + expect(newSummary.metadata?.originalMessageCount).toBe(5); + + // Simulate adding the new summary to history + const historyAfterCompaction = [...history, newSummary]; + + // Verify filterCompacted works correctly with the new summary + const filtered = filterCompacted(historyAfterCompaction); + + // Should return: [newSummary, 4 preserved messages] + // NOT: [newSummary, everything from index 2 onwards] + expect(filtered).toHaveLength(5); // 1 summary + 4 preserved + expect(filtered[0]?.metadata?.isRecompaction).toBe(true); + // The preserved messages should be the last 4 (indices 5-8 in original) + expect(filtered[1]?.role).toBe('user'); + expect(filtered[4]?.role).toBe('assistant'); + }); + + it('should ensure filterCompacted does not return old summary or pre-summary messages after re-compaction', async () => { + mockGenerateText.mockResolvedValue({ + text: 'New summary', + } as Awaited>); + + // Large history to make the bug more obvious + const history: InternalMessage[] = []; + // 50 old messages (indices 0-49) + for (let i = 0; i < 50; i++) { + history.push(createUserMessage(`Old Q${i}`, 1000 + i * 2)); + history.push(createAssistantMessage(`Old A${i}`, 1001 + i * 2)); + } + // Old summary at index 100 with originalMessageCount=90 + history.push(createSummaryMessage('Old summary', 90, 2000)); + // 30 more messages after the old summary (indices 101-130) + for (let i = 0; i < 15; i++) { + history.push(createUserMessage(`New Q${i}`, 3000 + i * 2)); + history.push(createAssistantMessage(`New A${i}`, 3001 + i * 2)); + } + + expect(history).toHaveLength(131); + + // Re-compaction should happen + const result = await strategy.compact(history); + expect(result).toHaveLength(1); + + const newSummary = result[0]!; + expect(newSummary.metadata?.isRecompaction).toBe(true); + + // Add new summary to history + const historyAfterCompaction = [...history, newSummary]; + + // filterCompacted should NOT return the old summary or pre-old-summary messages + const filtered = filterCompacted(historyAfterCompaction); + + // Check that the old summary is NOT in the filtered result + const hasOldSummary = filtered.some( + (msg) => msg.metadata?.isSummary && !msg.metadata?.isRecompaction + ); + expect(hasOldSummary).toBe(false); + + // The filtered result should be much smaller than the original + // With 30 messages after old summary, keeping ~20%, we should have: + // ~6 preserved messages + 1 new summary = ~7 messages + expect(filtered.length).toBeLessThan(20); + }); + + it('should handle three sequential compactions correctly', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Summary content', + } as Awaited>); + + // Helper to simulate adding messages and compacting + let history: InternalMessage[] = []; + + // === PHASE 1: First compaction === + // Add 20 messages (10 turns) + for (let i = 0; i < 10; i++) { + history.push(createUserMessage(`Q${i}`, 1000 + i * 2)); + history.push(createAssistantMessage(`A${i}`, 1001 + i * 2)); + } + expect(history).toHaveLength(20); + + // First compaction - no existing summary + const result1 = await strategy.compact(history); + expect(result1).toHaveLength(1); + const summary1 = result1[0]!; + expect(summary1.metadata?.isRecompaction).toBeUndefined(); + + // Add summary1 to history + history.push(summary1); + expect(history).toHaveLength(21); + + // Verify filterCompacted after first compaction + let filtered = filterCompacted(history); + expect(filtered.length).toBeLessThan(15); // Should be summary + few preserved + + // === PHASE 2: Add more messages, then second compaction === + // Add 20 more messages after summary1 + for (let i = 10; i < 20; i++) { + history.push(createUserMessage(`Q${i}`, 2000 + i * 2)); + history.push(createAssistantMessage(`A${i}`, 2001 + i * 2)); + } + expect(history).toHaveLength(41); + + // Second compaction - should detect summary1 + const result2 = await strategy.compact(history); + expect(result2).toHaveLength(1); + const summary2 = result2[0]!; + expect(summary2.metadata?.isRecompaction).toBe(true); + + // Add summary2 to history + history.push(summary2); + expect(history).toHaveLength(42); + + // Verify filterCompacted after second compaction + filtered = filterCompacted(history); + // Should return summary2 + preserved, NOT summary1 + expect(filtered[0]?.metadata?.isRecompaction).toBe(true); + const hasSummary1 = filtered.some( + (m) => m.metadata?.isSummary && !m.metadata?.isRecompaction + ); + expect(hasSummary1).toBe(false); + + // === PHASE 3: Add more messages, then third compaction === + // Add 20 more messages after summary2 + for (let i = 20; i < 30; i++) { + history.push(createUserMessage(`Q${i}`, 3000 + i * 2)); + history.push(createAssistantMessage(`A${i}`, 3001 + i * 2)); + } + expect(history).toHaveLength(62); + + // Third compaction - should detect summary2 (most recent) + const result3 = await strategy.compact(history); + expect(result3).toHaveLength(1); + const summary3 = result3[0]!; + expect(summary3.metadata?.isRecompaction).toBe(true); + + // Add summary3 to history + history.push(summary3); + expect(history).toHaveLength(63); + + // Verify filterCompacted after third compaction + filtered = filterCompacted(history); + + // Critical assertions: + // 1. Most recent summary (summary3) should be first + expect(filtered[0]?.metadata?.isRecompaction).toBe(true); + expect(filtered[0]).toBe(summary3); + + // 2. Neither summary1 nor summary2 should be in the result + const oldSummaries = filtered.filter((m) => m.metadata?.isSummary && m !== summary3); + expect(oldSummaries).toHaveLength(0); + + // 3. Result should be much smaller than total history + expect(filtered.length).toBeLessThan(20); + + // 4. All messages in filtered result should be either: + // - summary3, or + // - messages with timestamps from the most recent batch (3000+) + for (const msg of filtered) { + if (msg === summary3) continue; + // Recent messages should have timestamps >= 3000 + expect(msg.timestamp).toBeGreaterThanOrEqual(3000); + } + }); + + it('should work correctly with manual compaction followed by automatic compaction', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Summary', + } as Awaited>); + + // Simulate manual compaction first + let history: InternalMessage[] = []; + for (let i = 0; i < 10; i++) { + history.push(createUserMessage(`Q${i}`, 1000 + i)); + history.push(createAssistantMessage(`A${i}`, 1000 + i)); + } + + // Manual compaction (uses same compact() method) + const manualResult = await strategy.compact(history); + expect(manualResult).toHaveLength(1); + history.push(manualResult[0]!); + + // Add more messages + for (let i = 10; i < 20; i++) { + history.push(createUserMessage(`Q${i}`, 2000 + i)); + history.push(createAssistantMessage(`A${i}`, 2000 + i)); + } + + // Automatic compaction (also uses same compact() method) + const autoResult = await strategy.compact(history); + expect(autoResult).toHaveLength(1); + expect(autoResult[0]?.metadata?.isRecompaction).toBe(true); + history.push(autoResult[0]!); + + // Verify final state + const filtered = filterCompacted(history); + expect(filtered[0]?.metadata?.isRecompaction).toBe(true); + + // Only the most recent summary should be visible + const summaryCount = filtered.filter((m) => m.metadata?.isSummary).length; + expect(summaryCount).toBe(1); + }); + }); + + describe('compact() - history splitting', () => { + it('should preserve last N turns based on options', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Summary', + } as Awaited>); + + // Create strategy with custom preserveLastNTurns + const customStrategy = new ReactiveOverflowStrategy( + createMockModel(), + { preserveLastNTurns: 3 }, + logger + ); + + // 8 messages = 4 turns, with preserveLastNTurns=3, first turn should be summarized + const history: InternalMessage[] = [ + createUserMessage('Turn 1 Q', 1000), + createAssistantMessage('Turn 1 A', 1001), + createUserMessage('Turn 2 Q', 2000), + createAssistantMessage('Turn 2 A', 2001), + createUserMessage('Turn 3 Q', 3000), + createAssistantMessage('Turn 3 A', 3001), + createUserMessage('Turn 4 Q', 4000), + createAssistantMessage('Turn 4 A', 4001), + ]; + + const result = await customStrategy.compact(history); + + expect(result).toHaveLength(1); + // Only first turn (2 messages) should be summarized + expect(result[0]?.metadata?.originalMessageCount).toBe(2); + }); + + it('should return empty when message count is at or below minKeep threshold', async () => { + // The fallback logic uses minKeep=3, so with 3 or fewer messages + // nothing should be summarized + const history: InternalMessage[] = [ + createUserMessage('Q1', 1000), + createAssistantMessage('A1', 1001), + createUserMessage('Q2', 2000), + ]; + + const result = await strategy.compact(history); + + // 3 messages <= minKeep(3), so nothing to summarize + expect(result).toEqual([]); + expect(mockGenerateText).not.toHaveBeenCalled(); + }); + }); + + describe('compact() - LLM failure fallback', () => { + it('should create fallback summary when LLM call fails', async () => { + mockGenerateText.mockRejectedValue(new Error('LLM API error')); + + const history: InternalMessage[] = [ + createUserMessage('Question 1', 1000), + createAssistantMessage('Answer 1', 1001), + createUserMessage('Question 2', 2000), + createAssistantMessage('Answer 2', 2001), + createUserMessage('Question 3', 3000), + createAssistantMessage('Answer 3', 3001), + ]; + + const result = await strategy.compact(history); + + expect(result).toHaveLength(1); + expect(result[0]?.metadata?.isSummary).toBe(true); + // Fallback summary should still have XML structure + const content = result[0]?.content; + expect(content).toBeDefined(); + expect(content![0]).toMatchObject({ + type: 'text', + text: expect.stringContaining(''), + }); + expect(content![0]).toMatchObject({ + type: 'text', + text: expect.stringContaining('Fallback'), + }); + }); + + it('should include current task in fallback summary', async () => { + mockGenerateText.mockRejectedValue(new Error('LLM API error')); + + const history: InternalMessage[] = [ + createUserMessage('Old question', 1000), + createAssistantMessage('Old answer', 1001), + createUserMessage('Recent question 1', 2000), + createAssistantMessage('Recent answer 1', 2001), + createUserMessage('My current task is to fix the bug', 3000), + createAssistantMessage('Working on it', 3001), + ]; + + const result = await strategy.compact(history); + + expect(result).toHaveLength(1); + const content = result[0]!.content; + expect(content).not.toBeNull(); + const firstContent = content![0]; + const summaryText = firstContent?.type === 'text' ? firstContent.text : ''; + expect(summaryText).toContain(''); + }); + }); + + describe('compact() - summary content', () => { + it('should prefix summary with [Session Compaction Summary]', async () => { + mockGenerateText.mockResolvedValue({ + text: 'LLM generated content', + } as Awaited>); + + const history: InternalMessage[] = [ + createUserMessage('Q1', 1000), + createAssistantMessage('A1', 1001), + createUserMessage('Q2', 2000), + createAssistantMessage('A2', 2001), + createUserMessage('Q3', 3000), + createAssistantMessage('A3', 3001), + ]; + + const result = await strategy.compact(history); + + expect(result).toHaveLength(1); + const content = result[0]!.content; + expect(content).not.toBeNull(); + const firstContent = content![0]; + const summaryText = firstContent?.type === 'text' ? firstContent.text : ''; + expect(summaryText).toMatch(/^\[Session Compaction Summary\]/); + }); + + it('should pass conversation to LLM with proper formatting', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Summary', + } as Awaited>); + + const history: InternalMessage[] = [ + createUserMessage('What is 2+2?', 1000), + createAssistantMessage('The answer is 4', 1001), + createUserMessage('Thanks!', 2000), + createAssistantMessage('You are welcome', 2001), + createUserMessage('New question', 3000), + createAssistantMessage('New answer', 3001), + ]; + + await strategy.compact(history); + + expect(mockGenerateText).toHaveBeenCalledWith( + expect.objectContaining({ + prompt: expect.stringContaining('USER: What is 2+2?'), + }) + ); + expect(mockGenerateText).toHaveBeenCalledWith( + expect.objectContaining({ + prompt: expect.stringContaining('ASSISTANT: The answer is 4'), + }) + ); + }); + }); + + describe('compact() - tool message handling', () => { + it('should include tool call information in summary', async () => { + mockGenerateText.mockResolvedValue({ + text: 'Summary with tools', + } as Awaited>); + + const history: InternalMessage[] = [ + createUserMessage('Read the file', 1000), + { + role: 'assistant', + content: [{ type: 'text', text: 'Let me read that file' }], + timestamp: 1001, + toolCalls: [ + { + id: 'call-1', + type: 'function', + function: { name: 'read_file', arguments: '{"path": "/test.txt"}' }, + }, + ], + }, + { + role: 'tool', + content: [{ type: 'text', text: 'File contents here' }], + timestamp: 1002, + name: 'read_file', + toolCallId: 'call-1', + }, + createUserMessage('Q2', 2000), + createAssistantMessage('A2', 2001), + createUserMessage('Q3', 3000), + createAssistantMessage('A3', 3001), + ]; + + await strategy.compact(history); + + expect(mockGenerateText).toHaveBeenCalledWith( + expect.objectContaining({ + prompt: expect.stringContaining('[Used tools: read_file]'), + }) + ); + }); + }); +}); diff --git a/dexto/packages/core/src/context/compaction/strategies/reactive-overflow.ts b/dexto/packages/core/src/context/compaction/strategies/reactive-overflow.ts new file mode 100644 index 00000000..6b5b25d8 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/strategies/reactive-overflow.ts @@ -0,0 +1,489 @@ +import { generateText, type LanguageModel } from 'ai'; +import type { ICompactionStrategy } from '../types.js'; +import type { InternalMessage, ToolCall } from '../../types.js'; +import { isAssistantMessage, isToolMessage } from '../../types.js'; +import type { IDextoLogger } from '../../../logger/v2/types.js'; + +/** + * Configuration options for ReactiveOverflowStrategy. + */ +export interface ReactiveOverflowOptions { + /** + * Number of recent turns to preserve (not summarize). + * A "turn" is a user message + assistant response pair. + * Default: 2 + */ + preserveLastNTurns?: number; + + /** + * Maximum tokens for the summary output. + * Default: 2000 + */ + maxSummaryTokens?: number; + + /** + * Custom summary prompt template. + * Use {conversation} as placeholder for formatted messages. + */ + summaryPrompt?: string; +} + +const DEFAULT_OPTIONS: Required = { + preserveLastNTurns: 2, + maxSummaryTokens: 2000, + summaryPrompt: `You are a conversation summarizer creating a structured summary for session continuation. + +Analyze the conversation and produce a summary in the following XML format: + + + + A concise summary of what happened in the conversation: + - Tasks attempted and their outcomes (success/failure/in-progress) + - Important decisions made + - Key information discovered (file paths, configurations, errors encountered) + - Tools used and their results + + + + The most recent task or instruction the user requested that may still be in progress. + Be specific - include the exact request and current status. + + + + Critical state that must be preserved: + - File paths being worked on + - Variable values or configurations + - Error messages that need addressing + - Any pending actions or next steps + + + +IMPORTANT: The assistant will continue working based on this summary. Ensure the current_task section clearly states what needs to be done next. + +Conversation to summarize: +{conversation}`, +}; + +/** + * ReactiveOverflowStrategy implements reactive compaction. + * + * Key behaviors: + * - Triggers on overflow (after actual tokens exceed context limit) + * - Uses LLM to generate intelligent summary of older messages + * - Returns summary message to ADD to history (not replace) + * - Read-time filtering via filterCompacted() excludes pre-summary messages + * + * This strategy is designed to work with TurnExecutor's main loop: + * 1. After each step, check if overflow occurred + * 2. If yes, generate summary and ADD it to history + * 3. filterCompacted() in getFormattedMessages() excludes old messages + * 4. Continue with fresh context (summary + recent messages) + * + * NOTE: This does NOT replace history. The summary message is ADDED, + * and filterCompacted() handles excluding old messages at read-time. + * This preserves full history for audit/recovery purposes. + */ +export class ReactiveOverflowStrategy implements ICompactionStrategy { + readonly name = 'reactive-overflow'; + + private readonly model: LanguageModel; + private readonly options: Required; + private readonly logger: IDextoLogger; + + constructor(model: LanguageModel, options: ReactiveOverflowOptions = {}, logger: IDextoLogger) { + this.model = model; + this.options = { ...DEFAULT_OPTIONS, ...options }; + this.logger = logger; + } + + /** + * Generate a summary message for the old portion of history. + * + * IMPORTANT: This does NOT replace history. It returns a summary message + * that the caller should ADD to history via contextManager.addMessage(). + * Read-time filtering (filterCompacted) will then exclude pre-summary + * messages when formatting for LLM. + * + * @param history The full conversation history + * @returns Array with single summary message to add, or empty if nothing to summarize + */ + async compact(history: readonly InternalMessage[]): Promise { + // Don't compact if history is too short + if (history.length <= 2) { + this.logger.debug('ReactiveOverflowStrategy: History too short, skipping compaction'); + return []; + } + + // Check if there's already a summary in history + // If so, we need to work with messages AFTER the summary only + // Use reverse search to find the MOST RECENT summary (important for re-compaction) + let existingSummaryIndex = -1; + for (let i = history.length - 1; i >= 0; i--) { + const msg = history[i]; + if (msg?.metadata?.isSummary === true || msg?.metadata?.isSessionSummary === true) { + existingSummaryIndex = i; + break; + } + } + + if (existingSummaryIndex !== -1) { + // There's already a summary - only consider messages AFTER it + const messagesAfterSummary = history.slice(existingSummaryIndex + 1); + + // If there are very few messages after the summary, skip compaction + // (nothing meaningful to re-summarize) + if (messagesAfterSummary.length <= 4) { + this.logger.debug( + `ReactiveOverflowStrategy: Only ${messagesAfterSummary.length} messages after existing summary, skipping re-compaction` + ); + return []; + } + + this.logger.info( + `ReactiveOverflowStrategy: Found existing summary at index ${existingSummaryIndex}, ` + + `working with ${messagesAfterSummary.length} messages after it` + ); + + // Re-run compaction on the subset after the summary + // This prevents cascading summaries of summaries + return this.compactSubset(messagesAfterSummary, history, existingSummaryIndex); + } + + // Split history into messages to summarize and messages to keep + const { toSummarize, toKeep } = this.splitHistory(history); + + // If nothing to summarize, return empty (no summary needed) + if (toSummarize.length === 0) { + this.logger.debug('ReactiveOverflowStrategy: No messages to summarize'); + return []; + } + + // Find the most recent user message to understand current task + const currentTaskMessage = this.findCurrentTaskMessage(history); + + this.logger.info( + `ReactiveOverflowStrategy: Summarizing ${toSummarize.length} messages, keeping ${toKeep.length}` + ); + + // Generate LLM summary of old messages with current task context + const summary = await this.generateSummary(toSummarize, currentTaskMessage); + + // Create summary message (will be ADDED to history, not replace) + // originalMessageCount tells filterCompacted() how many messages were summarized + const summaryMessage: InternalMessage = { + role: 'assistant', + content: [{ type: 'text', text: summary }], + timestamp: Date.now(), + metadata: { + isSummary: true, + summarizedAt: Date.now(), + originalMessageCount: toSummarize.length, + originalFirstTimestamp: toSummarize[0]?.timestamp, + originalLastTimestamp: toSummarize[toSummarize.length - 1]?.timestamp, + }, + }; + + // Return just the summary message - caller adds it to history + // filterCompacted() will handle excluding old messages at read-time + return [summaryMessage]; + } + + /** + * Handle re-compaction when there's already a summary in history. + * Only summarizes messages AFTER the existing summary, preventing + * cascading summaries of summaries. + * + * @param messagesAfterSummary Messages after the existing summary + * @param fullHistory The complete history (for current task detection) + * @param existingSummaryIndex Index of the existing summary in fullHistory + * @returns Array with single summary message, or empty if nothing to summarize + */ + private async compactSubset( + messagesAfterSummary: readonly InternalMessage[], + fullHistory: readonly InternalMessage[], + existingSummaryIndex: number + ): Promise { + // Split the subset into messages to summarize and keep + const { toSummarize, toKeep } = this.splitHistory(messagesAfterSummary); + + if (toSummarize.length === 0) { + this.logger.debug('ReactiveOverflowStrategy: No messages to summarize in subset'); + return []; + } + + // Get current task from the full history + const currentTaskMessage = this.findCurrentTaskMessage(fullHistory); + + this.logger.info( + `ReactiveOverflowStrategy (re-compact): Summarizing ${toSummarize.length} messages after existing summary, keeping ${toKeep.length}` + ); + + // Generate summary + const summary = await this.generateSummary(toSummarize, currentTaskMessage); + + // Create summary message + // originalMessageCount must be an ABSOLUTE index for filterCompacted() to work correctly. + // filterCompacted() uses this as: history.slice(originalMessageCount, summaryIndex) + // to get the preserved messages. For re-compaction: + // - Messages 0 to existingSummaryIndex are the old summarized + preserved + old summary + // - Messages (existingSummaryIndex + 1) onwards are what we're re-compacting + // - We summarize toSummarize.length of those, so preserved starts at: + // (existingSummaryIndex + 1) + toSummarize.length + const absoluteOriginalMessageCount = existingSummaryIndex + 1 + toSummarize.length; + + const summaryMessage: InternalMessage = { + role: 'assistant', + content: [{ type: 'text', text: summary }], + timestamp: Date.now(), + metadata: { + isSummary: true, + summarizedAt: Date.now(), + originalMessageCount: absoluteOriginalMessageCount, + isRecompaction: true, // Mark that this is a re-compaction + originalFirstTimestamp: toSummarize[0]?.timestamp, + originalLastTimestamp: toSummarize[toSummarize.length - 1]?.timestamp, + }, + }; + + return [summaryMessage]; + } + + /** + * Find the most recent user message that represents the current task. + * This helps preserve context about what the user is currently asking for. + */ + private findCurrentTaskMessage(history: readonly InternalMessage[]): string | null { + // Search backwards for the most recent user message + for (let i = history.length - 1; i >= 0; i--) { + const msg = history[i]; + if (msg?.role === 'user') { + if (typeof msg.content === 'string') { + return msg.content; + } else if (Array.isArray(msg.content)) { + const textParts = msg.content + .filter( + (part): part is { type: 'text'; text: string } => part.type === 'text' + ) + .map((part) => part.text) + .join('\n'); + if (textParts.length > 0) { + return textParts; + } + } + } + } + return null; + } + + /** + * Split history into messages to summarize and messages to keep. + * Keeps the last N turns (user + assistant pairs) intact. + * + * For long agentic conversations with many tool calls, this also ensures + * we don't try to keep too many messages even within preserved turns. + */ + private splitHistory(history: readonly InternalMessage[]): { + toSummarize: readonly InternalMessage[]; + toKeep: readonly InternalMessage[]; + } { + const turnsToKeep = this.options.preserveLastNTurns; + + // Find indices of the last N user messages (start of each turn) + const userMessageIndices: number[] = []; + for (let i = history.length - 1; i >= 0; i--) { + if (history[i]?.role === 'user') { + userMessageIndices.unshift(i); + if (userMessageIndices.length >= turnsToKeep) { + break; + } + } + } + + // If we found turn boundaries, split at the first one + if (userMessageIndices.length > 0) { + const splitIndex = userMessageIndices[0]; + if (splitIndex !== undefined && splitIndex > 0) { + return { + toSummarize: history.slice(0, splitIndex), + toKeep: history.slice(splitIndex), + }; + } + } + + // Fallback for agentic conversations: if splitIndex is 0 (few user messages) + // or we can't identify turns, use a message-count-based approach. + // Keep only the last ~20% of messages or minimum 3 messages + // Note: We use a low minKeep because even a few messages can have huge token counts + // (e.g., tool outputs with large file contents). Token-based compaction needs to be + // aggressive about message counts when tokens are overflowing. + const minKeep = 3; + const maxKeepPercent = 0.2; + const keepCount = Math.max(minKeep, Math.floor(history.length * maxKeepPercent)); + + // But don't summarize if we'd keep everything anyway + if (keepCount >= history.length) { + return { + toSummarize: [], + toKeep: history, + }; + } + + this.logger.debug( + `splitHistory: Using fallback - keeping last ${keepCount} of ${history.length} messages` + ); + + return { + toSummarize: history.slice(0, -keepCount), + toKeep: history.slice(-keepCount), + }; + } + + /** + * Generate an LLM summary of the messages. + * + * @param messages Messages to summarize + * @param currentTask The most recent user message (current task context) + */ + private async generateSummary( + messages: readonly InternalMessage[], + currentTask: string | null + ): Promise { + const formattedConversation = this.formatMessagesForSummary(messages); + + // Add current task context to the prompt if available + let conversationWithContext = formattedConversation; + if (currentTask) { + conversationWithContext += `\n\n--- CURRENT TASK (most recent user request) ---\n${currentTask}`; + } + + const prompt = this.options.summaryPrompt.replace( + '{conversation}', + conversationWithContext + ); + + try { + const result = await generateText({ + model: this.model, + prompt, + maxOutputTokens: this.options.maxSummaryTokens, + }); + + // Return structured summary - the XML format from the LLM + return `[Session Compaction Summary]\n${result.text}`; + } catch (error) { + this.logger.error( + `ReactiveOverflowStrategy: Failed to generate summary - ${error instanceof Error ? error.message : String(error)}` + ); + // Fallback: return a simple truncated version with current task + return this.createFallbackSummary(messages, currentTask); + } + } + + /** + * Format messages for the summary prompt. + */ + private formatMessagesForSummary(messages: readonly InternalMessage[]): string { + return messages + .map((msg) => { + const role = msg.role.toUpperCase(); + let content: string; + + if (typeof msg.content === 'string') { + content = msg.content; + } else if (Array.isArray(msg.content)) { + // Extract text from content parts + content = msg.content + .filter( + (part): part is { type: 'text'; text: string } => part.type === 'text' + ) + .map((part) => part.text) + .join('\n'); + } else { + content = '[no content]'; + } + + // Truncate very long messages + if (content.length > 2000) { + content = content.slice(0, 2000) + '... [truncated]'; + } + + // Handle tool calls + if (isAssistantMessage(msg) && msg.toolCalls && msg.toolCalls.length > 0) { + const toolNames = msg.toolCalls + .map((tc: ToolCall) => tc.function.name) + .join(', '); + content += `\n[Used tools: ${toolNames}]`; + } + + // Handle tool results + if (isToolMessage(msg)) { + return `TOOL (${msg.name}): ${content.slice(0, 500)}${content.length > 500 ? '...' : ''}`; + } + + return `${role}: ${content}`; + }) + .join('\n\n'); + } + + /** + * Create a fallback summary if LLM call fails. + */ + private createFallbackSummary( + messages: readonly InternalMessage[], + currentTask: string | null + ): string { + const userMessages = messages.filter((m) => m.role === 'user'); + const assistantWithTools = messages.filter( + (m): m is InternalMessage & { role: 'assistant'; toolCalls: ToolCall[] } => + isAssistantMessage(m) && !!m.toolCalls && m.toolCalls.length > 0 + ); + + const userTopics = userMessages + .slice(-3) + .map((m) => { + const text = + typeof m.content === 'string' + ? m.content + : Array.isArray(m.content) + ? m.content + .filter( + (p): p is { type: 'text'; text: string } => p.type === 'text' + ) + .map((p) => p.text) + .join(' ') + : ''; + return text.slice(0, 100); + }) + .join('; '); + + const toolsUsed = [ + ...new Set( + assistantWithTools.flatMap((m) => m.toolCalls.map((tc) => tc.function.name)) + ), + ].join(', '); + + // Create XML-structured fallback + let fallback = `[Session Compaction Summary - Fallback] + + + User discussed: ${userTopics || 'various topics'} + Tools used: ${toolsUsed || 'none'} + Messages summarized: ${messages.length} + `; + + if (currentTask) { + fallback += ` + + ${currentTask.slice(0, 500)}${currentTask.length > 500 ? '...' : ''} + `; + } + + fallback += ` + + Note: This is a fallback summary due to LLM error. Context may be incomplete. + +`; + + return fallback; + } +} diff --git a/dexto/packages/core/src/context/compaction/types.ts b/dexto/packages/core/src/context/compaction/types.ts new file mode 100644 index 00000000..90b768f7 --- /dev/null +++ b/dexto/packages/core/src/context/compaction/types.ts @@ -0,0 +1,33 @@ +import { InternalMessage } from '../types.js'; + +/** + * Compaction strategy interface. + * + * Strategies are responsible for reducing conversation history size + * when context limits are exceeded. The strategy is called by TurnExecutor + * after detecting overflow via actual token usage from the API. + */ +export interface ICompactionStrategy { + /** Human-readable name for logging/UI */ + readonly name: string; + + /** + * Compacts the provided message history. + * + * The returned summary messages MUST include specific metadata fields for + * `filterCompacted()` to correctly exclude pre-summary messages at read-time: + * + * Required metadata: + * - `isSummary: true` - Marks the message as a compaction summary + * - `originalMessageCount: number` - Count of messages that were summarized + * (used by filterCompacted to determine which messages to exclude) + * + * Optional metadata: + * - `isRecompaction: true` - Set when re-compacting after a previous summary + * - `isSessionSummary: true` - Alternative to isSummary for session-level summaries + * + * @param history The current conversation history. + * @returns Summary messages to add to history. Empty array if nothing to compact. + */ + compact(history: readonly InternalMessage[]): Promise | InternalMessage[]; +} diff --git a/dexto/packages/core/src/context/error-codes.ts b/dexto/packages/core/src/context/error-codes.ts new file mode 100644 index 00000000..167c854b --- /dev/null +++ b/dexto/packages/core/src/context/error-codes.ts @@ -0,0 +1,41 @@ +/** + * Context-specific error codes + * Includes initialization, message validation, token processing, and formatting errors + */ +export enum ContextErrorCode { + // Message validation + MESSAGE_ROLE_MISSING = 'context_message_role_missing', + MESSAGE_CONTENT_EMPTY = 'context_message_content_empty', + + // User message validation + USER_MESSAGE_CONTENT_INVALID = 'context_user_message_content_invalid', + + // Assistant message validation + ASSISTANT_MESSAGE_CONTENT_OR_TOOLS_REQUIRED = 'context_assistant_message_content_or_tools_required', + ASSISTANT_MESSAGE_TOOL_CALLS_INVALID = 'context_assistant_message_tool_calls_invalid', + + // Tool message validation + TOOL_MESSAGE_FIELDS_MISSING = 'context_tool_message_fields_missing', + TOOL_CALL_ID_NAME_REQUIRED = 'context_tool_call_id_name_required', + + // System message validation + SYSTEM_MESSAGE_CONTENT_INVALID = 'context_system_message_content_invalid', + + TOKEN_COUNT_FAILED = 'context_token_count_failed', + // (removed) Operation/formatting wrappers; domain errors bubble up + // (removed) Token processing wrappers; domain errors bubble up + // (removed) Provider/model required; validated at LLM or agent layer + + // Compaction strategy configuration errors + PRESERVE_VALUES_NEGATIVE = 'context_preserve_values_negative', + MIN_MESSAGES_NEGATIVE = 'context_min_messages_negative', + COMPACTION_INVALID_TYPE = 'context_compaction_invalid_type', + COMPACTION_VALIDATION = 'context_compaction_validation', + COMPACTION_MISSING_LLM = 'context_compaction_missing_llm', + COMPACTION_PROVIDER_ALREADY_REGISTERED = 'context_compaction_provider_already_registered', + + // Message lookup errors + MESSAGE_NOT_FOUND = 'context_message_not_found', + MESSAGE_NOT_ASSISTANT = 'context_message_not_assistant', + ASSISTANT_CONTENT_NOT_STRING = 'context_assistant_content_not_string', +} diff --git a/dexto/packages/core/src/context/errors.ts b/dexto/packages/core/src/context/errors.ts new file mode 100644 index 00000000..36537b42 --- /dev/null +++ b/dexto/packages/core/src/context/errors.ts @@ -0,0 +1,210 @@ +import { DextoRuntimeError } from '../errors/index.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import { ContextErrorCode } from './error-codes.js'; + +/** + * Context runtime error factory methods + * Creates properly typed errors for context management operations + */ +export class ContextError { + // Message validation errors + static messageRoleMissing() { + return new DextoRuntimeError( + ContextErrorCode.MESSAGE_ROLE_MISSING, + ErrorScope.CONTEXT, + ErrorType.USER, + 'Message must have a role', + {}, + 'Ensure all messages have a valid role field' + ); + } + + static userMessageContentInvalid() { + return new DextoRuntimeError( + ContextErrorCode.USER_MESSAGE_CONTENT_INVALID, + ErrorScope.CONTEXT, + ErrorType.USER, + 'User message content should be a non-empty string or a non-empty array of parts', + {}, + 'Provide valid content for user messages' + ); + } + + static assistantMessageContentOrToolsRequired() { + return new DextoRuntimeError( + ContextErrorCode.ASSISTANT_MESSAGE_CONTENT_OR_TOOLS_REQUIRED, + ErrorScope.CONTEXT, + ErrorType.USER, + 'Assistant message must have content or toolCalls', + {}, + 'Provide either content or toolCalls for assistant messages' + ); + } + + static assistantMessageToolCallsInvalid() { + return new DextoRuntimeError( + ContextErrorCode.ASSISTANT_MESSAGE_TOOL_CALLS_INVALID, + ErrorScope.CONTEXT, + ErrorType.USER, + 'Invalid toolCalls structure in assistant message', + {}, + 'Ensure toolCalls have proper structure with function name and arguments' + ); + } + + static toolMessageFieldsMissing() { + return new DextoRuntimeError( + ContextErrorCode.TOOL_MESSAGE_FIELDS_MISSING, + ErrorScope.CONTEXT, + ErrorType.USER, + 'Tool message missing required fields (toolCallId, name, content)', + {}, + 'Ensure tool messages have toolCallId, name, and content fields' + ); + } + + static systemMessageContentInvalid() { + return new DextoRuntimeError( + ContextErrorCode.SYSTEM_MESSAGE_CONTENT_INVALID, + ErrorScope.CONTEXT, + ErrorType.USER, + 'System message content must be a non-empty string', + {}, + 'Provide valid string content for system messages' + ); + } + + static userMessageContentEmpty() { + return new DextoRuntimeError( + ContextErrorCode.MESSAGE_CONTENT_EMPTY, + ErrorScope.CONTEXT, + ErrorType.USER, + 'Content must be a non-empty string or have imageData/fileData', + {}, + 'Provide non-empty content or attach image/file data' + ); + } + + static toolCallIdNameRequired() { + return new DextoRuntimeError( + ContextErrorCode.TOOL_CALL_ID_NAME_REQUIRED, + ErrorScope.CONTEXT, + ErrorType.USER, + 'toolCallId and name are required', + {}, + 'Provide both toolCallId and name for tool results' + ); + } + + // Operation errors + // Removed operation and tokenization/formatting wrappers; let domain errors bubble + + // Compression strategy configuration errors + static preserveValuesNegative() { + return new DextoRuntimeError( + ContextErrorCode.PRESERVE_VALUES_NEGATIVE, + ErrorScope.CONTEXT, + ErrorType.USER, + 'preserveStart and preserveEnd must be non-negative', + {}, + 'Set preserveStart and preserveEnd to zero or positive values' + ); + } + + static tokenCountFailed(cause: string) { + return new DextoRuntimeError( + ContextErrorCode.TOKEN_COUNT_FAILED, + ErrorScope.CONTEXT, + ErrorType.SYSTEM, + `Failed to count tokens: ${cause}`, + { cause }, + 'Check tokenizer implementation and message content structure' + ); + } + + static minMessagesNegative() { + return new DextoRuntimeError( + ContextErrorCode.MIN_MESSAGES_NEGATIVE, + ErrorScope.CONTEXT, + ErrorType.USER, + 'minMessagesToKeep must be non-negative', + {}, + 'Set minMessagesToKeep to zero or positive value' + ); + } + + static compactionInvalidType(type: string, available: string[]) { + return new DextoRuntimeError( + ContextErrorCode.COMPACTION_INVALID_TYPE, + ErrorScope.CONTEXT, + ErrorType.USER, + `Unknown compaction provider type: '${type}'`, + { type, available }, + `Use one of the available types: ${available.join(', ')}` + ); + } + + static compactionValidation(type: string, errors: unknown) { + return new DextoRuntimeError( + ContextErrorCode.COMPACTION_VALIDATION, + ErrorScope.CONTEXT, + ErrorType.USER, + `Invalid configuration for compaction provider '${type}'`, + { type, errors }, + 'Check the configuration schema for this provider' + ); + } + + static compactionMissingLLM(type: string) { + return new DextoRuntimeError( + ContextErrorCode.COMPACTION_MISSING_LLM, + ErrorScope.CONTEXT, + ErrorType.USER, + `Compaction provider '${type}' requires LLM service but none provided`, + { type }, + 'Ensure LLM service is initialized before creating this compaction provider' + ); + } + + static compactionProviderAlreadyRegistered(type: string) { + return new DextoRuntimeError( + ContextErrorCode.COMPACTION_PROVIDER_ALREADY_REGISTERED, + ErrorScope.CONTEXT, + ErrorType.USER, + `Compaction provider '${type}' is already registered`, + { type }, + 'Each provider type can only be registered once' + ); + } + + // Message lookup errors + static messageNotFound(messageId: string) { + return new DextoRuntimeError( + ContextErrorCode.MESSAGE_NOT_FOUND, + ErrorScope.CONTEXT, + ErrorType.NOT_FOUND, + `Message with ID ${messageId} not found`, + { messageId } + ); + } + + static messageNotAssistant(messageId: string) { + return new DextoRuntimeError( + ContextErrorCode.MESSAGE_NOT_ASSISTANT, + ErrorScope.CONTEXT, + ErrorType.USER, + `Message with ID ${messageId} is not an assistant message`, + { messageId } + ); + } + + static assistantContentNotString() { + return new DextoRuntimeError( + ContextErrorCode.ASSISTANT_CONTENT_NOT_STRING, + ErrorScope.CONTEXT, + ErrorType.USER, + 'Cannot append text to non-string assistant message content', + {} + ); + } +} diff --git a/dexto/packages/core/src/context/index.ts b/dexto/packages/core/src/context/index.ts new file mode 100644 index 00000000..8757bb4d --- /dev/null +++ b/dexto/packages/core/src/context/index.ts @@ -0,0 +1,4 @@ +export * from './manager.js'; +export * from './types.js'; +export { getFileMediaKind, getResourceKind } from './media-helpers.js'; +export * from './compaction/index.js'; diff --git a/dexto/packages/core/src/context/manager.test.ts b/dexto/packages/core/src/context/manager.test.ts new file mode 100644 index 00000000..6a5c8d19 --- /dev/null +++ b/dexto/packages/core/src/context/manager.test.ts @@ -0,0 +1,1311 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { ContextManager } from './manager.js'; +import { MemoryHistoryProvider } from '../session/history/memory.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; +import type { ContentPart, SanitizedToolResult } from './types.js'; +import type { ValidatedLLMConfig } from '../llm/schemas.js'; +import type { VercelMessageFormatter } from '../llm/formatters/vercel.js'; +import type { SystemPromptManager } from '../systemPrompt/manager.js'; +import type { ResourceManager } from '../resources/manager.js'; +import type { BlobStore } from '../storage/blob/types.js'; +import type { DynamicContributorContext } from '../systemPrompt/types.js'; +import type { MCPManager } from '../mcp/manager.js'; + +// Create mock dependencies +const mockLogger = createMockLogger(); + +/** + * Create a typed mock for DynamicContributorContext. + * MCPManager is stubbed since tests don't actually use MCP features. + */ +function createMockContributorContext(): DynamicContributorContext { + return { + mcpManager: {} as MCPManager, + }; +} + +function createMockLLMConfig(): ValidatedLLMConfig { + return { + provider: 'openai', + model: 'gpt-4', + apiKey: 'test-key', + maxIterations: 10, + } as ValidatedLLMConfig; +} + +function createMockFormatter(): VercelMessageFormatter { + return { + format: vi.fn().mockReturnValue([]), + formatSystemPrompt: vi.fn().mockReturnValue(null), + } as unknown as VercelMessageFormatter; +} + +function createMockSystemPromptManager(): SystemPromptManager { + return { + build: vi.fn().mockResolvedValue('You are a helpful assistant.'), + } as unknown as SystemPromptManager; +} + +function createMockBlobStore(): BlobStore { + return { + store: vi.fn(), + retrieve: vi.fn(), + exists: vi.fn(), + delete: vi.fn(), + cleanup: vi.fn(), + getStats: vi.fn(), + listBlobs: vi.fn(), + getStoragePath: vi.fn(), + connect: vi.fn(), + disconnect: vi.fn(), + isConnected: vi.fn().mockReturnValue(true), + getStoreType: vi.fn().mockReturnValue('mock'), + } as unknown as BlobStore; +} + +function createMockResourceManager(): ResourceManager { + const mockBlobStore = createMockBlobStore(); + return { + read: vi.fn(), + getBlobStore: vi.fn().mockReturnValue(mockBlobStore), + } as unknown as ResourceManager; +} + +function createContextManager() { + const historyProvider = new MemoryHistoryProvider(mockLogger); + const formatter = createMockFormatter(); + const systemPromptManager = createMockSystemPromptManager(); + const resourceManager = createMockResourceManager(); + const llmConfig = createMockLLMConfig(); + + return new ContextManager( + llmConfig, + formatter, + systemPromptManager, + 4096, // maxInputTokens + historyProvider, + 'test-session-id', + resourceManager, + mockLogger + ); +} + +describe('ContextManager', () => { + let contextManager: ContextManager; + + beforeEach(() => { + vi.clearAllMocks(); + contextManager = createContextManager(); + }); + + describe('addUserMessage', () => { + it('should add a user message with text content', async () => { + const content: ContentPart[] = [{ type: 'text', text: 'Hello, world!' }]; + + await contextManager.addUserMessage(content); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(1); + expect(history[0]?.role).toBe('user'); + expect(history[0]?.content).toEqual([{ type: 'text', text: 'Hello, world!' }]); + }); + + it('should add a user message with multiple text parts', async () => { + const content: ContentPart[] = [ + { type: 'text', text: 'First part' }, + { type: 'text', text: 'Second part' }, + ]; + + await contextManager.addUserMessage(content); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(1); + expect(history[0]?.content).toHaveLength(2); + }); + + it('should add a user message with image content', async () => { + const content: ContentPart[] = [ + { type: 'text', text: 'Check this image' }, + { type: 'image', image: 'base64data', mimeType: 'image/png' }, + ]; + + await contextManager.addUserMessage(content); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(1); + expect(history[0]?.content).toHaveLength(2); + const imagePart = (history[0]?.content as ContentPart[])?.[1]; + expect(imagePart?.type).toBe('image'); + }); + + it('should add a user message with file content', async () => { + const content: ContentPart[] = [ + { type: 'text', text: 'Here is a file' }, + { + type: 'file', + data: 'filedata', + mimeType: 'application/pdf', + filename: 'doc.pdf', + }, + ]; + + await contextManager.addUserMessage(content); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(1); + const filePart = (history[0]?.content as ContentPart[])?.[1]; + expect(filePart?.type).toBe('file'); + }); + + it('should throw error for empty content array', async () => { + await expect(contextManager.addUserMessage([])).rejects.toThrow(); + }); + + it('should throw error for content with only whitespace text', async () => { + const content: ContentPart[] = [{ type: 'text', text: ' ' }]; + + await expect(contextManager.addUserMessage(content)).rejects.toThrow(); + }); + + it('should allow empty text with image attachment', async () => { + const content: ContentPart[] = [ + { type: 'text', text: '' }, + { type: 'image', image: 'base64data', mimeType: 'image/png' }, + ]; + + await contextManager.addUserMessage(content); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(1); + // Empty text parts are filtered out, only image remains + expect((history[0]?.content as ContentPart[])?.some((p) => p.type === 'image')).toBe( + true + ); + }); + + it('should generate message id and timestamp', async () => { + const content: ContentPart[] = [{ type: 'text', text: 'Hello' }]; + + await contextManager.addUserMessage(content); + + const history = await contextManager.getHistory(); + expect(history[0]?.id).toBeDefined(); + expect(history[0]?.timestamp).toBeDefined(); + expect(typeof history[0]?.id).toBe('string'); + expect(typeof history[0]?.timestamp).toBe('number'); + }); + }); + + describe('addAssistantMessage', () => { + it('should add an assistant message with string content', async () => { + await contextManager.addAssistantMessage('Hello from assistant'); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(1); + expect(history[0]?.role).toBe('assistant'); + // String content should be wrapped in ContentPart[] + expect(history[0]?.content).toEqual([{ type: 'text', text: 'Hello from assistant' }]); + }); + + it('should add an assistant message with null content and tool calls', async () => { + const toolCalls = [ + { + id: 'call-1', + type: 'function' as const, + function: { name: 'test_tool', arguments: '{}' }, + }, + ]; + + await contextManager.addAssistantMessage(null, toolCalls); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(1); + expect(history[0]?.role).toBe('assistant'); + expect(history[0]?.content).toBeNull(); + expect((history[0] as any).toolCalls).toHaveLength(1); + }); + + it('should add an assistant message with content and tool calls', async () => { + const toolCalls = [ + { + id: 'call-1', + type: 'function' as const, + function: { name: 'test_tool', arguments: '{}' }, + }, + ]; + + await contextManager.addAssistantMessage('Let me help', toolCalls); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(1); + expect(history[0]?.content).toEqual([{ type: 'text', text: 'Let me help' }]); + expect((history[0] as any).toolCalls).toHaveLength(1); + }); + + it('should throw error when neither content nor tool calls provided', async () => { + await expect(contextManager.addAssistantMessage(null, [])).rejects.toThrow(); + await expect(contextManager.addAssistantMessage(null, undefined)).rejects.toThrow(); + }); + + it('should include token usage metadata', async () => { + const tokenUsage = { + promptTokens: 100, + completionTokens: 50, + totalTokens: 150, + }; + + await contextManager.addAssistantMessage('Response', undefined, { tokenUsage }); + + const history = await contextManager.getHistory(); + expect((history[0] as any).tokenUsage).toEqual(tokenUsage); + }); + + it('should include reasoning metadata', async () => { + await contextManager.addAssistantMessage('Response', undefined, { + reasoning: 'I thought about this carefully', + }); + + const history = await contextManager.getHistory(); + expect((history[0] as any).reasoning).toBe('I thought about this carefully'); + }); + + it('should enrich with provider and model from config', async () => { + await contextManager.addAssistantMessage('Response'); + + const history = await contextManager.getHistory(); + expect((history[0] as any).provider).toBe('openai'); + expect((history[0] as any).model).toBe('gpt-4'); + }); + }); + + describe('appendAssistantText', () => { + it('should append text to existing assistant message with null content', async () => { + // First add an assistant message with tool calls only (null content) + const toolCalls = [ + { + id: 'call-1', + type: 'function' as const, + function: { name: 'test_tool', arguments: '{}' }, + }, + ]; + await contextManager.addAssistantMessage(null, toolCalls); + + const history = await contextManager.getHistory(); + const messageId = history[0]?.id!; + + // Append text + await contextManager.appendAssistantText(messageId, 'New text'); + + const updatedHistory = await contextManager.getHistory(); + expect(updatedHistory[0]?.content).toEqual([{ type: 'text', text: 'New text' }]); + }); + + it('should append text to existing text part', async () => { + await contextManager.addAssistantMessage('Initial'); + + const history = await contextManager.getHistory(); + const messageId = history[0]?.id!; + + await contextManager.appendAssistantText(messageId, ' appended'); + + const updatedHistory = await contextManager.getHistory(); + expect(updatedHistory[0]?.content).toEqual([ + { type: 'text', text: 'Initial appended' }, + ]); + }); + + it('should throw error for non-existent message', async () => { + await expect( + contextManager.appendAssistantText('non-existent-id', 'text') + ).rejects.toThrow(); + }); + + it('should throw error for non-assistant message', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'User message' }]); + + const history = await contextManager.getHistory(); + const messageId = history[0]?.id!; + + await expect(contextManager.appendAssistantText(messageId, 'text')).rejects.toThrow(); + }); + }); + + describe('addToolResult', () => { + it('should add a tool result message', async () => { + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Tool output' }], + meta: { + toolName: 'test_tool', + toolCallId: 'call-123', + success: true, + }, + }; + + await contextManager.addToolResult('call-123', 'test_tool', sanitizedResult); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(1); + expect(history[0]?.role).toBe('tool'); + expect(history[0]?.content).toEqual([{ type: 'text', text: 'Tool output' }]); + expect((history[0] as any).toolCallId).toBe('call-123'); + expect((history[0] as any).name).toBe('test_tool'); + expect((history[0] as any).success).toBe(true); + }); + + it('should add a tool result with image content', async () => { + const sanitizedResult: SanitizedToolResult = { + content: [ + { type: 'text', text: 'Here is the screenshot' }, + { type: 'image', image: 'base64screenshot', mimeType: 'image/png' }, + ], + meta: { + toolName: 'screenshot_tool', + toolCallId: 'call-456', + success: true, + }, + }; + + await contextManager.addToolResult('call-456', 'screenshot_tool', sanitizedResult); + + const history = await contextManager.getHistory(); + expect(history[0]?.content).toHaveLength(2); + }); + + it('should add a failed tool result', async () => { + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Error: Tool failed' }], + meta: { + toolName: 'failing_tool', + toolCallId: 'call-789', + success: false, + }, + }; + + await contextManager.addToolResult('call-789', 'failing_tool', sanitizedResult); + + const history = await contextManager.getHistory(); + expect((history[0] as any).success).toBe(false); + }); + + it('should include approval metadata', async () => { + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Approved result' }], + meta: { + toolName: 'dangerous_tool', + toolCallId: 'call-approved', + success: true, + }, + }; + + await contextManager.addToolResult('call-approved', 'dangerous_tool', sanitizedResult, { + requireApproval: true, + approvalStatus: 'approved', + }); + + const history = await contextManager.getHistory(); + expect((history[0] as any).requireApproval).toBe(true); + expect((history[0] as any).approvalStatus).toBe('approved'); + }); + + it('should throw error when toolCallId is missing', async () => { + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Output' }], + meta: { + toolName: 'test_tool', + toolCallId: 'call-123', + success: true, + }, + }; + + await expect( + contextManager.addToolResult('', 'test_tool', sanitizedResult) + ).rejects.toThrow(); + }); + + it('should throw error when name is missing', async () => { + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Output' }], + meta: { + toolName: 'test_tool', + toolCallId: 'call-123', + success: true, + }, + }; + + await expect( + contextManager.addToolResult('call-123', '', sanitizedResult) + ).rejects.toThrow(); + }); + }); + + describe('getHistory', () => { + it('should return empty array initially', async () => { + const history = await contextManager.getHistory(); + expect(history).toEqual([]); + }); + + it('should return all messages in order', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'User 1' }]); + await contextManager.addAssistantMessage('Assistant 1'); + await contextManager.addUserMessage([{ type: 'text', text: 'User 2' }]); + + const history = await contextManager.getHistory(); + expect(history).toHaveLength(3); + expect(history[0]?.role).toBe('user'); + expect(history[1]?.role).toBe('assistant'); + expect(history[2]?.role).toBe('user'); + }); + + it('should return defensive copy', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + const history1 = await contextManager.getHistory(); + const history2 = await contextManager.getHistory(); + + expect(history1).not.toBe(history2); + expect(history1).toEqual(history2); + }); + }); + + describe('resetConversation', () => { + it('should clear all messages', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await contextManager.addAssistantMessage('Hi'); + + let history = await contextManager.getHistory(); + expect(history).toHaveLength(2); + + await contextManager.resetConversation(); + + history = await contextManager.getHistory(); + expect(history).toEqual([]); + }); + }); + + describe('markMessagesAsCompacted', () => { + it('should mark tool messages as compacted', async () => { + // Add a tool result + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Tool output' }], + meta: { toolName: 'test', toolCallId: 'call-1', success: true }, + }; + await contextManager.addToolResult('call-1', 'test', sanitizedResult); + + const history = await contextManager.getHistory(); + const messageId = history[0]?.id!; + + const count = await contextManager.markMessagesAsCompacted([messageId]); + + expect(count).toBe(1); + const updatedHistory = await contextManager.getHistory(); + expect((updatedHistory[0] as any).compactedAt).toBeDefined(); + }); + + it('should not mark non-tool messages', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + const history = await contextManager.getHistory(); + const messageId = history[0]?.id!; + + const count = await contextManager.markMessagesAsCompacted([messageId]); + + expect(count).toBe(0); + }); + + it('should skip already compacted messages', async () => { + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Tool output' }], + meta: { toolName: 'test', toolCallId: 'call-1', success: true }, + }; + await contextManager.addToolResult('call-1', 'test', sanitizedResult); + + const history = await contextManager.getHistory(); + const messageId = history[0]?.id!; + + // Mark once + await contextManager.markMessagesAsCompacted([messageId]); + // Mark again + const count = await contextManager.markMessagesAsCompacted([messageId]); + + expect(count).toBe(0); + }); + + it('should return 0 for empty array', async () => { + const count = await contextManager.markMessagesAsCompacted([]); + expect(count).toBe(0); + }); + }); + + describe('lastActualInputTokens', () => { + it('should return null initially and store/update values', () => { + expect(contextManager.getLastActualInputTokens()).toBeNull(); + + contextManager.setLastActualInputTokens(5000); + expect(contextManager.getLastActualInputTokens()).toBe(5000); + + contextManager.setLastActualInputTokens(7500); + expect(contextManager.getLastActualInputTokens()).toBe(7500); + }); + }); + + describe('prepareHistory', () => { + it('should transform pruned tool messages to placeholders', async () => { + // Add a tool result with known content + const originalContent = 'This is a long tool output that will be pruned'; + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: originalContent }], + meta: { toolName: 'test', toolCallId: 'call-1', success: true }, + }; + await contextManager.addToolResult('call-1', 'test', sanitizedResult); + + // Mark it as compacted (pruned) + const history = await contextManager.getHistory(); + const toolMessageId = history.find((m) => m.role === 'tool')?.id; + expect(toolMessageId).toBeDefined(); + await contextManager.markMessagesAsCompacted([toolMessageId!]); + + const result = await contextManager.prepareHistory(); + + // Verify transformation happened + expect(result.stats.prunedToolCount).toBe(1); + const toolMsg = result.preparedHistory.find((m) => m.role === 'tool'); + expect(toolMsg?.content).toEqual([ + { type: 'text', text: '[Old tool result content cleared]' }, + ]); + + // Verify original content is NOT in prepared history + const toolMsgText = (toolMsg?.content as any)?.[0]?.text; + expect(toolMsgText).not.toContain(originalContent); + }); + + it('should not transform non-pruned tool messages', async () => { + const originalContent = 'This tool output should remain intact'; + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: originalContent }], + meta: { toolName: 'test', toolCallId: 'call-1', success: true }, + }; + await contextManager.addToolResult('call-1', 'test', sanitizedResult); + + // Don't mark as compacted + const result = await contextManager.prepareHistory(); + + expect(result.stats.prunedToolCount).toBe(0); + const toolMsg = result.preparedHistory.find((m) => m.role === 'tool'); + expect((toolMsg?.content as any)?.[0]?.text).toBe(originalContent); + }); + }); + + describe('getContextTokenEstimate', () => { + const mockContributorContext = createMockContributorContext(); + const mockTools = { + 'test-tool': { + name: 'test-tool', + description: 'A test tool', + parameters: { type: 'object', properties: {} }, + }, + }; + + it('should calculate total as sum of breakdown components', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + const result = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + const calculatedTotal = + result.breakdown.systemPrompt + + result.breakdown.tools.total + + result.breakdown.messages; + + expect(result.estimated).toBe(calculatedTotal); + expect(result.actual).toBeNull(); // No LLM call made + }); + + it('should return actual tokens when set', async () => { + contextManager.setLastActualInputTokens(12500); + + const result = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + expect(result.actual).toBe(12500); + }); + + it('should reduce estimate when tool messages are pruned', async () => { + // Add a tool result with substantial content (~250 tokens) + const sanitizedResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'A'.repeat(1000) }], + meta: { toolName: 'test', toolCallId: 'call-1', success: true }, + }; + await contextManager.addToolResult('call-1', 'test', sanitizedResult); + + const beforePrune = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Mark the tool message as compacted + const history = await contextManager.getHistory(); + const toolMessageId = history.find((m) => m.role === 'tool')?.id; + await contextManager.markMessagesAsCompacted([toolMessageId!]); + + const afterPrune = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Estimate should be significantly lower (placeholder ~10 tokens vs ~250) + expect(afterPrune.breakdown.messages).toBeLessThan(beforePrune.breakdown.messages); + expect(beforePrune.breakdown.messages - afterPrune.breakdown.messages).toBeGreaterThan( + 200 + ); + expect(afterPrune.stats.prunedToolCount).toBe(1); + }); + + it('should use actuals-based formula when all actuals are available', async () => { + // Add initial message + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + // Simulate LLM call completing - set all actuals + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(200); + await contextManager.recordLastCallMessageCount(); + + // Add assistant response (simulating what happens after LLM call) + await contextManager.addAssistantMessage('Hi there!', []); + + // Add a "new" message (tool result) + const toolResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Tool output here' }], + meta: { toolName: 'test', toolCallId: 'call-1', success: true }, + }; + await contextManager.addToolResult('call-1', 'test', toolResult); + + const result = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Should have calculationBasis with method 'actuals' + expect(result.calculationBasis).toBeDefined(); + expect(result.calculationBasis?.method).toBe('actuals'); + expect(result.calculationBasis?.lastInputTokens).toBe(5000); + expect(result.calculationBasis?.lastOutputTokens).toBe(200); + expect(result.calculationBasis?.newMessagesEstimate).toBeGreaterThan(0); + + // The estimated total should be: lastInput + lastOutput + newMessagesEstimate + const expectedTotal = 5000 + 200 + (result.calculationBasis?.newMessagesEstimate ?? 0); + expect(result.estimated).toBe(expectedTotal); + }); + + it('should fall back to pure estimation when actuals not available', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + // Don't set any actuals + const result = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Should have calculationBasis with method 'estimate' + expect(result.calculationBasis).toBeDefined(); + expect(result.calculationBasis?.method).toBe('estimate'); + expect(result.calculationBasis?.lastInputTokens).toBeUndefined(); + expect(result.calculationBasis?.lastOutputTokens).toBeUndefined(); + }); + }); + + describe('lastActualOutputTokens', () => { + it('should return null initially and store/update values', () => { + expect(contextManager.getLastActualOutputTokens()).toBeNull(); + + contextManager.setLastActualOutputTokens(300); + expect(contextManager.getLastActualOutputTokens()).toBe(300); + + contextManager.setLastActualOutputTokens(500); + expect(contextManager.getLastActualOutputTokens()).toBe(500); + }); + }); + + describe('lastCallMessageCount', () => { + it('should return null initially', () => { + expect(contextManager.getLastCallMessageCount()).toBeNull(); + }); + + it('should record current history length', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await contextManager.addAssistantMessage('Hi!', []); + + await contextManager.recordLastCallMessageCount(); + + expect(contextManager.getLastCallMessageCount()).toBe(2); + }); + + it('should update when new messages are added and recorded again', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await contextManager.recordLastCallMessageCount(); + expect(contextManager.getLastCallMessageCount()).toBe(1); + + await contextManager.addAssistantMessage('Hi!', []); + await contextManager.recordLastCallMessageCount(); + expect(contextManager.getLastCallMessageCount()).toBe(2); + }); + }); + + describe('resetActualTokenTracking', () => { + it('should reset all tracking values to null', async () => { + // Set all values + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(300); + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await contextManager.recordLastCallMessageCount(); + + // Verify they're set + expect(contextManager.getLastActualInputTokens()).toBe(5000); + expect(contextManager.getLastActualOutputTokens()).toBe(300); + expect(contextManager.getLastCallMessageCount()).toBe(1); + + // Reset + contextManager.resetActualTokenTracking(); + + // Verify all are null + expect(contextManager.getLastActualInputTokens()).toBeNull(); + expect(contextManager.getLastActualOutputTokens()).toBeNull(); + expect(contextManager.getLastCallMessageCount()).toBeNull(); + }); + + it('should cause estimation to fall back to pure estimation', async () => { + const mockContributorContext = createMockContributorContext(); + const mockTools = {}; + + // Set actuals + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(200); + await contextManager.recordLastCallMessageCount(); + + // Verify actuals-based method + const beforeReset = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + expect(beforeReset.calculationBasis?.method).toBe('actuals'); + + // Reset + contextManager.resetActualTokenTracking(); + + // Verify fallback to pure estimation + const afterReset = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + expect(afterReset.calculationBasis?.method).toBe('estimate'); + }); + }); + + describe('getEstimatedNextInputTokens', () => { + const mockTools = { + 'test-tool': { + name: 'test-tool', + description: 'A test tool', + parameters: { type: 'object', properties: {} }, + }, + }; + + it('should calculate formula exactly: lastInput + lastOutput + newMessagesEstimate', async () => { + // Setup: one message BEFORE recording + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + // Simulate LLM call: set actuals and record boundary + const LAST_INPUT = 5000; + const LAST_OUTPUT = 200; + contextManager.setLastActualInputTokens(LAST_INPUT); + contextManager.setLastActualOutputTokens(LAST_OUTPUT); + await contextManager.recordLastCallMessageCount(); // count = 1 + + // Add NEW messages after the boundary (these are the "new" messages) + // Use a known string length so we can predict token estimate + const newMessageText = 'A'.repeat(400); // ~100 tokens with length/4 heuristic + await contextManager.addAssistantMessage(newMessageText, []); + + const { preparedHistory } = await contextManager.prepareHistory(); + const systemPrompt = 'System'; + + const estimated = await contextManager.getEstimatedNextInputTokens( + systemPrompt, + preparedHistory, + mockTools + ); + + // Get the actual newMessagesEstimate by checking the raw history + const history = await contextManager.getHistory(); + const newMessages = history.slice(1); // slice from lastCallMessageCount (1) + expect(newMessages.length).toBe(1); // Just the assistant message we added + + // The formula should be: LAST_INPUT + LAST_OUTPUT + newMessagesEstimate + // We can verify the formula is applied by checking the result is exactly this + const { estimateMessagesTokens } = await import('./utils.js'); + const expectedNewEstimate = estimateMessagesTokens(newMessages); + const expectedTotal = LAST_INPUT + LAST_OUTPUT + expectedNewEstimate; + + expect(estimated).toBe(expectedTotal); + }); + + it('should only count messages AFTER lastCallMessageCount as new', async () => { + // Add messages BEFORE recording + await contextManager.addUserMessage([{ type: 'text', text: 'Message 1' }]); + await contextManager.addAssistantMessage('Response 1', []); + + // Record boundary at 2 messages + contextManager.setLastActualInputTokens(10000); + contextManager.setLastActualOutputTokens(500); + await contextManager.recordLastCallMessageCount(); + expect(contextManager.getLastCallMessageCount()).toBe(2); + + // Add messages AFTER recording - these should be the only "new" messages + await contextManager.addUserMessage([{ type: 'text', text: 'Message 2' }]); + + const { preparedHistory } = await contextManager.prepareHistory(); + const systemPrompt = 'System'; + + // Get estimate with actuals + const estimated = await contextManager.getEstimatedNextInputTokens( + systemPrompt, + preparedHistory, + mockTools + ); + + // Verify by calculating expected value + const history = await contextManager.getHistory(); + expect(history.length).toBe(3); // 2 before + 1 after + + const newMessages = history.slice(2); // Only messages after lastCallMessageCount + expect(newMessages.length).toBe(1); // Just "Message 2" + + const { estimateMessagesTokens } = await import('./utils.js'); + const newEstimate = estimateMessagesTokens(newMessages); + const expectedTotal = 10000 + 500 + newEstimate; + + expect(estimated).toBe(expectedTotal); + }); + + it('should return zero newMessagesEstimate when no new messages', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + // Record boundary at current history length + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(200); + await contextManager.recordLastCallMessageCount(); + + // Don't add any new messages + + const { preparedHistory } = await contextManager.prepareHistory(); + const systemPrompt = 'System'; + + const estimated = await contextManager.getEstimatedNextInputTokens( + systemPrompt, + preparedHistory, + mockTools + ); + + // Should be exactly lastInput + lastOutput + 0 + expect(estimated).toBe(5000 + 200); + }); + + it('should fall back to pure estimation when only lastInput is set', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + // Only set lastInput, not lastOutput or lastCallMessageCount + contextManager.setLastActualInputTokens(5000); + // lastOutput is null, lastCallMessageCount is null + + const mockContributorContext = createMockContributorContext(); + + // Should fall back to pure estimation since not all actuals are set + const fullEstimate = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Verify it's using pure estimation (not actuals-based) + expect(fullEstimate.calculationBasis?.method).toBe('estimate'); + + // The estimated value should NOT be based on the lastInput we set (5000) + // It should be a pure estimate based on system + tools + messages + // If it were using actuals, it would be much larger (5000 + something) + expect(fullEstimate.estimated).toBeLessThan(1000); // Pure estimate of small message + }); + + it('should fall back to pure estimation when lastCallMessageCount is null', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + // Set input and output but NOT message count + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(200); + // Don't call recordLastCallMessageCount() + + const mockContributorContext = createMockContributorContext(); + + const fullEstimate = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Should fall back to pure estimation + expect(fullEstimate.calculationBasis?.method).toBe('estimate'); + }); + + it('should return same result as getContextTokenEstimate for consistency', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(200); + await contextManager.recordLastCallMessageCount(); + + const { preparedHistory } = await contextManager.prepareHistory(); + const systemPrompt = 'You are a helpful assistant.'; + const mockContributorContext = createMockContributorContext(); + + const estimatedViaMethod = await contextManager.getEstimatedNextInputTokens( + systemPrompt, + preparedHistory, + mockTools + ); + + const fullEstimate = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Both should use the same formula and return the same total + expect(estimatedViaMethod).toBe(fullEstimate.estimated); + expect(fullEstimate.calculationBasis?.method).toBe('actuals'); + }); + + it('should include all messages since last successful call when tracking is not updated (cancellation scenario)', async () => { + // Scenario: User sends message, LLM responds successfully, then user sends another + // message but the LLM call gets cancelled. On cancellation, we don't update tracking. + // The next estimate should include all content since the last successful call. + // + // In real execution, the sequence is: + // 1. User message added to history + // 2. LLM stream starts, assistant message added on first delta + // 3. Stream finishes with usage data + // 4. We set lastInputTokens, lastOutputTokens + // 5. We call recordLastCallMessageCount() - captures count AFTER assistant is added + // + // So the boundary is set AFTER the assistant response, not before. + + // === Successful call 1 === + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + // Assistant response added during stream (before we record) + await contextManager.addAssistantMessage('Hi there! How can I help?', []); + // Now we record tracking (simulating what happens after stream completes) + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(200); + await contextManager.recordLastCallMessageCount(); // boundary = 2 (user1 + assistant1) + + // Get estimate after successful call - should have zero newMessagesEstimate + const mockContributorContext = createMockContributorContext(); + const estimateAfterSuccess = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // newMessagesEstimate should be 0 - no messages after boundary + expect(estimateAfterSuccess.calculationBasis?.newMessagesEstimate).toBe(0); + expect(contextManager.getLastCallMessageCount()).toBe(2); + + // === Cancelled call 2 === + // User sends new message + await contextManager.addUserMessage([ + { type: 'text', text: 'Tell me a long story about dragons' }, + ]); + + // Partial assistant response gets saved to history (what happens on cancel) + await contextManager.addAssistantMessage( + 'Once upon a time, in a land far away, there lived a magnificent dragon...', + [] + ); + + // IMPORTANT: On cancellation, we do NOT update tracking: + // - No setLastActualInputTokens() + // - No setLastActualOutputTokens() + // - No recordLastCallMessageCount() + + // Get estimate after cancelled call + const estimateAfterCancel = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Should still use actuals-based method (from successful call 1) + expect(estimateAfterCancel.calculationBasis?.method).toBe('actuals'); + expect(estimateAfterCancel.calculationBasis?.lastInputTokens).toBe(5000); + expect(estimateAfterCancel.calculationBasis?.lastOutputTokens).toBe(200); + + // newMessagesEstimate should now include: + // - User message from call 2 + // - Partial assistant response from cancelled call 2 + const afterCancelNewMsgs = estimateAfterCancel.calculationBasis?.newMessagesEstimate; + expect(afterCancelNewMsgs).toBeDefined(); + expect(afterCancelNewMsgs!).toBeGreaterThan(0); + + // Verify the total estimate increased + expect(estimateAfterCancel.estimated).toBeGreaterThan(estimateAfterSuccess.estimated); + + // Verify by checking history and expected new messages + const history = await contextManager.getHistory(); + expect(history.length).toBe(4); // user1, assistant1, user2, assistant2-partial + + // lastCallMessageCount should still be 2 (from successful call) + expect(contextManager.getLastCallMessageCount()).toBe(2); + + // So newMessages should be history.slice(2) = [user2, assistant2-partial] + const { estimateMessagesTokens } = await import('./utils.js'); + const expectedNewMessages = history.slice(2); + expect(expectedNewMessages.length).toBe(2); + const expectedNewEstimate = estimateMessagesTokens(expectedNewMessages); + expect(afterCancelNewMsgs).toBe(expectedNewEstimate); + }); + + it('should accumulate multiple cancelled calls in newMessagesEstimate', async () => { + // Scenario: Successful call, then 2 cancelled calls in a row + // All cancelled content should accumulate in newMessagesEstimate + + // === Successful call === + await contextManager.addUserMessage([{ type: 'text', text: 'Hi' }]); + await contextManager.addAssistantMessage('Hello!', []); // Added during stream + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(100); + await contextManager.recordLastCallMessageCount(); // boundary = 2 (after assistant added) + + // === Cancelled call 1 === + await contextManager.addUserMessage([ + { type: 'text', text: 'First cancelled request' }, + ]); + await contextManager.addAssistantMessage('Starting to respond...', []); + // No tracking update (cancelled) + + // === Cancelled call 2 === + await contextManager.addUserMessage([ + { type: 'text', text: 'Second cancelled request' }, + ]); + await contextManager.addAssistantMessage('Another partial...', []); + // No tracking update (cancelled) + + const mockContributorContext = createMockContributorContext(); + const estimate = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Should still use actuals from the one successful call + expect(estimate.calculationBasis?.method).toBe('actuals'); + expect(estimate.calculationBasis?.lastInputTokens).toBe(5000); + expect(estimate.calculationBasis?.lastOutputTokens).toBe(100); + + // History: [user1, assistant1, user2, assistant2-partial, user3, assistant3-partial] + const history = await contextManager.getHistory(); + expect(history.length).toBe(6); + expect(contextManager.getLastCallMessageCount()).toBe(2); + + // newMessages = history.slice(2) = 4 messages from cancelled calls + const { estimateMessagesTokens } = await import('./utils.js'); + const newMessages = history.slice(2); + expect(newMessages.length).toBe(4); + expect(estimate.calculationBasis?.newMessagesEstimate).toBe( + estimateMessagesTokens(newMessages) + ); + }); + + it('should self-correct after successful call following cancellation', async () => { + // Scenario: Success → Cancel → Success + // The second success should provide fresh actuals + + // === Successful call 1 === + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await contextManager.addAssistantMessage('Hi!', []); + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(100); + await contextManager.recordLastCallMessageCount(); // boundary = 2 + + // === Cancelled call === + await contextManager.addUserMessage([{ type: 'text', text: 'Tell me a story' }]); + await contextManager.addAssistantMessage('Once upon a time...', []); + // No tracking update + + // Verify we're estimating the cancelled content + const mockContributorContext = createMockContributorContext(); + const estimateAfterCancel = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + expect(estimateAfterCancel.calculationBasis?.method).toBe('actuals'); + expect(estimateAfterCancel.calculationBasis?.lastInputTokens).toBe(5000); + expect(estimateAfterCancel.calculationBasis?.newMessagesEstimate).toBeGreaterThan(0); + + // === Successful call 2 === + await contextManager.addUserMessage([{ type: 'text', text: 'Just say hi' }]); + await contextManager.addAssistantMessage('Hi again!', []); + // This time we DO update tracking (successful call) + contextManager.setLastActualInputTokens(5800); // Fresh actual from API + contextManager.setLastActualOutputTokens(50); + await contextManager.recordLastCallMessageCount(); // boundary = 6 + + // Now estimate should use fresh actuals, newMessagesEstimate = 0 + const estimateAfterSuccess = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + expect(estimateAfterSuccess.calculationBasis?.method).toBe('actuals'); + expect(estimateAfterSuccess.calculationBasis?.lastInputTokens).toBe(5800); + expect(estimateAfterSuccess.calculationBasis?.lastOutputTokens).toBe(50); + expect(estimateAfterSuccess.calculationBasis?.newMessagesEstimate).toBe(0); + + // History should have 6 messages, boundary at 6 + const history = await contextManager.getHistory(); + expect(history.length).toBe(6); + expect(contextManager.getLastCallMessageCount()).toBe(6); + }); + + it('should fall back to pure estimation when first call is cancelled', async () => { + // Scenario: Very first call gets cancelled, no prior actuals exist + + // === First call - cancelled === + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await contextManager.addAssistantMessage('Hi...', []); // Partial response + // No tracking update - all values remain null + + expect(contextManager.getLastActualInputTokens()).toBeNull(); + expect(contextManager.getLastActualOutputTokens()).toBeNull(); + expect(contextManager.getLastCallMessageCount()).toBeNull(); + + const mockContributorContext = createMockContributorContext(); + const estimate = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Should fall back to pure estimation since no actuals available + expect(estimate.calculationBasis?.method).toBe('estimate'); + expect(estimate.calculationBasis?.lastInputTokens).toBeUndefined(); + expect(estimate.calculationBasis?.lastOutputTokens).toBeUndefined(); + }); + + it('should update tracking on each successful step in multi-step tool flow', async () => { + // Scenario: LLM makes tool call (step 1), tool executes, LLM responds (step 2) + // Tracking is updated after each step + + // === Step 1: User message, LLM calls tool === + await contextManager.addUserMessage([{ type: 'text', text: 'What is the weather?' }]); + await contextManager.addAssistantMessage('', [ + { + id: 'tool-1', + type: 'function' as const, + function: { name: 'get_weather', arguments: '{"city":"NYC"}' }, + }, + ]); + // Step 1 completes with usage data + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(50); + await contextManager.recordLastCallMessageCount(); // boundary = 2 + + // Verify state after step 1 + expect(contextManager.getLastCallMessageCount()).toBe(2); + + // === Tool executes, result added === + const toolResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Weather in NYC: Sunny, 72°F' }], + meta: { toolName: 'get_weather', toolCallId: 'tool-1', success: true }, + }; + await contextManager.addToolResult('tool-1', 'get_weather', toolResult); + + // Before step 2, estimate should include tool result in newMessagesEstimate + const mockContributorContext = createMockContributorContext(); + const estimateBeforeStep2 = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + expect(estimateBeforeStep2.calculationBasis?.method).toBe('actuals'); + expect(estimateBeforeStep2.calculationBasis?.newMessagesEstimate).toBeGreaterThan(0); + + // === Step 2: LLM responds with final answer === + await contextManager.addAssistantMessage('The weather in NYC is sunny and 72°F!', []); + // Step 2 completes with fresh usage data + contextManager.setLastActualInputTokens(5200); // Includes tool result now + contextManager.setLastActualOutputTokens(30); + await contextManager.recordLastCallMessageCount(); // boundary = 4 + + // Verify state after step 2 + expect(contextManager.getLastCallMessageCount()).toBe(4); + + // After step 2, newMessagesEstimate should be 0 + const estimateAfterStep2 = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + expect(estimateAfterStep2.calculationBasis?.method).toBe('actuals'); + expect(estimateAfterStep2.calculationBasis?.lastInputTokens).toBe(5200); + expect(estimateAfterStep2.calculationBasis?.lastOutputTokens).toBe(30); + expect(estimateAfterStep2.calculationBasis?.newMessagesEstimate).toBe(0); + + // History should have 4 messages + const history = await contextManager.getHistory(); + expect(history.length).toBe(4); // user, assistant-tool-call, tool-result, assistant-final + }); + + it('should handle tool call flow with cancellation during tool execution', async () => { + // Scenario: LLM makes tool call, then cancelled before tool completes + // Tool result is persisted as "cancelled" + + // === Step 1: LLM calls tool (successful step) === + await contextManager.addUserMessage([{ type: 'text', text: 'What is the weather?' }]); + // Assistant message with tool call added during stream + await contextManager.addAssistantMessage('', [ + { + id: 'tool-1', + type: 'function' as const, + function: { name: 'get_weather', arguments: '{"city":"NYC"}' }, + }, + ]); + // Step 1 completes with usage data + contextManager.setLastActualInputTokens(5000); + contextManager.setLastActualOutputTokens(50); + await contextManager.recordLastCallMessageCount(); // boundary = 2 + + // === Cancelled during tool execution === + // Tool result is added as "cancelled" by persistCancelledToolResults() + const cancelledResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Cancelled by user' }], + meta: { toolName: 'get_weather', toolCallId: 'tool-1', success: false }, + }; + await contextManager.addToolResult('tool-1', 'get_weather', cancelledResult); + // No tracking update since overall turn was cancelled + + const mockContributorContext = createMockContributorContext(); + const estimate = await contextManager.getContextTokenEstimate( + mockContributorContext, + mockTools + ); + + // Should use actuals from the successful tool-call step + expect(estimate.calculationBasis?.method).toBe('actuals'); + expect(estimate.calculationBasis?.lastInputTokens).toBe(5000); + expect(estimate.calculationBasis?.lastOutputTokens).toBe(50); + + // newMessagesEstimate should include the cancelled tool result + const history = await contextManager.getHistory(); + expect(history.length).toBe(3); // user, assistant-tool-call, tool-result + expect(contextManager.getLastCallMessageCount()).toBe(2); + + const { estimateMessagesTokens } = await import('./utils.js'); + const newMessages = history.slice(2); // Just the tool result + expect(newMessages.length).toBe(1); + expect(estimate.calculationBasis?.newMessagesEstimate).toBe( + estimateMessagesTokens(newMessages) + ); + }); + }); +}); diff --git a/dexto/packages/core/src/context/manager.ts b/dexto/packages/core/src/context/manager.ts new file mode 100644 index 00000000..97228b11 --- /dev/null +++ b/dexto/packages/core/src/context/manager.ts @@ -0,0 +1,1204 @@ +import { randomUUID } from 'crypto'; +import { VercelMessageFormatter } from '@core/llm/formatters/vercel.js'; +import { LLMContext } from '../llm/types.js'; +import type { InternalMessage, AssistantMessage, ToolCall } from './types.js'; +import { isSystemMessage, isUserMessage, isAssistantMessage, isToolMessage } from './types.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import { eventBus } from '../events/index.js'; +import { + expandBlobReferences, + isLikelyBase64String, + filterCompacted, + estimateContextTokens, + estimateMessagesTokens, +} from './utils.js'; +import type { SanitizedToolResult } from './types.js'; +import { DynamicContributorContext } from '../systemPrompt/types.js'; +import { SystemPromptManager } from '../systemPrompt/manager.js'; +import { IConversationHistoryProvider } from '@core/session/history/types.js'; +import { ContextError } from './errors.js'; +import { ValidatedLLMConfig } from '../llm/schemas.js'; + +/** + * Manages conversation history and provides message formatting capabilities for the LLM context. + * The ContextManager is responsible for: + * - Validating and storing conversation messages via the history provider + * - Managing the system prompt + * - Formatting messages for specific LLM providers through an injected formatter + * - Providing access to conversation history + * + * Note: All conversation history is stored and retrieved via the injected ConversationHistoryProvider. + * The ContextManager does not maintain an internal history cache. + * Token counting is handled by the LLM API response, not local estimation. + * + * @template TMessage The message type for the specific LLM provider (e.g., MessageParam, ChatCompletionMessageParam, ModelMessage) + */ +export class ContextManager { + /** + * The validated LLM configuration. + */ + private llmConfig: ValidatedLLMConfig; + + /** + * SystemPromptManager used to generate/manage the system prompt + */ + private systemPromptManager: SystemPromptManager; + + /** + * Formatter used to convert internal messages to LLM-specific format + */ + private formatter: VercelMessageFormatter; + + /** + * Maximum number of tokens allowed in the conversation (if specified) + */ + private maxInputTokens: number; + + /** + * Last known actual input token count from the LLM API response. + * Updated after each LLM call. Used by /context for accurate reporting. + */ + private lastActualInputTokens: number | null = null; + + /** + * Last known actual output token count from the LLM API response. + * Updated after each LLM call. Used in the context estimation formula: + * estimatedNextInput = lastInputTokens + lastOutputTokens + newMessagesEstimate + */ + private lastActualOutputTokens: number | null = null; + + /** + * Message count at the time of the last LLM call. + * Used to identify which messages are "new" since the last call. + * Messages after this index are estimated with length/4 heuristic. + */ + private lastCallMessageCount: number | null = null; + + private historyProvider: IConversationHistoryProvider; + private readonly sessionId: string; + + /** + * ResourceManager for resolving blob references in message content. + * Blob references like @blob:abc123 are resolved to actual data + * before passing messages to the LLM formatter. + */ + private resourceManager: import('../resources/index.js').ResourceManager; + + private logger: IDextoLogger; + + /** + * Creates a new ContextManager instance + * @param llmConfig The validated LLM configuration. + * @param formatter Formatter implementation for the target LLM provider + * @param systemPromptManager SystemPromptManager instance for the conversation + * @param maxInputTokens Maximum token limit for the conversation history. + * @param historyProvider Session-scoped ConversationHistoryProvider instance for managing conversation history + * @param sessionId Unique identifier for the conversation session (readonly, for debugging) + * @param resourceManager ResourceManager for resolving blob references in messages + * @param logger Logger instance for logging + */ + constructor( + llmConfig: ValidatedLLMConfig, + formatter: VercelMessageFormatter, + systemPromptManager: SystemPromptManager, + maxInputTokens: number, + historyProvider: IConversationHistoryProvider, + sessionId: string, + resourceManager: import('../resources/index.js').ResourceManager, + logger: IDextoLogger + ) { + this.llmConfig = llmConfig; + this.formatter = formatter; + this.systemPromptManager = systemPromptManager; + this.maxInputTokens = maxInputTokens; + this.historyProvider = historyProvider; + this.sessionId = sessionId; + this.resourceManager = resourceManager; + this.logger = logger.createChild(DextoLogComponent.CONTEXT); + + this.logger.debug( + `ContextManager: Initialized for session ${sessionId} - history will be managed by ${historyProvider.constructor.name}` + ); + } + + /** + * Get the ResourceManager instance + */ + public getResourceManager(): import('../resources/index.js').ResourceManager { + return this.resourceManager; + } + + /** + * Process user input data - store as blob if large, otherwise return as-is. + * Returns either the original data or a blob reference (@blob:id). + */ + private async processUserInput( + data: string | Uint8Array | Buffer | ArrayBuffer | URL, + metadata: { + mimeType: string; + originalName?: string; + source?: 'user' | 'system'; + } + ): Promise { + const blobService = this.resourceManager.getBlobStore(); + + // Estimate data size to decide if we should store as blob + let shouldStoreAsBlob = false; + let estimatedSize = 0; + + if (typeof data === 'string') { + if (data.startsWith('data:')) { + // Data URI - estimate base64 size + const commaIndex = data.indexOf(','); + if (commaIndex !== -1) { + const base64Data = data.substring(commaIndex + 1); + estimatedSize = Math.floor((base64Data.length * 3) / 4); + } + } else if (data.length > 100 && data.match(/^[A-Za-z0-9+/=]+$/)) { + // Likely base64 string + estimatedSize = Math.floor((data.length * 3) / 4); + } else { + estimatedSize = Buffer.byteLength(data, 'utf8'); + } + } else if (data instanceof Buffer || data instanceof Uint8Array) { + estimatedSize = data.length; + } else if (data instanceof ArrayBuffer) { + estimatedSize = data.byteLength; + } else if (data instanceof URL) { + // URLs are small, don't store as blob + return data; + } + + const isLikelyBinary = + metadata.mimeType.startsWith('image/') || + metadata.mimeType.startsWith('audio/') || + metadata.mimeType.startsWith('video/') || + metadata.mimeType === 'application/pdf'; + + // Store all binary attachments (images/audio/video/pdf) or anything over 5KB + shouldStoreAsBlob = isLikelyBinary || estimatedSize > 5 * 1024; + + if (shouldStoreAsBlob) { + try { + const blobInput = + typeof data === 'string' && + !data.startsWith('data:') && + !isLikelyBase64String(data) && + !isLikelyBinary + ? Buffer.from(data, 'utf-8') + : data; + + const blobRef = await blobService.store(blobInput, { + mimeType: metadata.mimeType, + originalName: metadata.originalName, + source: metadata.source || 'user', + }); + + this.logger.info( + `Stored user input as blob: ${blobRef.uri} (${estimatedSize} bytes, ${metadata.mimeType})` + ); + + // Emit event to invalidate resource cache so uploaded images appear in @ autocomplete + eventBus.emit('resource:cache-invalidated', { + resourceUri: blobRef.uri, + serverName: 'internal', + action: 'blob_stored', + }); + + return `@${blobRef.uri}`; // Return @blob:id reference for ResourceManager + } catch (error) { + this.logger.warn(`Failed to store user input as blob: ${String(error)}`); + // Fallback to storing original data + return data; + } + } + + return data; + } + + /** + * Returns the configured maximum number of input tokens for the conversation. + */ + getMaxInputTokens(): number { + return this.maxInputTokens; + } + + /** + * Returns the last known actual input token count from the LLM API. + * Returns null if no LLM call has been made yet. + */ + getLastActualInputTokens(): number | null { + return this.lastActualInputTokens; + } + + /** + * Updates the last known actual input token count. + * Called after each LLM response with the actual usage from the API. + */ + setLastActualInputTokens(tokens: number): void { + this.lastActualInputTokens = tokens; + this.logger.debug(`Updated lastActualInputTokens: ${tokens}`); + } + + /** + * Returns the last known actual output token count from the LLM API. + * Returns null if no LLM call has been made yet. + */ + getLastActualOutputTokens(): number | null { + return this.lastActualOutputTokens; + } + + /** + * Updates the last known actual output token count. + * Called after each LLM response with the actual usage from the API. + */ + setLastActualOutputTokens(tokens: number): void { + this.lastActualOutputTokens = tokens; + this.logger.debug(`Updated lastActualOutputTokens: ${tokens}`); + } + + /** + * Returns the message count at the time of the last LLM call. + * Returns null if no LLM call has been made yet. + */ + getLastCallMessageCount(): number | null { + return this.lastCallMessageCount; + } + + /** + * Records the current message count after an LLM call completes. + * This marks the boundary for "new messages" calculation. + */ + async recordLastCallMessageCount(): Promise { + const history = await this.historyProvider.getHistory(); + this.lastCallMessageCount = history.length; + this.logger.debug(`Recorded lastCallMessageCount: ${this.lastCallMessageCount}`); + } + + /** + * Resets the actual token tracking state. + * Called after compaction since the context has fundamentally changed. + */ + resetActualTokenTracking(): void { + this.lastActualInputTokens = null; + this.lastActualOutputTokens = null; + this.lastCallMessageCount = null; + this.logger.debug('Reset actual token tracking state (after compaction)'); + } + + // ============= HISTORY PREPARATION ============= + + /** + * Placeholder text used when tool outputs are pruned. + * Shared constant to ensure consistency between preparation and estimation. + */ + private static readonly PRUNED_TOOL_PLACEHOLDER = '[Old tool result content cleared]'; + + /** + * Prepares conversation history for LLM consumption. + * This is the single source of truth for history transformation logic. + * + * Transformations applied: + * 1. filterCompacted - Remove pre-summary messages (messages before the most recent summary) + * 2. Transform pruned tool messages - Replace compactedAt messages with placeholder text + * + * Used by both: + * - getFormattedMessagesForLLM() - For actual LLM calls + * - getContextTokenEstimate() - For /context command estimation + * + * @returns Prepared history and statistics about the transformations + */ + async prepareHistory(): Promise<{ + preparedHistory: InternalMessage[]; + stats: { + /** Total messages in raw history */ + originalCount: number; + /** Messages after filterCompacted (removed pre-summary) */ + filteredCount: number; + /** Messages with compactedAt that were transformed to placeholders */ + prunedToolCount: number; + }; + }> { + const fullHistory = await this.historyProvider.getHistory(); + const originalCount = fullHistory.length; + + // Step 1: Filter compacted (remove pre-summary messages) + let history = filterCompacted(fullHistory); + const filteredCount = history.length; + + if (filteredCount < originalCount) { + this.logger.debug( + `prepareHistory: filterCompacted reduced from ${originalCount} to ${filteredCount} messages` + ); + } + + // Step 2: Transform compacted tool messages to placeholders + // Original content is preserved in storage, placeholder sent to LLM + let prunedToolCount = 0; + history = history.map((msg) => { + if (msg.role === 'tool' && msg.compactedAt) { + prunedToolCount++; + return { + ...msg, + content: [ + { type: 'text' as const, text: ContextManager.PRUNED_TOOL_PLACEHOLDER }, + ], + }; + } + return msg; + }); + + if (prunedToolCount > 0) { + this.logger.debug( + `prepareHistory: Transformed ${prunedToolCount} pruned tool messages to placeholders` + ); + } + + return { + preparedHistory: history, + stats: { + originalCount, + filteredCount, + prunedToolCount, + }, + }; + } + + /** + * Assembles and returns the current system prompt by invoking the SystemPromptManager. + */ + async getSystemPrompt(context: DynamicContributorContext): Promise { + const prompt = await this.systemPromptManager.build(context); + this.logger.debug(`[SystemPrompt] Built system prompt:\n${prompt}`); + return prompt; + } + + /** + * Gets the raw conversation history + * Returns a defensive copy to prevent modification + * + * @returns Promise that resolves to a read-only copy of the conversation history + */ + async getHistory(): Promise> { + const history = await this.historyProvider.getHistory(); + return [...history]; + } + + /** + * Flush any pending history updates to durable storage. + * Should be called at turn boundaries (after streaming completes, on cancel, on error). + * This ensures all message updates are persisted before returning control to the caller. + */ + async flush(): Promise { + await this.historyProvider.flush(); + } + + /** + * Clears the context window without deleting history. + * + * This adds a "context clear" marker to the conversation history. When the + * context is loaded for LLM via getFormattedMessagesWithCompression(), + * filterCompacted() excludes all messages before this marker. + * + * The full history remains in the database for review via /resume or session history. + */ + async clearContext(): Promise { + const clearMarker: InternalMessage = { + id: `clear-${Date.now()}`, + role: 'assistant', + content: [{ type: 'text', text: '[Context cleared]' }], + timestamp: Date.now(), + metadata: { + isSummary: true, + clearedAt: Date.now(), + }, + }; + + await this.addMessage(clearMarker); + this.resetActualTokenTracking(); + this.logger.debug(`Context cleared for session: ${this.sessionId}`); + } + + /** + * Appends text to an existing assistant message. + * Used for streaming responses. + */ + async appendAssistantText(messageId: string, text: string): Promise { + const history = await this.historyProvider.getHistory(); + const messageIndex = history.findIndex((m) => m.id === messageId); + + if (messageIndex === -1) { + throw ContextError.messageNotFound(messageId); + } + + const message = history[messageIndex]; + if (!message) { + throw ContextError.messageNotFound(messageId); + } + + if (message.role !== 'assistant') { + throw ContextError.messageNotAssistant(messageId); + } + + // Append text to content array + if (message.content === null) { + message.content = [{ type: 'text', text }]; + } else if (Array.isArray(message.content)) { + // Find last text part and append, or add new text part + const lastPart = message.content[message.content.length - 1]; + if (lastPart && lastPart.type === 'text') { + lastPart.text += text; + } else { + message.content.push({ type: 'text', text }); + } + } + + await this.historyProvider.updateMessage(message); + } + + /** + * Adds a tool call to an existing assistant message. + * Used for streaming responses. + */ + async addToolCall(messageId: string, toolCall: ToolCall): Promise { + const history = await this.historyProvider.getHistory(); + const messageIndex = history.findIndex((m) => m.id === messageId); + + if (messageIndex === -1) { + throw ContextError.messageNotFound(messageId); + } + + const message = history[messageIndex]; + if (!message) { + throw ContextError.messageNotFound(messageId); + } + + if (message.role !== 'assistant') { + throw ContextError.messageNotAssistant(messageId); + } + + if (!message.toolCalls) { + message.toolCalls = []; + } + + message.toolCalls.push(toolCall); + await this.historyProvider.updateMessage(message); + } + + /** + * Updates an existing assistant message with new properties. + * Used for finalizing streaming responses (e.g. adding token usage). + */ + async updateAssistantMessage( + messageId: string, + updates: Partial + ): Promise { + const history = await this.historyProvider.getHistory(); + const messageIndex = history.findIndex((m) => m.id === messageId); + + if (messageIndex === -1) { + throw ContextError.messageNotFound(messageId); + } + + const message = history[messageIndex]; + if (!message) { + throw ContextError.messageNotFound(messageId); + } + + if (message.role !== 'assistant') { + throw ContextError.messageNotAssistant(messageId); + } + + Object.assign(message, updates); + await this.historyProvider.updateMessage(message); + } + + /** + * Marks tool messages as compacted (pruned). + * Sets the compactedAt timestamp - content transformation happens at format time + * in getFormattedMessagesWithCompression(). Original content is preserved in + * storage for debugging/audit. + * + * Used by TurnExecutor's pruneOldToolOutputs() to reclaim token space + * by marking old tool outputs that are no longer needed for context. + * + * @param messageIds Array of message IDs to mark as compacted + * @returns Number of messages successfully marked + */ + async markMessagesAsCompacted(messageIds: string[]): Promise { + if (messageIds.length === 0) { + return 0; + } + + const history = await this.historyProvider.getHistory(); + const timestamp = Date.now(); + let markedCount = 0; + + for (const messageId of messageIds) { + const message = history.find((m) => m.id === messageId); + + if (!message) { + this.logger.warn(`markMessagesAsCompacted: Message ${messageId} not found`); + continue; + } + + if (message.role !== 'tool') { + this.logger.warn( + `markMessagesAsCompacted: Message ${messageId} is not a tool message (role=${message.role})` + ); + continue; + } + + if (message.compactedAt) { + // Already compacted, skip + continue; + } + + message.compactedAt = timestamp; + await this.historyProvider.updateMessage(message); + markedCount++; + } + + if (markedCount > 0) { + this.logger.debug( + `markMessagesAsCompacted: Marked ${markedCount} messages as compacted` + ); + } + + return markedCount; + } + + /** + * Adds a message to the conversation history. + * Performs validation based on message role and required fields. + * Note: Compression based on token limits is applied lazily when calling `getFormattedMessages`, not immediately upon adding. + * + * @param message The message to add to the history + * @throws Error if message validation fails + */ + async addMessage(message: InternalMessage): Promise { + switch (message.role) { + case 'user': + // User messages must have non-empty content array + if (!Array.isArray(message.content) || message.content.length === 0) { + throw ContextError.userMessageContentInvalid(); + } + break; + + case 'assistant': + // Content can be null if toolCalls are present, but one must exist + if ( + message.content === null && + (!message.toolCalls || message.toolCalls.length === 0) + ) { + throw ContextError.assistantMessageContentOrToolsRequired(); + } + if (message.toolCalls) { + if ( + !Array.isArray(message.toolCalls) || + message.toolCalls.some( + (tc) => !tc.id || !tc.function?.name || !tc.function?.arguments + ) + ) { + throw ContextError.assistantMessageToolCallsInvalid(); + } + } + + // Enrich assistant messages with LLM config metadata + message.provider = this.llmConfig.provider; + message.model = this.llmConfig.model; + break; + + case 'tool': + if (!message.toolCallId || !message.name || message.content === null) { + throw ContextError.toolMessageFieldsMissing(); + } + break; + + case 'system': { + // System messages should be handled via SystemPromptManager, not added to history + this.logger.warn( + 'ContextManager: Adding system message directly to history. Use SystemPromptManager instead.' + ); + // Extract text from content array for validation + const textContent = message.content + ?.filter((p): p is import('./types.js').TextPart => p.type === 'text') + .map((p) => p.text) + .join(''); + if (!textContent || textContent.trim() === '') { + throw ContextError.systemMessageContentInvalid(); + } + break; + } + } + + // Generate ID and timestamp if not provided + if (!message.id) { + message.id = randomUUID(); + } + if (!message.timestamp) { + message.timestamp = Date.now(); + } + + this.logger.debug( + `ContextManager: Adding message to history provider: ${JSON.stringify(message, null, 2)}` + ); + + // Save to history provider + await this.historyProvider.saveMessage(message); + + // Get updated history for logging + const history = await this.historyProvider.getHistory(); + this.logger.debug(`ContextManager: History now contains ${history.length} messages`); + + // Note: Compression is currently handled lazily in getFormattedMessages + } + + /** + * Adds a user message to the conversation. + * Supports multiple images and files via ContentPart[]. + * + * @param content Array of content parts (text, images, files) + * @throws Error if content is empty or invalid + */ + async addUserMessage(content: import('./types.js').ContentPart[]): Promise { + if (!Array.isArray(content) || content.length === 0) { + throw ContextError.userMessageContentEmpty(); + } + + // Validate at least one text part or attachment exists + const hasText = content.some((p) => p.type === 'text' && p.text.trim() !== ''); + const hasAttachment = content.some((p) => p.type === 'image' || p.type === 'file'); + + if (!hasText && !hasAttachment) { + throw ContextError.userMessageContentEmpty(); + } + + // Process all parts, storing large attachments as blobs + const processedParts: InternalMessage['content'] = []; + + for (const part of content) { + if (part.type === 'text') { + if (part.text.trim()) { + processedParts.push({ type: 'text', text: part.text }); + } + } else if (part.type === 'image') { + const processedImage = await this.processUserInput(part.image, { + mimeType: part.mimeType || 'image/jpeg', + source: 'user', + }); + + processedParts.push({ + type: 'image', + image: processedImage, + mimeType: part.mimeType || 'image/jpeg', + }); + } else if (part.type === 'file') { + const metadata: { + mimeType: string; + originalName?: string; + source?: 'user' | 'system'; + } = { + mimeType: part.mimeType, + source: 'user', + }; + if (part.filename) { + metadata.originalName = part.filename; + } + + const processedData = await this.processUserInput(part.data, metadata); + + processedParts.push({ + type: 'file', + data: processedData, + mimeType: part.mimeType, + ...(part.filename && { filename: part.filename }), + }); + } + } + + // Count parts for logging + const textParts = processedParts.filter((p) => p.type === 'text'); + const imageParts = processedParts.filter((p) => p.type === 'image'); + const fileParts = processedParts.filter((p) => p.type === 'file'); + + this.logger.info('User message received', { + textParts: textParts.length, + imageParts: imageParts.length, + fileParts: fileParts.length, + totalParts: processedParts.length, + }); + + await this.addMessage({ role: 'user', content: processedParts }); + } + + /** + * Adds an assistant message to the conversation + * Can include tool calls if the assistant is requesting tool execution + * + * @param content The assistant's response text (can be null if only tool calls) + * @param toolCalls Optional tool calls requested by the assistant + * @param metadata Optional metadata including token usage, reasoning, and model info + * @throws Error if neither content nor toolCalls are provided + */ + async addAssistantMessage( + content: string | null, + toolCalls?: AssistantMessage['toolCalls'], + metadata?: { + tokenUsage?: AssistantMessage['tokenUsage']; + reasoning?: string; + } + ): Promise { + // Validate that either content or toolCalls is provided + if (content === null && (!toolCalls || toolCalls.length === 0)) { + throw ContextError.assistantMessageContentOrToolsRequired(); + } + // Convert string content to content array + const contentArray: InternalMessage['content'] = + content !== null ? [{ type: 'text', text: content }] : null; + // Further validation happens within addMessage + // addMessage will populate llm config metadata also + await this.addMessage({ + role: 'assistant' as const, + content: contentArray, + ...(toolCalls && toolCalls.length > 0 && { toolCalls }), + ...(metadata?.tokenUsage && { tokenUsage: metadata.tokenUsage }), + ...(metadata?.reasoning && { reasoning: metadata.reasoning }), + }); + } + + /** + * Adds a tool result message to the conversation. + * The result must already be sanitized - this method only persists it. + * + * Success status is read from sanitizedResult.meta.success (single source of truth). + * + * @param toolCallId ID of the tool call this result is responding to + * @param name Name of the tool that executed + * @param sanitizedResult The already-sanitized result to store (includes success in meta) + * @param metadata Optional approval-related metadata + * @throws Error if required parameters are missing + */ + async addToolResult( + toolCallId: string, + name: string, + sanitizedResult: SanitizedToolResult, + metadata?: { + requireApproval?: boolean; + approvalStatus?: 'approved' | 'rejected'; + } + ): Promise { + if (!toolCallId || !name) { + throw ContextError.toolCallIdNameRequired(); + } + + const summary = sanitizedResult.content + .map((p) => + p.type === 'text' + ? `text(${p.text.length})` + : p.type === 'image' + ? `image(${p.mimeType || 'image'})` + : p.type === 'ui-resource' + ? `ui-resource(${p.uri})` + : `file(${p.mimeType || 'file'})` + ) + .join(', '); + this.logger.debug(`ContextManager: Storing tool result (parts) for ${name}: [${summary}]`); + + await this.addMessage({ + role: 'tool', + content: sanitizedResult.content, + toolCallId, + name, + // Success status comes from sanitizedResult.meta (single source of truth) + success: sanitizedResult.meta.success, + // Persist display data for rich rendering on session resume + ...(sanitizedResult.meta.display !== undefined && { + displayData: sanitizedResult.meta.display, + }), + // Persist approval metadata for frontend display after reload + ...(metadata?.requireApproval !== undefined && { + requireApproval: metadata.requireApproval, + }), + ...(metadata?.approvalStatus !== undefined && { + approvalStatus: metadata.approvalStatus, + }), + }); + } + + /** + * Gets the conversation history formatted for the target LLM provider. + * Applies compression strategies sequentially if the manager is configured with a `maxInputTokens` limit + * and a `tokenizer`, and the current token count exceeds the limit. Compression happens *before* formatting. + * Uses the injected formatter to convert internal messages (potentially compressed) to the provider's format. + * + * @param contributorContext The DynamicContributorContext for system prompt contributors and formatting + * @param llmContext The llmContext for the formatter to decide which messages to include based on the model's capabilities + * @param systemPrompt (Optional) Precomputed system prompt string. If provided, it will be used instead of recomputing the system prompt. Useful for avoiding duplicate computation when both the formatted messages and the raw system prompt are needed in the same request. + * @param history (Optional) Pre-fetched and potentially compressed history. If not provided, will fetch from history provider. + * @returns Formatted messages ready to send to the LLM provider API + * @throws Error if formatting or compression fails critically + */ + async getFormattedMessages( + contributorContext: DynamicContributorContext, + llmContext: LLMContext, + systemPrompt?: string | undefined, + history?: InternalMessage[] + ): Promise { + // TMessage type is provided by the service that instantiates ContextManager + // Use provided history or fetch from provider + let messageHistory: InternalMessage[] = + history ?? (await this.historyProvider.getHistory()); + + // Determine allowed media types for expansion + // Priority: User-specified config > Model capabilities from registry + let allowedMediaTypes: string[] | undefined = this.llmConfig.allowedMediaTypes; + if (!allowedMediaTypes) { + // Fall back to model capabilities from registry + try { + const { getSupportedFileTypesForModel } = await import('../llm/registry.js'); + const { fileTypesToMimePatterns } = await import('./utils.js'); + const supportedFileTypes = getSupportedFileTypesForModel( + llmContext.provider, + llmContext.model + ); + allowedMediaTypes = fileTypesToMimePatterns(supportedFileTypes, this.logger); + this.logger.debug( + `Using model capabilities for media filtering: ${allowedMediaTypes.join(', ')}` + ); + } catch (error) { + this.logger.warn( + `Could not determine model capabilities, allowing all media types: ${String(error)}` + ); + // If we can't determine capabilities, allow everything + allowedMediaTypes = undefined; + } + } else { + this.logger.debug( + `Using user-configured allowedMediaTypes: ${allowedMediaTypes.join(', ')}` + ); + } + + // Resolve blob references using resource manager with filtering + // Only user and tool messages can contain blob references (images, files) + // System and assistant messages have string-only content - no blob expansion needed + this.logger.debug('Resolving blob references in message history before formatting'); + messageHistory = await Promise.all( + messageHistory.map(async (message): Promise => { + if (isSystemMessage(message) || isAssistantMessage(message)) { + // System/assistant messages have string content, no blob refs + return message; + } + if (isUserMessage(message)) { + const expandedContent = await expandBlobReferences( + message.content, + this.resourceManager, + this.logger, + allowedMediaTypes + ); + return { ...message, content: expandedContent }; + } + if (isToolMessage(message)) { + const expandedContent = await expandBlobReferences( + message.content, + this.resourceManager, + this.logger, + allowedMediaTypes + ); + return { ...message, content: expandedContent }; + } + // Should never reach here, but TypeScript needs exhaustive check + return message; + }) + ); + + // Use pre-computed system prompt if provided + const prompt = systemPrompt ?? (await this.getSystemPrompt(contributorContext)); + return this.formatter.format([...messageHistory], llmContext, prompt) as TMessage[]; + } + + /** + * Gets the conversation ready for LLM consumption with proper flow: + * 1. Get system prompt + * 2. Prepare history (filter + transform pruned messages) + * 3. Format messages for LLM API + * + * @param contributorContext The DynamicContributorContext for system prompt contributors and formatting + * @param llmContext The llmContext for the formatter to decide which messages to include based on the model's capabilities + * @returns Object containing formatted messages, system prompt, and prepared history + */ + async getFormattedMessagesForLLM( + contributorContext: DynamicContributorContext, + llmContext: LLMContext + ): Promise<{ + formattedMessages: TMessage[]; + systemPrompt: string; + preparedHistory: InternalMessage[]; + }> { + // Step 1: Get system prompt + const systemPrompt = await this.getSystemPrompt(contributorContext); + + // Step 2: Prepare history (single source of truth for transformations) + const { preparedHistory } = await this.prepareHistory(); + + // Step 3: Format messages with prepared history + const formattedMessages = await this.getFormattedMessages( + contributorContext, + llmContext, + systemPrompt, + preparedHistory + ); + + return { + formattedMessages, + systemPrompt, + preparedHistory, + }; + } + + /** + * Estimates context token usage for the /context command and compaction decisions. + * Uses the same prepareHistory() logic as getFormattedMessagesForLLM() to ensure consistency. + * + * When actuals are available from previous LLM calls: + * estimatedNextInput = lastInputTokens + lastOutputTokens + newMessagesEstimate + * + * This formula is more accurate because: + * - lastInputTokens: exactly what the API processed (ground truth) + * - lastOutputTokens: exactly what the LLM returned (ground truth) + * - newMessagesEstimate: only estimate the delta (tool results, new user messages) + * + * When no LLM call has been made yet (or after compaction), falls back to pure estimation. + * + * @param contributorContext Context for building the system prompt + * @param tools Tool definitions to include in the estimate + * @returns Token estimates with breakdown and comparison to actual (if available) + */ + async getContextTokenEstimate( + contributorContext: DynamicContributorContext, + tools: Record + ): Promise<{ + /** Total estimated tokens */ + estimated: number; + /** Last actual token count from LLM API (null if no calls made yet) */ + actual: number | null; + /** Breakdown by category */ + breakdown: { + systemPrompt: number; + tools: { + total: number; + perTool: Array<{ name: string; tokens: number }>; + }; + messages: number; + }; + /** Preparation stats */ + stats: { + originalMessageCount: number; + filteredMessageCount: number; + prunedToolCount: number; + }; + /** Calculation basis for debugging/display */ + calculationBasis?: { + /** Whether we used the actual-based formula or pure estimation */ + method: 'actuals' | 'estimate'; + /** Last actual input tokens from API (if method is 'actuals') */ + lastInputTokens?: number; + /** Last actual output tokens from API (if method is 'actuals') */ + lastOutputTokens?: number; + /** Estimated tokens for new messages since last call (if method is 'actuals') */ + newMessagesEstimate?: number; + }; + }> { + // Step 1: Get system prompt (same as LLM preparation) + const systemPrompt = await this.getSystemPrompt(contributorContext); + + // Step 2: Prepare history (same as LLM preparation - single source of truth) + const { preparedHistory, stats } = await this.prepareHistory(); + + // Step 3: Calculate tokens using Phase 4 formula when actuals are available + // Formula: estimatedNextInput = lastInputTokens + lastOutputTokens + newMessagesEstimate + const lastInput = this.lastActualInputTokens; + const lastOutput = this.lastActualOutputTokens; + const lastMsgCount = this.lastCallMessageCount; + const currentHistory = await this.historyProvider.getHistory(); + + // Get pure estimate as fallback and for breakdown calculation + const pureEstimate = estimateContextTokens(systemPrompt, preparedHistory, tools); + + let total: number; + let calculationBasis: { + method: 'actuals' | 'estimate'; + lastInputTokens?: number; + lastOutputTokens?: number; + newMessagesEstimate?: number; + }; + + // Use actuals-based formula if we have all the required values + if (lastInput !== null && lastOutput !== null && lastMsgCount !== null) { + // Calculate estimate for messages added AFTER the last LLM call + // These are: tool results from the last assistant's tool calls + any new user messages + const newMessages = currentHistory.slice(lastMsgCount); + const newMessagesEstimate = estimateMessagesTokens(newMessages); + + // Apply the formula + total = lastInput + lastOutput + newMessagesEstimate; + + calculationBasis = { + method: 'actuals', + lastInputTokens: lastInput, + lastOutputTokens: lastOutput, + newMessagesEstimate, + }; + + this.logger.info( + `Context estimate (actuals-based): lastInput=${lastInput}, lastOutput=${lastOutput}, ` + + `newMsgs=${newMessagesEstimate} (${newMessages.length} messages), total=${total}` + ); + } else { + // Fallback to pure estimation when no actuals available + total = pureEstimate.total; + + calculationBasis = { + method: 'estimate', + }; + + this.logger.debug( + `Context estimate (pure estimate): total=${total} (no actuals available yet)` + ); + } + + // Step 4: Calculate breakdown for display + // System and tools are always estimated. Messages is back-calculated so numbers add up. + const systemPromptTokens = pureEstimate.breakdown.systemPrompt; + const toolsTokens = pureEstimate.breakdown.tools; + + // Back-calculate messages so: systemPrompt + tools + messages = total + const messagesDisplay = Math.max(0, total - systemPromptTokens - toolsTokens.total); + + // Log calibration info when we have actuals to compare against pure estimate + if (lastInput !== null) { + const pureTotal = pureEstimate.total; + const diff = pureTotal - lastInput; + const diffPercent = lastInput > 0 ? ((diff / lastInput) * 100).toFixed(1) : '0.0'; + this.logger.info( + `Context token calibration: pureEstimate=${pureTotal}, lastActual=${lastInput}, ` + + `diff=${diff} (${diffPercent}%)` + ); + } + + return { + estimated: total, + actual: lastInput, + breakdown: { + systemPrompt: systemPromptTokens, + tools: toolsTokens, + messages: messagesDisplay, + }, + stats: { + originalMessageCount: stats.originalCount, + filteredMessageCount: stats.filteredCount, + prunedToolCount: stats.prunedToolCount, + }, + calculationBasis, + }; + } + + /** + * Estimates the next input token count using actual token data from the previous LLM call. + * This is a lightweight version for compaction pre-checks that only returns the total. + * + * ## Formula (when actuals are available): + * estimatedNextInput = lastInputTokens + lastOutputTokens + newMessagesEstimate + * + * ## Why this formula works: + * + * Consider two consecutive LLM calls: + * + * ``` + * Call N: + * Input sent: system + tools + [user1] = lastInput tokens + * Output received: assistant response = lastOutput tokens + * + * Call N+1: + * Input will be: system + tools + [user1, assistant1, user2, ...] + * ≈ lastInput + assistant1_as_input + new_messages + * ≈ lastInput + lastOutput + newMessagesEstimate + * ``` + * + * The assistant's response (lastOutput) becomes part of the next input as conversation + * history. Text tokenizes similarly whether sent as input or received as output. + * + * ## No double-counting: + * + * The assistant message is added to history DURING streaming (before this method runs), + * and recordLastCallMessageCount() captures the count INCLUDING that message. + * Therefore, newMessages = history.slice(lastMsgCount) EXCLUDES the assistant message, + * so lastOutput and newMessages don't overlap. + * + * ## Pruning caveat: + * + * If tool output pruning occurs between calls, lastInput may be stale (higher than + * actual). This causes OVERESTIMATION, which is SAFE - we'd trigger compaction + * earlier rather than risk context overflow. + * + * @param systemPrompt The system prompt string + * @param preparedHistory Message history AFTER filterCompacted and pruning + * @param tools Tool definitions + * @returns Estimated total input tokens for the next LLM call + */ + async getEstimatedNextInputTokens( + systemPrompt: string, + preparedHistory: readonly InternalMessage[], + tools: Record + ): Promise { + const lastInput = this.lastActualInputTokens; + const lastOutput = this.lastActualOutputTokens; + const lastMsgCount = this.lastCallMessageCount; + const currentHistory = await this.historyProvider.getHistory(); + + // Use actuals-based formula if we have all the required values + if (lastInput !== null && lastOutput !== null && lastMsgCount !== null) { + const newMessages = currentHistory.slice(lastMsgCount); + const newMessagesEstimate = estimateMessagesTokens(newMessages); + const total = lastInput + lastOutput + newMessagesEstimate; + + this.logger.debug( + `Estimated next input (actuals-based): ${lastInput} + ${lastOutput} + ${newMessagesEstimate} = ${total}` + ); + return total; + } + + // Fallback to pure estimation + const pureEstimate = estimateContextTokens(systemPrompt, preparedHistory, tools); + this.logger.debug(`Estimated next input (pure estimate): ${pureEstimate.total}`); + return pureEstimate.total; + } + + /** + * Gets the system prompt formatted for the target LLM provider + * Some providers handle system prompts differently + * + * @returns Formatted system prompt or null/undefined based on formatter implementation + * @throws Error if formatting fails + */ + async getFormattedSystemPrompt( + _context: DynamicContributorContext + ): Promise { + // Vercel formatter handles system prompts in the messages array, not separately + return this.formatter.formatSystemPrompt?.(); + } + + /** + * Resets the conversation history + * Does not reset the system prompt + */ + async resetConversation(): Promise { + // Clear persisted history + await this.historyProvider.clearHistory(); + this.resetActualTokenTracking(); + this.logger.debug( + `ContextManager: Conversation history cleared for session ${this.sessionId}` + ); + } +} diff --git a/dexto/packages/core/src/context/media-helpers.ts b/dexto/packages/core/src/context/media-helpers.ts new file mode 100644 index 00000000..100275db --- /dev/null +++ b/dexto/packages/core/src/context/media-helpers.ts @@ -0,0 +1,27 @@ +/** + * Browser-safe media kind helpers. + * These functions have no dependencies and can be safely imported in browser environments. + */ + +/** + * Derive file media kind from MIME type. + * This is the canonical way to determine media kind - use this instead of storing redundant fields. + */ +export function getFileMediaKind(mimeType: string | undefined): 'audio' | 'video' | 'binary' { + if (mimeType?.startsWith('audio/')) return 'audio'; + if (mimeType?.startsWith('video/')) return 'video'; + return 'binary'; +} + +/** + * Derive resource kind from MIME type (includes images). + * Use this to determine the kind of resource for display/rendering purposes. + */ +export function getResourceKind( + mimeType: string | undefined +): 'image' | 'audio' | 'video' | 'binary' { + if (mimeType?.startsWith('image/')) return 'image'; + if (mimeType?.startsWith('audio/')) return 'audio'; + if (mimeType?.startsWith('video/')) return 'video'; + return 'binary'; +} diff --git a/dexto/packages/core/src/context/types.ts b/dexto/packages/core/src/context/types.ts new file mode 100644 index 00000000..fd3e486a --- /dev/null +++ b/dexto/packages/core/src/context/types.ts @@ -0,0 +1,336 @@ +import type { LLMProvider, TokenUsage } from '../llm/types.js'; +import type { ToolDisplayData } from '../tools/display-types.js'; + +// ============================================================================= +// Content Part Types +// ============================================================================= + +/** + * Base interface for image data. + * Supports multiple formats for flexibility across different use cases. + */ +export interface ImageData { + image: string | Uint8Array | Buffer | ArrayBuffer | URL; + mimeType?: string; +} + +/** + * Base interface for file data. + * Supports multiple formats for flexibility across different use cases. + */ +export interface FileData { + data: string | Uint8Array | Buffer | ArrayBuffer | URL; + mimeType: string; + filename?: string; +} + +/** + * Text content part. + */ +export interface TextPart { + type: 'text'; + text: string; +} + +/** + * Image content part. + */ +export interface ImagePart extends ImageData { + type: 'image'; +} + +/** + * File content part. + */ +export interface FilePart extends FileData { + type: 'file'; +} + +/** + * UI Resource content part for MCP-UI interactive components. + * Enables MCP servers to return rich, interactive UI (live streams, dashboards, forms). + * @see https://mcpui.dev/ for MCP-UI specification + */ +export interface UIResourcePart { + type: 'ui-resource'; + /** URI identifying the UI resource, must start with ui:// */ + uri: string; + /** MIME type: text/html, text/uri-list, or application/vnd.mcp-ui.remote-dom */ + mimeType: string; + /** Inline HTML content or URL (for text/html and text/uri-list) */ + content?: string; + /** Base64-encoded content (alternative to content field) */ + blob?: string; + /** Optional metadata for the UI resource */ + metadata?: { + /** Display title for the UI resource */ + title?: string; + /** Preferred rendering size in pixels */ + preferredSize?: { width: number; height: number }; + }; +} + +/** + * Union of all content part types. + * Discriminated by the `type` field. + */ +export type ContentPart = TextPart | ImagePart | FilePart | UIResourcePart; + +// ============================================================================= +// Content Part Type Guards +// ============================================================================= + +/** + * Type guard for TextPart. + */ +export function isTextPart(part: ContentPart): part is TextPart { + return part.type === 'text'; +} + +/** + * Type guard for ImagePart. + */ +export function isImagePart(part: ContentPart): part is ImagePart { + return part.type === 'image'; +} + +/** + * Type guard for FilePart. + */ +export function isFilePart(part: ContentPart): part is FilePart { + return part.type === 'file'; +} + +/** + * Type guard for UIResourcePart. + */ +export function isUIResourcePart(part: ContentPart): part is UIResourcePart { + return part.type === 'ui-resource'; +} + +// ============================================================================= +// Tool Result Types +// ============================================================================= + +/** + * Sanitized tool execution result with content parts and resource references. + */ +export interface SanitizedToolResult { + /** Ordered content parts ready for rendering or provider formatting */ + content: ContentPart[]; + /** + * Resource references created during sanitization (e.g. blob store URIs). + * Consumers can dereference these via ResourceManager APIs. + */ + resources?: Array<{ + uri: string; + kind: 'image' | 'audio' | 'video' | 'binary'; + mimeType: string; + filename?: string; + }>; + meta: { + toolName: string; + toolCallId: string; + /** Whether the tool execution succeeded. Always set by sanitizeToolResult(). */ + success: boolean; + /** Structured display data for tool-specific rendering (diffs, shell output, etc.) */ + display?: ToolDisplayData; + }; +} + +// ============================================================================= +// Shared Message Types +// ============================================================================= + +// TokenUsage imported from llm/types.ts (used by AssistantMessage) + +/** + * Tool call request from an assistant message. + */ +export interface ToolCall { + /** Unique identifier for this tool call */ + id: string; + /** The type of tool call (currently only 'function' is supported) */ + type: 'function'; + /** Function call details */ + function: { + /** Name of the function to call */ + name: string; + /** Arguments for the function in JSON string format */ + arguments: string; + }; + /** + * Provider-specific options (e.g., thought signatures for Gemini 3). + * These are opaque tokens passed through to maintain model state across tool calls. + * Not intended for display - purely for API round-tripping. + */ + providerOptions?: Record; +} + +/** + * Approval status for tool message executions. + * (Not to be confused with ApprovalStatus enum from approval module) + */ +export type ToolApprovalStatus = 'pending' | 'approved' | 'rejected'; + +// ============================================================================= +// Message Types (Discriminated Union by 'role') +// ============================================================================= + +/** + * Base interface for all message types. + * Contains fields common to all messages. + */ +interface MessageBase { + /** + * Unique message identifier (UUID). + * Auto-generated by ContextManager.addMessage() if not provided. + */ + id?: string; + + /** + * Timestamp when the message was created (Unix timestamp in milliseconds). + * Auto-generated by ContextManager.addMessage() if not provided. + */ + timestamp?: number; + + /** + * Optional metadata for the message. + * Used for tracking summary status, original message IDs, etc. + */ + metadata?: Record; +} + +/** + * System message containing instructions or context for the LLM. + */ +export interface SystemMessage extends MessageBase { + role: 'system'; + /** System prompt content as array of content parts */ + content: ContentPart[]; +} + +/** + * User message containing end-user input. + * Content can be text, images, files, or UI resources. + */ +export interface UserMessage extends MessageBase { + role: 'user'; + /** User input content as array of content parts */ + content: ContentPart[]; +} + +/** + * Assistant message containing LLM response. + * May include text content, reasoning, and/or tool calls. + */ +export interface AssistantMessage extends MessageBase { + role: 'assistant'; + /** Response content - null if message only contains tool calls */ + content: ContentPart[] | null; + + /** + * Model reasoning text associated with this response. + * Present when the provider supports reasoning and returns a final reasoning trace. + */ + reasoning?: string; + + /** + * Provider-specific metadata for reasoning, used for round-tripping. + * Contains opaque tokens (e.g., OpenAI itemId, Gemini thought signatures) + * that must be passed back to the provider on subsequent requests. + */ + reasoningMetadata?: Record; + + /** Token usage accounting for this response */ + tokenUsage?: TokenUsage; + + /** Model identifier that generated this response */ + model?: string; + + /** Provider identifier for this response */ + provider?: LLMProvider; + + /** + * Tool calls requested by the assistant. + * Present when the LLM requests tool execution. + */ + toolCalls?: ToolCall[]; +} + +/** + * Tool message containing the result of a tool execution. + * Links back to the original tool call via toolCallId. + */ +export interface ToolMessage extends MessageBase { + role: 'tool'; + /** Tool execution result as array of content parts */ + content: ContentPart[]; + + /** ID of the tool call this message is responding to (REQUIRED) */ + toolCallId: string; + + /** Name of the tool that produced this result (REQUIRED) */ + name: string; + + /** Whether the tool execution was successful */ + success?: boolean; + + /** Whether this tool call required user approval before execution */ + requireApproval?: boolean; + + /** The approval status for this tool call */ + approvalStatus?: ToolApprovalStatus; + + /** + * Timestamp when the tool output was compacted/pruned. + * Present when the tool result has been summarized to save context space. + */ + compactedAt?: number; + + /** + * Structured display data for tool-specific rendering (diffs, shell output, etc.) + * Persisted from SanitizedToolResult.meta.display for proper rendering on session resume. + */ + displayData?: ToolDisplayData; +} + +/** + * Union of all message types. + * Discriminated by the `role` field. + * + * Use type guards (isSystemMessage, isUserMessage, etc.) for type narrowing. + */ +export type InternalMessage = SystemMessage | UserMessage | AssistantMessage | ToolMessage; + +// ============================================================================= +// Message Type Guards +// ============================================================================= + +/** + * Type guard for SystemMessage. + */ +export function isSystemMessage(msg: InternalMessage): msg is SystemMessage { + return msg.role === 'system'; +} + +/** + * Type guard for UserMessage. + */ +export function isUserMessage(msg: InternalMessage): msg is UserMessage { + return msg.role === 'user'; +} + +/** + * Type guard for AssistantMessage. + */ +export function isAssistantMessage(msg: InternalMessage): msg is AssistantMessage { + return msg.role === 'assistant'; +} + +/** + * Type guard for ToolMessage. + */ +export function isToolMessage(msg: InternalMessage): msg is ToolMessage { + return msg.role === 'tool'; +} diff --git a/dexto/packages/core/src/context/utils.test.ts b/dexto/packages/core/src/context/utils.test.ts new file mode 100644 index 00000000..2133fdfd --- /dev/null +++ b/dexto/packages/core/src/context/utils.test.ts @@ -0,0 +1,1809 @@ +import { describe, expect, it, test, beforeEach, vi } from 'vitest'; +import type { + BlobStore, + BlobInput, + BlobMetadata, + BlobReference, + BlobData, + BlobStats, + StoredBlobMetadata, +} from '../storage/blob/types.js'; +import { + normalizeToolResult, + persistToolMedia, + filterMessagesByLLMCapabilities, + parseDataUri, + isLikelyBase64String, + getFileMediaKind, + getResourceKind, + matchesMimePattern, + matchesAnyMimePattern, + fileTypesToMimePatterns, + filterCompacted, + sanitizeToolResultToContentWithBlobs, + estimateStringTokens, + estimateImageTokens, + estimateFileTokens, + estimateContentPartTokens, + estimateMessagesTokens, + estimateToolsTokens, + estimateContextTokens, +} from './utils.js'; +import { InternalMessage, ContentPart, FilePart } from './types.js'; +import { LLMContext } from '../llm/types.js'; +import * as registry from '../llm/registry.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; + +// Mock the registry module +vi.mock('../llm/registry.js'); +const mockValidateModelFileSupport = vi.mocked(registry.validateModelFileSupport); + +// Create a mock logger for tests +const mockLogger = createMockLogger(); + +class FakeBlobStore implements BlobStore { + private counter = 0; + private connected = true; + private readonly storage = new Map(); + private readonly metadata = new Map(); + + async store(input: BlobInput, metadata: BlobMetadata = {}): Promise { + const buffer = this.toBuffer(input); + const id = `fake-${this.counter++}`; + const storedMetadata: StoredBlobMetadata = { + id, + mimeType: metadata.mimeType ?? 'application/octet-stream', + originalName: metadata.originalName, + createdAt: metadata.createdAt ?? new Date(), + size: buffer.length, + hash: id, + source: metadata.source, + }; + + this.storage.set(id, buffer); + this.metadata.set(id, storedMetadata); + + return { + id, + uri: `blob:${id}`, + metadata: storedMetadata, + }; + } + + async retrieve( + _reference: string, + _format?: 'base64' | 'buffer' | 'path' | 'stream' | 'url' + ): Promise { + throw new Error('Not implemented in FakeBlobStore'); + } + + async exists(reference: string): Promise { + return this.storage.has(this.parse(reference)); + } + + async delete(reference: string): Promise { + const id = this.parse(reference); + this.storage.delete(id); + this.metadata.delete(id); + } + + async cleanup(_olderThan?: Date | undefined): Promise { + const count = this.storage.size; + this.storage.clear(); + this.metadata.clear(); + return count; + } + + async getStats(): Promise { + let totalSize = 0; + for (const buffer of this.storage.values()) { + totalSize += buffer.length; + } + return { + count: this.storage.size, + totalSize, + backendType: 'fake', + storePath: 'memory://fake', + }; + } + + async listBlobs(): Promise { + return Array.from(this.metadata.values()).map((meta) => ({ + id: meta.id, + uri: `blob:${meta.id}`, + metadata: meta, + })); + } + + getStoragePath(): string | undefined { + return undefined; + } + + async connect(): Promise { + this.connected = true; + } + + async disconnect(): Promise { + this.connected = false; + } + + isConnected(): boolean { + return this.connected; + } + + getStoreType(): string { + return 'fake'; + } + + private toBuffer(input: BlobInput): Buffer { + if (Buffer.isBuffer(input)) { + return input; + } + if (input instanceof Uint8Array) { + return Buffer.from(input); + } + if (input instanceof ArrayBuffer) { + return Buffer.from(new Uint8Array(input)); + } + if (typeof input === 'string') { + try { + return Buffer.from(input, 'base64'); + } catch { + return Buffer.from(input, 'utf-8'); + } + } + throw new Error('Unsupported blob input'); + } + + private parse(reference: string): string { + return reference.startsWith('blob:') ? reference.slice(5) : reference; + } +} + +import { sanitizeToolResult } from './utils.js'; + +describe('sanitizeToolResult success tracking', () => { + it('should include success=true in meta when tool succeeds', async () => { + const result = await sanitizeToolResult( + { data: 'test result' }, + { + toolName: 'test_tool', + toolCallId: 'call-123', + success: true, + }, + mockLogger + ); + + expect(result.meta.success).toBe(true); + expect(result.meta.toolName).toBe('test_tool'); + expect(result.meta.toolCallId).toBe('call-123'); + }); + + it('should include success=false in meta when tool fails', async () => { + const result = await sanitizeToolResult( + { error: 'Something went wrong' }, + { + toolName: 'failing_tool', + toolCallId: 'call-456', + success: false, + }, + mockLogger + ); + + expect(result.meta.success).toBe(false); + expect(result.meta.toolName).toBe('failing_tool'); + expect(result.meta.toolCallId).toBe('call-456'); + }); + + it('should preserve success status through blob storage', async () => { + const store = new FakeBlobStore(); + const payload = Buffer.alloc(4096, 7); + const dataUri = `data:image/jpeg;base64,${payload.toString('base64')}`; + + const result = await sanitizeToolResult( + dataUri, + { + blobStore: store, + toolName: 'image_tool', + toolCallId: 'call-789', + success: true, + }, + mockLogger + ); + + expect(result.meta.success).toBe(true); + // Expect 2 parts: image + blob reference annotation text + expect(result.content).toHaveLength(2); + expect(result.content[0]?.type).toBe('image'); + expect(result.content[1]?.type).toBe('text'); + }); + + it('should track failed tool results with complex output', async () => { + const errorOutput = { + error: 'Tool execution failed', + details: { code: 'TIMEOUT', message: 'Request timed out after 30s' }, + }; + + const result = await sanitizeToolResult( + errorOutput, + { + toolName: 'api_call', + toolCallId: 'call-error', + success: false, + }, + mockLogger + ); + + expect(result.meta.success).toBe(false); + // Content should still be present even for failures + expect(result.content).toBeDefined(); + expect(result.content.length).toBeGreaterThan(0); + }); + + it('should handle empty result with success status', async () => { + const result = await sanitizeToolResult( + '', + { + toolName: 'void_tool', + toolCallId: 'call-empty', + success: true, + }, + mockLogger + ); + + expect(result.meta.success).toBe(true); + // Should have fallback content + expect(result.content).toBeDefined(); + }); +}); + +describe('tool result normalization pipeline', () => { + it('normalizes data URI media into typed parts with inline media hints', async () => { + const payload = Buffer.alloc(2048, 1); + const dataUri = `data:image/png;base64,${payload.toString('base64')}`; + + const normalized = await normalizeToolResult(dataUri, mockLogger); + + expect(normalized.parts).toHaveLength(1); + const part = normalized.parts[0]; + if (!part) { + throw new Error('expected normalized image part'); + } + expect(part.type).toBe('image'); + expect(normalized.inlineMedia).toHaveLength(1); + const hint = normalized.inlineMedia[0]; + if (!hint) { + throw new Error('expected inline media hint for data URI'); + } + expect(hint.mimeType).toBe('image/png'); + }); + + it('persists large inline media to the blob store and produces resource descriptors', async () => { + const payload = Buffer.alloc(4096, 7); + const dataUri = `data:image/jpeg;base64,${payload.toString('base64')}`; + + const normalized = await normalizeToolResult(dataUri, mockLogger); + const store = new FakeBlobStore(); + + const persisted = await persistToolMedia( + normalized, + { + blobStore: store, + toolName: 'image_tool', + toolCallId: 'call-123', + }, + mockLogger + ); + + // Expect 2 parts: image + blob reference annotation text + expect(persisted.parts).toHaveLength(2); + const part = persisted.parts[0]; + if (!part || part.type !== 'image') { + throw new Error('expected image part after persistence'); + } + expect(typeof part.image).toBe('string'); + expect(part.image).toMatch(/^@blob:/); + + // Verify annotation text part + // Uses "resource_ref:" prefix to avoid expansion by expandBlobsInText() + const annotationPart = persisted.parts[1]; + expect(annotationPart?.type).toBe('text'); + if (annotationPart?.type === 'text') { + expect(annotationPart.text).toContain('resource_ref:blob:'); + expect(annotationPart.text).toContain('image/jpeg'); + // Should NOT contain @blob: which would trigger expansion + expect(annotationPart.text).not.toContain('@blob:'); + } + + expect(persisted.resources).toBeDefined(); + expect(persisted.resources?.[0]?.kind).toBe('image'); + }); + + it('always persists video media regardless of payload size', async () => { + const payload = Buffer.alloc(256, 3); + const raw = [ + { + type: 'file', + data: payload.toString('base64'), + mimeType: 'video/mp4', + }, + ]; + + const normalized = await normalizeToolResult(raw, mockLogger); + const hint = normalized.inlineMedia[0]; + if (!hint) { + throw new Error('expected inline media hint for video payload'); + } + // Video should be persisted regardless of size + expect(hint.mimeType).toBe('video/mp4'); + + const store = new FakeBlobStore(); + const persisted = await persistToolMedia( + normalized, + { + blobStore: store, + toolName: 'video_tool', + toolCallId: 'call-456', + }, + mockLogger + ); + + const filePart = persisted.parts[0]; + if (!filePart || filePart.type !== 'file') { + throw new Error('expected file part after persistence'); + } + expect(typeof filePart.data).toBe('string'); + expect(filePart.data).toMatch(/^@blob:/); + expect(persisted.resources?.[0]?.kind).toBe('video'); + }); +}); + +describe('filterMessagesByLLMCapabilities', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test('should keep text and image parts unchanged', () => { + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Hello' }, + { type: 'image', image: 'base64data', mimeType: 'image/png' }, + ], + }, + ]; + + const config: LLMContext = { provider: 'openai', model: 'gpt-5' }; + + const result = filterMessagesByLLMCapabilities(messages, config, mockLogger); + + expect(result).toEqual(messages); + }); + + test('should filter out unsupported file attachments', () => { + // Mock validation to reject PDF files for gpt-3.5-turbo + mockValidateModelFileSupport.mockReturnValue({ + isSupported: false, + error: 'Model gpt-3.5-turbo does not support PDF files', + }); + + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Analyze this document' }, + { + type: 'file', + data: 'pdfdata', + mimeType: 'application/pdf', + filename: 'doc.pdf', + }, + ], + }, + ]; + + const config: LLMContext = { provider: 'openai', model: 'gpt-3.5-turbo' }; + + const result = filterMessagesByLLMCapabilities(messages, config, mockLogger); + + expect(result[0]!.content).toEqual([{ type: 'text', text: 'Analyze this document' }]); + expect(mockValidateModelFileSupport).toHaveBeenCalledWith( + config.provider, + config.model, + 'application/pdf' + ); + }); + + test('should keep supported file attachments for models that support them', () => { + // Mock validation to accept PDF files for gpt-5 + mockValidateModelFileSupport.mockReturnValue({ + isSupported: true, + fileType: 'pdf', + }); + + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Analyze this document' }, + { + type: 'file', + data: 'pdfdata', + mimeType: 'application/pdf', + filename: 'doc.pdf', + }, + ], + }, + ]; + + const config: LLMContext = { provider: 'openai', model: 'gpt-5' }; + + const result = filterMessagesByLLMCapabilities(messages, config, mockLogger); + + expect(result[0]!.content).toEqual([ + { type: 'text', text: 'Analyze this document' }, + { type: 'file', data: 'pdfdata', mimeType: 'application/pdf', filename: 'doc.pdf' }, + ]); + }); + + test('should handle audio file filtering for different models', () => { + // Mock validation to reject audio for regular models but accept for audio-preview models + mockValidateModelFileSupport + .mockReturnValueOnce({ + isSupported: false, + error: 'Model gpt-5 does not support audio files', + }) + .mockReturnValueOnce({ + isSupported: true, + fileType: 'audio', + }); + + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Transcribe this audio' }, + { + type: 'file', + data: 'audiodata', + mimeType: 'audio/mp3', + filename: 'recording.mp3', + }, + ], + }, + ]; + + // Test with regular gpt-5 (should filter out audio) + const config1: LLMContext = { provider: 'openai', model: 'gpt-5' }; + const result1 = filterMessagesByLLMCapabilities(messages, config1, mockLogger); + + expect(result1[0]!.content).toEqual([{ type: 'text', text: 'Transcribe this audio' }]); + + // Test with gpt-4o-audio-preview (should keep audio) + const config2: LLMContext = { provider: 'openai', model: 'gpt-4o-audio-preview' }; + const result2 = filterMessagesByLLMCapabilities(messages, config2, mockLogger); + + expect(result2[0]!.content).toEqual([ + { type: 'text', text: 'Transcribe this audio' }, + { type: 'file', data: 'audiodata', mimeType: 'audio/mp3', filename: 'recording.mp3' }, + ]); + }); + + test('should add placeholder text when all content is filtered out', () => { + // Mock validation to reject all files + mockValidateModelFileSupport.mockReturnValue({ + isSupported: false, + error: 'File type not supported by current LLM', + }); + + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { + type: 'file', + data: 'pdfdata', + mimeType: 'application/pdf', + filename: 'doc.pdf', + }, + ], + }, + ]; + + const config: LLMContext = { provider: 'openai', model: 'gpt-3.5-turbo' }; + + const result = filterMessagesByLLMCapabilities(messages, config, mockLogger); + + expect(result[0]!.content).toEqual([ + { type: 'text', text: '[File attachment removed - not supported by gpt-3.5-turbo]' }, + ]); + }); + + test('should only filter user messages with array content', () => { + // Note: system, assistant, and tool messages use string content for simplicity in tests. + // The function only processes user messages with array content. + const messages = [ + { + role: 'system', + content: 'You are a helpful assistant', + }, + { + role: 'assistant', + content: 'Hello! How can I help you?', + }, + { + role: 'tool', + content: 'Tool result', + name: 'search', + toolCallId: 'call_123', + }, + { + role: 'user', + content: [ + { type: 'text', text: 'Analyze this' }, + { type: 'file', data: 'data', mimeType: 'application/pdf' }, + ], + }, + ] as unknown as InternalMessage[]; + + // Mock validation to reject the file + mockValidateModelFileSupport.mockReturnValue({ + isSupported: false, + error: 'PDF not supported', + }); + + const config: LLMContext = { provider: 'openai', model: 'gpt-3.5-turbo' }; + + const result = filterMessagesByLLMCapabilities(messages, config, mockLogger); + + // Only the user message with array content should be modified + expect(result[0]).toEqual(messages[0]); // system unchanged + expect(result[1]).toEqual(messages[1]); // assistant unchanged + expect(result[2]).toEqual(messages[2]); // tool unchanged + expect(result[3]!.content).toEqual([{ type: 'text', text: 'Analyze this' }]); // user message filtered + }); + + test('should keep unknown part types unchanged', () => { + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Hello' }, + { type: 'custom', data: 'some data' } as any, // Test case: unknown message part type + ], + }, + ]; + + const config: LLMContext = { provider: 'openai', model: 'gpt-5' }; + + const result = filterMessagesByLLMCapabilities(messages, config, mockLogger); + + expect(result).toEqual(messages); + }); + + test('should handle files without mimeType gracefully', () => { + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Hello' }, + { type: 'file', data: 'data' } as any, // Test case: file without mimeType property + ], + }, + ]; + + const config: LLMContext = { provider: 'openai', model: 'gpt-5' }; + + const result = filterMessagesByLLMCapabilities(messages, config, mockLogger); + + // Should keep the malformed file part since it doesn't have mimeType to validate + expect(result).toEqual(messages); + }); + + test('should handle empty message content array', () => { + const messages: InternalMessage[] = [ + { + role: 'user', + content: [], + }, + ]; + + const config: LLMContext = { provider: 'openai', model: 'gpt-5' }; + + const result = filterMessagesByLLMCapabilities(messages, config, mockLogger); + + // Should add placeholder text for empty content + expect(result[0]!.content).toEqual([ + { type: 'text', text: '[File attachment removed - not supported by gpt-5]' }, + ]); + }); +}); + +describe('parseDataUri', () => { + test('should parse valid data URI with image/png', () => { + const dataUri = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=='; + const result = parseDataUri(dataUri); + + expect(result).toEqual({ + mediaType: 'image/png', + base64: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==', + }); + }); + + test('should parse valid data URI with application/pdf', () => { + const dataUri = + 'data:application/pdf;base64,JVBERi0xLjQKJdPr6eEKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwo+PgplbmRvYmoKdHJhaWxlcgo8PAovU2l6ZSAxCi9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo5CiUlRU9G'; + const result = parseDataUri(dataUri); + + expect(result).toEqual({ + mediaType: 'application/pdf', + base64: 'JVBERi0xLjQKJdPr6eEKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwo+PgplbmRvYmoKdHJhaWxlcgo8PAovU2l6ZSAxCi9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo5CiUlRU9G', + }); + }); + + test('should default to application/octet-stream when no mediaType specified', () => { + const dataUri = 'data:;base64,SGVsbG9Xb3JsZA=='; + const result = parseDataUri(dataUri); + + expect(result).toEqual({ + mediaType: 'application/octet-stream', + base64: 'SGVsbG9Xb3JsZA==', + }); + }); + + test('should return null for non-data URI strings', () => { + expect(parseDataUri('https://example.com/image.png')).toBeNull(); + expect(parseDataUri('SGVsbG9Xb3JsZA==')).toBeNull(); + expect(parseDataUri('plain text')).toBeNull(); + }); + + test('should return null for malformed data URIs without comma', () => { + expect(parseDataUri('data:image/png;base64')).toBeNull(); + expect(parseDataUri('data:image/png')).toBeNull(); + }); + + test('should return null for data URIs without base64 encoding', () => { + expect(parseDataUri('data:text/plain,Hello World')).toBeNull(); + expect(parseDataUri('data:image/png;charset=utf8,test')).toBeNull(); + }); + + test('should handle case insensitive base64 suffix', () => { + const dataUri = + 'data:image/png;BASE64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=='; + const result = parseDataUri(dataUri); + + expect(result).toEqual({ + mediaType: 'image/png', + base64: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==', + }); + }); +}); + +describe('isLikelyBase64String', () => { + test('should identify valid base64 strings above minimum length', () => { + // Valid base64 string longer than default minimum (512 chars) + const longBase64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=='.repeat( + 10 + ); + expect(isLikelyBase64String(longBase64)).toBe(true); + }); + + test('should reject regular text even if long', () => { + const longText = 'This is a regular sentence with normal words and punctuation. '.repeat( + 10 + ); + expect(isLikelyBase64String(longText)).toBe(false); + }); + + test('should reject short strings regardless of content', () => { + expect(isLikelyBase64String('SGVsbG8=')).toBe(false); // Short but valid base64 + expect(isLikelyBase64String('abc123')).toBe(false); + }); + + test('should accept custom minimum length', () => { + const shortBase64 = 'SGVsbG9Xb3JsZA=='; // "HelloWorld" in base64 + expect(isLikelyBase64String(shortBase64, 10)).toBe(true); + expect(isLikelyBase64String(shortBase64, 20)).toBe(false); + }); + + test('should handle null and undefined gracefully', () => { + expect(isLikelyBase64String('')).toBe(false); + expect(isLikelyBase64String(null as any)).toBe(false); + expect(isLikelyBase64String(undefined as any)).toBe(false); + }); + + test('should identify data URIs as base64-like content', () => { + const dataUri = + 'data:image/png;base64,' + + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=='.repeat( + 10 + ); + expect(isLikelyBase64String(dataUri)).toBe(true); + }); + + test('should use heuristic to distinguish base64 from natural text', () => { + // Base64 has high ratio of alphanumeric chars and specific padding + const base64Like = + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=='.repeat(10); + expect(isLikelyBase64String(base64Like)).toBe(true); + + // Natural text has more variety and word boundaries + const naturalText = + 'The quick brown fox jumps over the lazy dog multiple times in this long sentence that repeats itself.'.repeat( + 5 + ); + expect(isLikelyBase64String(naturalText)).toBe(false); + }); +}); + +describe('getFileMediaKind', () => { + test('should detect audio from MIME type', () => { + expect(getFileMediaKind('audio/mp3')).toBe('audio'); + expect(getFileMediaKind('audio/mpeg')).toBe('audio'); + expect(getFileMediaKind('audio/wav')).toBe('audio'); + expect(getFileMediaKind('audio/ogg')).toBe('audio'); + }); + + test('should detect video from MIME type', () => { + expect(getFileMediaKind('video/mp4')).toBe('video'); + expect(getFileMediaKind('video/webm')).toBe('video'); + expect(getFileMediaKind('video/quicktime')).toBe('video'); + expect(getFileMediaKind('video/x-msvideo')).toBe('video'); + }); + + test('should default to binary for other types', () => { + expect(getFileMediaKind('application/pdf')).toBe('binary'); + expect(getFileMediaKind('text/plain')).toBe('binary'); + expect(getFileMediaKind('application/json')).toBe('binary'); + expect(getFileMediaKind('image/png')).toBe('binary'); + }); + + test('should handle undefined gracefully', () => { + expect(getFileMediaKind(undefined)).toBe('binary'); + }); + + test('should handle empty string', () => { + expect(getFileMediaKind('')).toBe('binary'); + }); +}); + +describe('getResourceKind', () => { + test('should detect image from MIME type', () => { + expect(getResourceKind('image/png')).toBe('image'); + expect(getResourceKind('image/jpeg')).toBe('image'); + expect(getResourceKind('image/gif')).toBe('image'); + expect(getResourceKind('image/webp')).toBe('image'); + }); + + test('should detect audio from MIME type', () => { + expect(getResourceKind('audio/mp3')).toBe('audio'); + expect(getResourceKind('audio/mpeg')).toBe('audio'); + expect(getResourceKind('audio/wav')).toBe('audio'); + }); + + test('should detect video from MIME type', () => { + expect(getResourceKind('video/mp4')).toBe('video'); + expect(getResourceKind('video/webm')).toBe('video'); + expect(getResourceKind('video/quicktime')).toBe('video'); + }); + + test('should default to binary for other types', () => { + expect(getResourceKind('application/pdf')).toBe('binary'); + expect(getResourceKind('text/plain')).toBe('binary'); + expect(getResourceKind('application/json')).toBe('binary'); + }); + + test('should handle undefined gracefully', () => { + expect(getResourceKind(undefined)).toBe('binary'); + }); + + test('should handle empty string', () => { + expect(getResourceKind('')).toBe('binary'); + }); +}); + +describe('matchesMimePattern', () => { + test('should match exact MIME types', () => { + expect(matchesMimePattern('image/png', 'image/png')).toBe(true); + expect(matchesMimePattern('video/mp4', 'video/mp4')).toBe(true); + expect(matchesMimePattern('application/pdf', 'application/pdf')).toBe(true); + }); + + test('should match wildcard patterns', () => { + expect(matchesMimePattern('image/png', 'image/*')).toBe(true); + expect(matchesMimePattern('image/jpeg', 'image/*')).toBe(true); + expect(matchesMimePattern('video/mp4', 'video/*')).toBe(true); + expect(matchesMimePattern('audio/mpeg', 'audio/*')).toBe(true); + }); + + test('should match universal wildcard', () => { + expect(matchesMimePattern('image/png', '*')).toBe(true); + expect(matchesMimePattern('video/mp4', '*/*')).toBe(true); + expect(matchesMimePattern('application/pdf', '*')).toBe(true); + }); + + test('should not match different types', () => { + expect(matchesMimePattern('image/png', 'video/*')).toBe(false); + expect(matchesMimePattern('video/mp4', 'image/*')).toBe(false); + expect(matchesMimePattern('application/pdf', 'image/*')).toBe(false); + }); + + test('should be case insensitive', () => { + expect(matchesMimePattern('IMAGE/PNG', 'image/*')).toBe(true); + expect(matchesMimePattern('image/png', 'IMAGE/*')).toBe(true); + expect(matchesMimePattern('Video/MP4', 'video/*')).toBe(true); + }); + + test('should handle undefined MIME type', () => { + expect(matchesMimePattern(undefined, 'image/*')).toBe(false); + expect(matchesMimePattern(undefined, '*')).toBe(false); + }); + + test('should trim whitespace', () => { + expect(matchesMimePattern(' image/png ', 'image/*')).toBe(true); + expect(matchesMimePattern('image/png', ' image/* ')).toBe(true); + }); +}); + +describe('matchesAnyMimePattern', () => { + test('should match if any pattern matches', () => { + expect(matchesAnyMimePattern('image/png', ['video/*', 'image/*'])).toBe(true); + expect(matchesAnyMimePattern('video/mp4', ['image/*', 'video/*', 'audio/*'])).toBe(true); + }); + + test('should not match if no patterns match', () => { + expect(matchesAnyMimePattern('image/png', ['video/*', 'audio/*'])).toBe(false); + expect(matchesAnyMimePattern('application/pdf', ['image/*', 'video/*'])).toBe(false); + }); + + test('should handle empty pattern array', () => { + expect(matchesAnyMimePattern('image/png', [])).toBe(false); + }); + + test('should handle exact and wildcard mix', () => { + expect(matchesAnyMimePattern('image/png', ['video/mp4', 'image/*'])).toBe(true); + expect(matchesAnyMimePattern('video/mp4', ['video/mp4', 'audio/*'])).toBe(true); + }); +}); + +describe('fileTypesToMimePatterns', () => { + test('should convert image file type', () => { + expect(fileTypesToMimePatterns(['image'], mockLogger)).toEqual(['image/*']); + }); + + test('should convert pdf file type', () => { + expect(fileTypesToMimePatterns(['pdf'], mockLogger)).toEqual(['application/pdf']); + }); + + test('should convert audio file type', () => { + expect(fileTypesToMimePatterns(['audio'], mockLogger)).toEqual(['audio/*']); + }); + + test('should convert video file type', () => { + expect(fileTypesToMimePatterns(['video'], mockLogger)).toEqual(['video/*']); + }); + + test('should convert multiple file types', () => { + expect(fileTypesToMimePatterns(['image', 'pdf', 'audio'], mockLogger)).toEqual([ + 'image/*', + 'application/pdf', + 'audio/*', + ]); + }); + + test('should handle empty array', () => { + expect(fileTypesToMimePatterns([], mockLogger)).toEqual([]); + }); + + test('should skip unknown file types', () => { + // Unknown types are logged as warnings but not added to patterns + expect(fileTypesToMimePatterns(['image', 'unknown', 'pdf'], mockLogger)).toEqual([ + 'image/*', + 'application/pdf', + ]); + }); +}); + +describe('expandBlobReferences', () => { + // Import the function we're testing + let expandBlobReferences: typeof import('./utils.js').expandBlobReferences; + + beforeEach(async () => { + const utils = await import('./utils.js'); + expandBlobReferences = utils.expandBlobReferences; + }); + + // Create a mock ResourceManager + function createMockResourceManager( + blobData: Record + ) { + return { + read: vi.fn(async (uri: string) => { + const id = uri.startsWith('blob:') ? uri.slice(5) : uri; + const data = blobData[id]; + if (!data) { + throw new Error(`Blob not found: ${uri}`); + } + return { + contents: [ + { + ...(data.blob ? { blob: data.blob } : {}), + ...(data.mimeType ? { mimeType: data.mimeType } : {}), + ...(data.text ? { text: data.text } : {}), + }, + ], + _meta: {}, + }; + }), + } as unknown as import('../resources/index.js').ResourceManager; + } + + test('should return empty array for null content', async () => { + const resourceManager = createMockResourceManager({}); + const result = await expandBlobReferences(null, resourceManager, mockLogger); + expect(result).toEqual([]); + }); + + test('should return TextPart unchanged if no blob references', async () => { + const resourceManager = createMockResourceManager({}); + const result = await expandBlobReferences( + [{ type: 'text', text: 'Hello world' }], + resourceManager, + mockLogger + ); + expect(result).toEqual([{ type: 'text', text: 'Hello world' }]); + }); + + test('should expand blob reference in TextPart', async () => { + const resourceManager = createMockResourceManager({ + abc123: { blob: 'base64imagedata', mimeType: 'image/png' }, + }); + + const result = await expandBlobReferences( + [{ type: 'text', text: 'Check this image: @blob:abc123' }], + resourceManager, + mockLogger + ); + + expect(result.length).toBe(2); + expect(result[0]).toEqual({ type: 'text', text: 'Check this image: ' }); + expect(result[1]).toMatchObject({ + type: 'image', + image: 'base64imagedata', + mimeType: 'image/png', + }); + }); + + test('should expand multiple blob references in TextPart', async () => { + const resourceManager = createMockResourceManager({ + aaa111: { blob: 'imagedata1', mimeType: 'image/png' }, + bbb222: { blob: 'imagedata2', mimeType: 'image/jpeg' }, + }); + + const result = await expandBlobReferences( + [{ type: 'text', text: 'Image 1: @blob:aaa111 and Image 2: @blob:bbb222' }], + resourceManager, + mockLogger + ); + + expect(result.length).toBe(4); + expect(result[0]).toEqual({ type: 'text', text: 'Image 1: ' }); + expect(result[1]).toMatchObject({ type: 'image', image: 'imagedata1' }); + expect(result[2]).toEqual({ type: 'text', text: ' and Image 2: ' }); + expect(result[3]).toMatchObject({ type: 'image', image: 'imagedata2' }); + }); + + test('should pass through array content without blob references', async () => { + const resourceManager = createMockResourceManager({}); + const content = [ + { type: 'text' as const, text: 'Hello' }, + { type: 'image' as const, image: 'regularbase64', mimeType: 'image/png' }, + ]; + + const result = await expandBlobReferences(content, resourceManager, mockLogger); + + expect(result).toEqual(content); + }); + + test('should expand blob reference in image part', async () => { + const resourceManager = createMockResourceManager({ + aaa000bbb111: { blob: 'resolvedimagedata', mimeType: 'image/png' }, + }); + + const content = [ + { type: 'image' as const, image: '@blob:aaa000bbb111', mimeType: 'image/png' }, + ]; + + const result = await expandBlobReferences(content, resourceManager, mockLogger); + + expect(result.length).toBe(1); + expect(result[0]).toMatchObject({ + type: 'image', + image: 'resolvedimagedata', + mimeType: 'image/png', + }); + }); + + test('should expand blob reference in file part', async () => { + const resourceManager = createMockResourceManager({ + fff000eee111: { blob: 'resolvedfiledata', mimeType: 'application/pdf' }, + }); + + const content = [ + { + type: 'file' as const, + data: '@blob:fff000eee111', + mimeType: 'application/pdf', + filename: 'doc.pdf', + }, + ]; + + const result = await expandBlobReferences(content, resourceManager, mockLogger); + + expect(result.length).toBe(1); + expect(result[0]).toMatchObject({ + type: 'file', + data: 'resolvedfiledata', + mimeType: 'application/pdf', + }); + }); + + test('should handle failed blob resolution gracefully', async () => { + const resourceManager = createMockResourceManager({}); // No blobs available + + const result = await expandBlobReferences( + [{ type: 'text', text: 'Check: @blob:abc000def111' }], + resourceManager, + mockLogger + ); + + // Should return a fallback text part + expect(result.length).toBe(2); + expect(result[0]).toEqual({ type: 'text', text: 'Check: ' }); + expect(result[1]).toMatchObject({ + type: 'text', + text: expect.stringContaining('unavailable'), + }); + }); + + test('should preserve UI resource parts unchanged', async () => { + const resourceManager = createMockResourceManager({}); + const content = [ + { type: 'text' as const, text: 'Hello' }, + { + type: 'ui-resource' as const, + uri: 'ui://example', + mimeType: 'text/html', + content: '
test
', + }, + ]; + + const result = await expandBlobReferences(content, resourceManager, mockLogger); + + expect(result).toEqual(content); + }); + + test('should filter blobs by allowedMediaTypes', async () => { + const resourceManager = { + read: vi.fn(async (_uri: string) => { + return { + contents: [{ blob: 'videodata', mimeType: 'video/mp4' }], + _meta: { size: 1000, originalName: 'video.mp4' }, + }; + }), + } as unknown as import('../resources/index.js').ResourceManager; + + const result = await expandBlobReferences( + [{ type: 'text', text: '@blob:abc123def456' }], + resourceManager, + mockLogger, + ['image/*'] // Only allow images + ); + + // Should return a placeholder since video is not in allowedMediaTypes + expect(result.length).toBe(1); + expect(result[0]).toMatchObject({ + type: 'text', + text: expect.stringContaining('Video'), + }); + }); + + test('should expand allowed media types', async () => { + const resourceManager = { + read: vi.fn(async (_uri: string) => { + return { + contents: [{ blob: 'imagedata', mimeType: 'image/png' }], + _meta: { size: 1000 }, + }; + }), + } as unknown as import('../resources/index.js').ResourceManager; + + const result = await expandBlobReferences( + [{ type: 'text', text: '@blob:abc123def456' }], + resourceManager, + mockLogger, + ['image/*'] // Allow images + ); + + expect(result.length).toBe(1); + expect(result[0]).toMatchObject({ + type: 'image', + image: 'imagedata', + mimeType: 'image/png', + }); + }); +}); + +describe('filterCompacted', () => { + // Note: These tests use string content for simplicity. The actual InternalMessage type + // requires MessageContentPart[], but filterCompacted only checks metadata.isSummary + // and slices the array - it doesn't inspect content structure. + + it('should return all messages if no summary exists', () => { + const messages = [ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there' }, + { role: 'user', content: 'How are you?' }, + ] as unknown as InternalMessage[]; + + const result = filterCompacted(messages); + + expect(result).toEqual(messages); + expect(result).toHaveLength(3); + }); + + it('should return summary and messages after it when summary exists', () => { + // Layout: [summarized, summarized, summary, afterSummary, afterSummary] + // originalMessageCount=2 means first 2 messages were summarized + const messages = [ + { role: 'user', content: 'Old message 1' }, + { role: 'assistant', content: 'Old response 1' }, + { + role: 'assistant', + content: 'Summary of conversation', + metadata: { isSummary: true, originalMessageCount: 2 }, + }, + { role: 'user', content: 'New message' }, + { role: 'assistant', content: 'New response' }, + ] as unknown as InternalMessage[]; + + const result = filterCompacted(messages); + + expect(result).toHaveLength(3); + expect(result[0]?.content).toBe('Summary of conversation'); + expect(result[0]?.metadata?.isSummary).toBe(true); + expect(result[1]?.content).toBe('New message'); + expect(result[2]?.content).toBe('New response'); + }); + + it('should use most recent summary when multiple exist', () => { + // Layout: [summarized, firstSummary, preserved, secondSummary, afterSummary] + // The second summary at index 3 summarized messages 0-2 (originalMessageCount=3) + const messages = [ + { role: 'user', content: 'Very old' }, + { + role: 'assistant', + content: 'First summary', + metadata: { isSummary: true, originalMessageCount: 1 }, + }, + { role: 'user', content: 'Medium old' }, + { + role: 'assistant', + content: 'Second summary', + metadata: { isSummary: true, originalMessageCount: 3 }, + }, + { role: 'user', content: 'Recent message' }, + ] as unknown as InternalMessage[]; + + const result = filterCompacted(messages); + + expect(result).toHaveLength(2); + expect(result[0]?.content).toBe('Second summary'); + expect(result[1]?.content).toBe('Recent message'); + }); + + it('should handle empty history', () => { + const result = filterCompacted([]); + + expect(result).toEqual([]); + expect(result).toHaveLength(0); + }); + + it('should handle history with only a summary', () => { + const messages = [ + { role: 'assistant', content: 'Just a summary', metadata: { isSummary: true } }, + ] as unknown as InternalMessage[]; + + const result = filterCompacted(messages); + + expect(result).toHaveLength(1); + expect(result[0]?.metadata?.isSummary).toBe(true); + }); + + it('should not treat messages with other metadata as summaries', () => { + const messages = [ + { role: 'user', content: 'Message 1' }, + { role: 'assistant', content: 'Response with metadata', metadata: { important: true } }, + { role: 'user', content: 'Message 2' }, + ] as unknown as InternalMessage[]; + + const result = filterCompacted(messages); + + expect(result).toEqual(messages); + expect(result).toHaveLength(3); + }); + + it('should handle summary at the end of history', () => { + // Layout: [summarized, summarized, summary] + // originalMessageCount=2 means first 2 messages were summarized, no preserved messages + const messages = [ + { role: 'user', content: 'Old message' }, + { role: 'assistant', content: 'Old response' }, + { + role: 'assistant', + content: 'Final summary', + metadata: { isSummary: true, originalMessageCount: 2 }, + }, + ] as unknown as InternalMessage[]; + + const result = filterCompacted(messages); + + expect(result).toHaveLength(1); + expect(result[0]?.content).toBe('Final summary'); + }); + + it('should preserve messages between summarized portion and summary', () => { + // This is the typical case after compaction: + // Layout: [summarized, summarized, preserved, preserved, summary] + // originalMessageCount=2 means first 2 messages were summarized + // Messages at indices 2,3 should be preserved + const messages = [ + { role: 'user', content: 'Old message 1' }, + { role: 'assistant', content: 'Old response 1' }, + { role: 'user', content: 'Recent message' }, + { role: 'assistant', content: 'Recent response' }, + { + role: 'system', + content: 'Summary', + metadata: { isSummary: true, originalMessageCount: 2 }, + }, + ] as unknown as InternalMessage[]; + + const result = filterCompacted(messages); + + // Should return: [summary, preserved1, preserved2] + expect(result).toHaveLength(3); + expect(result[0]?.content).toBe('Summary'); + expect(result[1]?.content).toBe('Recent message'); + expect(result[2]?.content).toBe('Recent response'); + }); +}); + +describe('sanitizeToolResultToContentWithBlobs', () => { + describe('string input handling', () => { + it('should wrap simple string in ContentPart array', async () => { + const result = await sanitizeToolResultToContentWithBlobs('Hello, world!', mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]).toEqual({ type: 'text', text: 'Hello, world!' }); + }); + + it('should wrap empty string in ContentPart array', async () => { + const result = await sanitizeToolResultToContentWithBlobs('', mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]).toEqual({ type: 'text', text: '' }); + }); + + it('should truncate long strings and return ContentPart array', async () => { + // MAX_TOOL_TEXT_CHARS is 8000, so create a string longer than that + // Use text with spaces/punctuation to avoid being detected as base64-like + const longString = 'This is a sample text line. '.repeat(400); // ~11200 chars + + const result = await sanitizeToolResultToContentWithBlobs(longString, mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]?.type).toBe('text'); + // Verify truncation happened + const textPart = result![0] as { type: 'text'; text: string }; + expect(textPart.text).toContain('chars omitted'); + expect(textPart.text.length).toBeLessThan(longString.length); + }); + + it('should convert data URI to image ContentPart', async () => { + const dataUri = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + + const result = await sanitizeToolResultToContentWithBlobs(dataUri, mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]?.type).toBe('image'); + }); + + it('should convert data URI to file ContentPart for non-image types', async () => { + const dataUri = 'data:application/pdf;base64,JVBERi0xLjQK'; + + const result = await sanitizeToolResultToContentWithBlobs(dataUri, mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]?.type).toBe('file'); + expect((result![0] as any).mimeType).toBe('application/pdf'); + }); + + it('should treat base64-like strings as text (MCP tools use structured content)', async () => { + // MCP-compliant tools return structured { type: 'image', data: base64, mimeType } + // Raw base64 strings should be treated as regular text + const base64String = Buffer.from('test binary data here for base64 encoding test') + .toString('base64') + .repeat(20); // ~1280 chars + + const result = await sanitizeToolResultToContentWithBlobs(base64String, mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + // Should be text, not file - raw base64 strings aren't valid MCP content + expect(result![0]?.type).toBe('text'); + expect((result![0] as any).text).toBe(base64String); + }); + }); + + describe('object input handling', () => { + it('should stringify simple object and wrap in ContentPart array', async () => { + const result = await sanitizeToolResultToContentWithBlobs( + { key: 'value', number: 42 }, + mockLogger + ); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]?.type).toBe('text'); + const text = (result![0] as { type: 'text'; text: string }).text; + expect(text).toContain('key'); + expect(text).toContain('value'); + }); + + it('should handle null input', async () => { + const result = await sanitizeToolResultToContentWithBlobs(null, mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]?.type).toBe('text'); + }); + + it('should handle undefined input', async () => { + const result = await sanitizeToolResultToContentWithBlobs(undefined, mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]?.type).toBe('text'); + }); + }); + + describe('array input handling', () => { + it('should process array of strings into ContentPart array', async () => { + const result = await sanitizeToolResultToContentWithBlobs( + ['first', 'second', 'third'], + mockLogger + ); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(3); + expect(result![0]).toEqual({ type: 'text', text: 'first' }); + expect(result![1]).toEqual({ type: 'text', text: 'second' }); + expect(result![2]).toEqual({ type: 'text', text: 'third' }); + }); + + it('should handle mixed array with strings and objects', async () => { + const result = await sanitizeToolResultToContentWithBlobs( + ['text message', { data: 123 }], + mockLogger + ); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(2); + expect(result![0]).toEqual({ type: 'text', text: 'text message' }); + expect(result![1]?.type).toBe('text'); + }); + + it('should skip null items in array', async () => { + const result = await sanitizeToolResultToContentWithBlobs( + ['first', null, 'third'], + mockLogger + ); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(2); + expect(result![0]).toEqual({ type: 'text', text: 'first' }); + expect(result![1]).toEqual({ type: 'text', text: 'third' }); + }); + + it('should handle empty array', async () => { + const result = await sanitizeToolResultToContentWithBlobs([], mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(0); + }); + }); + + describe('MCP content handling', () => { + it('should handle MCP text content type', async () => { + const mcpResult = { + content: [{ type: 'text', text: 'MCP text response' }], + }; + + const result = await sanitizeToolResultToContentWithBlobs(mcpResult, mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]).toEqual({ type: 'text', text: 'MCP text response' }); + }); + + it('should handle MCP image content type', async () => { + const mcpResult = { + content: [ + { + type: 'image', + data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk', + mimeType: 'image/png', + }, + ], + }; + + const result = await sanitizeToolResultToContentWithBlobs(mcpResult, mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]?.type).toBe('image'); + }); + }); + + describe('error handling', () => { + it('should handle circular references gracefully', async () => { + const circular: Record = { a: 1 }; + circular.self = circular; + + // Should not throw, should return some text representation + const result = await sanitizeToolResultToContentWithBlobs(circular, mockLogger); + + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(1); + expect(result![0]?.type).toBe('text'); + }); + }); +}); + +describe('Token Estimation Functions', () => { + describe('estimateStringTokens', () => { + it('should return 0 for empty string', () => { + expect(estimateStringTokens('')).toBe(0); + }); + + it('should return 0 for null/undefined', () => { + expect(estimateStringTokens(null as unknown as string)).toBe(0); + expect(estimateStringTokens(undefined as unknown as string)).toBe(0); + }); + + it('should estimate ~4 chars per token', () => { + // 100 chars should be ~25 tokens + const text = 'a'.repeat(100); + expect(estimateStringTokens(text)).toBe(25); + }); + + it('should round to nearest integer', () => { + // 10 chars = 2.5 -> rounds to 3 + expect(estimateStringTokens('a'.repeat(10))).toBe(3); + // 8 chars = 2 -> exactly 2 + expect(estimateStringTokens('a'.repeat(8))).toBe(2); + }); + + it('should handle realistic text content', () => { + const systemPrompt = `You are a helpful coding assistant. + You help users write, debug, and understand code. + Always provide clear explanations.`; + // ~150 chars -> ~38 tokens + const tokens = estimateStringTokens(systemPrompt); + expect(tokens).toBeGreaterThan(30); + expect(tokens).toBeLessThan(50); + }); + }); + + describe('estimateImageTokens', () => { + it('should return fixed 1000 tokens for images', () => { + expect(estimateImageTokens()).toBe(1000); + }); + }); + + describe('estimateFileTokens', () => { + it('should estimate based on content when provided', () => { + const content = 'a'.repeat(400); // 400 chars = 100 tokens + expect(estimateFileTokens(content)).toBe(100); + }); + + it('should return 1000 when no content provided', () => { + expect(estimateFileTokens()).toBe(1000); + expect(estimateFileTokens(undefined)).toBe(1000); + }); + }); + + describe('estimateContentPartTokens', () => { + it('should estimate text parts using string estimation', () => { + const textPart = { type: 'text' as const, text: 'a'.repeat(100) }; + expect(estimateContentPartTokens(textPart)).toBe(25); + }); + + it('should estimate image parts as 1000 tokens', () => { + const imagePart = { + type: 'image' as const, + image: 'base64data', + mimeType: 'image/png' as const, + }; + expect(estimateContentPartTokens(imagePart)).toBe(1000); + }); + + it('should return fallback for file parts', () => { + // File data could be base64-encoded or binary, so we use a conservative fallback + const filePart = { + type: 'file' as const, + data: 'some-file-data', + mimeType: 'text/plain' as const, + }; + expect(estimateContentPartTokens(filePart)).toBe(1000); + }); + + it('should return fallback for file parts with binary data', () => { + // Binary data also uses fallback (can't easily estimate tokens from bytes) + const filePart: FilePart = { + type: 'file', + data: new Uint8Array([1, 2, 3]), + mimeType: 'application/pdf', + }; + expect(estimateContentPartTokens(filePart)).toBe(1000); + }); + + it('should return 0 for unknown part types', () => { + const unknownPart = { type: 'unknown' } as unknown as ContentPart; + expect(estimateContentPartTokens(unknownPart)).toBe(0); + }); + }); + + describe('estimateMessagesTokens', () => { + it('should return 0 for empty messages array', () => { + expect(estimateMessagesTokens([])).toBe(0); + }); + + it('should estimate single text message', () => { + const messages: InternalMessage[] = [ + { + role: 'user', + content: [{ type: 'text', text: 'a'.repeat(100) }], + }, + ]; + expect(estimateMessagesTokens(messages)).toBe(25); + }); + + it('should sum tokens across multiple messages', () => { + const messages: InternalMessage[] = [ + { + role: 'user', + content: [{ type: 'text', text: 'a'.repeat(100) }], // 25 tokens + }, + { + role: 'assistant', + content: [{ type: 'text', text: 'a'.repeat(200) }], // 50 tokens + }, + ]; + expect(estimateMessagesTokens(messages)).toBe(75); + }); + + it('should sum tokens across multiple content parts', () => { + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'a'.repeat(100) }, // 25 tokens + { type: 'image', image: 'base64', mimeType: 'image/png' as const }, // 1000 tokens + ], + }, + ]; + expect(estimateMessagesTokens(messages)).toBe(1025); + }); + + it('should handle messages with non-array content', () => { + const messages: InternalMessage[] = [ + { + role: 'user', + content: 'plain string content' as any, // Not an array - should be skipped + }, + ]; + expect(estimateMessagesTokens(messages)).toBe(0); + }); + + it('should handle mixed content types', () => { + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Hello' }, // ~1-2 tokens + { type: 'image', image: 'base64', mimeType: 'image/png' as const }, // 1000 tokens + { type: 'file', data: 'base64', mimeType: 'application/pdf' as const }, // 1000 tokens + ], + }, + ]; + const tokens = estimateMessagesTokens(messages); + expect(tokens).toBeGreaterThanOrEqual(2001); // At least 2000 + some text + }); + }); +}); + +// Note: getOutputBuffer and DEFAULT_OUTPUT_BUFFER were removed from overflow.ts +// The output buffer concept was flawed - input and output tokens have separate limits +// in LLM APIs, so reserving input space for output was unnecessary. + +describe('estimateToolsTokens', () => { + it('should return 0 total for empty tools object', () => { + const result = estimateToolsTokens({}); + expect(result.total).toBe(0); + expect(result.perTool).toEqual([]); + }); + + it('should estimate tokens for single tool', () => { + const tools = { + search: { + name: 'search', + description: 'Search the web for information', + parameters: { type: 'object', properties: { query: { type: 'string' } } }, + }, + }; + const result = estimateToolsTokens(tools); + expect(result.total).toBeGreaterThan(0); + expect(result.perTool).toHaveLength(1); + expect(result.perTool[0]?.name).toBe('search'); + expect(result.perTool[0]?.tokens).toBeGreaterThan(0); + }); + + it('should estimate tokens for multiple tools', () => { + const tools = { + read_file: { + name: 'read_file', + description: 'Read a file from disk', + parameters: { type: 'object', properties: { path: { type: 'string' } } }, + }, + write_file: { + name: 'write_file', + description: 'Write content to a file', + parameters: { + type: 'object', + properties: { path: { type: 'string' }, content: { type: 'string' } }, + }, + }, + }; + const result = estimateToolsTokens(tools); + expect(result.total).toBeGreaterThan(0); + expect(result.perTool).toHaveLength(2); + // Total should equal sum of per-tool tokens + const sumOfPerTool = result.perTool.reduce((sum, t) => sum + t.tokens, 0); + expect(result.total).toBe(sumOfPerTool); + }); + + it('should use key as tool name when name property is missing', () => { + const tools = { + my_tool: { + description: 'A tool without a name property', + parameters: {}, + }, + }; + const result = estimateToolsTokens(tools); + expect(result.perTool[0]?.name).toBe('my_tool'); + }); + + it('should handle tools with complex parameters', () => { + const tools = { + complex_tool: { + name: 'complex_tool', + description: 'A tool with complex nested parameters', + parameters: { + type: 'object', + properties: { + nested: { + type: 'object', + properties: { + array: { type: 'array', items: { type: 'string' } }, + number: { type: 'number' }, + }, + }, + }, + }, + }, + }; + const result = estimateToolsTokens(tools); + // Complex parameters should result in more tokens + expect(result.total).toBeGreaterThan(20); + }); +}); + +describe('estimateContextTokens', () => { + it('should return total and breakdown with all components', () => { + const systemPrompt = 'You are a helpful assistant.'; + const messages: InternalMessage[] = [ + { role: 'user', content: [{ type: 'text', text: 'Hello!' }] }, + ]; + const tools = { + search: { + name: 'search', + description: 'Search the web', + parameters: {}, + }, + }; + + const result = estimateContextTokens(systemPrompt, messages, tools); + + expect(result.total).toBeGreaterThan(0); + expect(result.breakdown.systemPrompt).toBeGreaterThan(0); + expect(result.breakdown.messages).toBeGreaterThan(0); + expect(result.breakdown.tools.total).toBeGreaterThan(0); + expect(result.breakdown.tools.perTool).toHaveLength(1); + }); + + it('should return 0 for tools when no tools provided', () => { + const systemPrompt = 'You are helpful.'; + const messages: InternalMessage[] = [ + { role: 'user', content: [{ type: 'text', text: 'Hi' }] }, + ]; + + const result = estimateContextTokens(systemPrompt, messages); + + expect(result.breakdown.tools.total).toBe(0); + expect(result.breakdown.tools.perTool).toEqual([]); + }); + + it('should have total equal to sum of breakdown components', () => { + const systemPrompt = 'System instructions here.'; + const messages: InternalMessage[] = [ + { role: 'user', content: [{ type: 'text', text: 'User message' }] }, + { role: 'assistant', content: [{ type: 'text', text: 'Assistant response' }] }, + ]; + const tools = { + tool1: { name: 'tool1', description: 'First tool', parameters: {} }, + tool2: { name: 'tool2', description: 'Second tool', parameters: {} }, + }; + + const result = estimateContextTokens(systemPrompt, messages, tools); + + const expectedTotal = + result.breakdown.systemPrompt + + result.breakdown.messages + + result.breakdown.tools.total; + expect(result.total).toBe(expectedTotal); + }); + + it('should handle empty messages array', () => { + const systemPrompt = 'System prompt'; + const messages: InternalMessage[] = []; + + const result = estimateContextTokens(systemPrompt, messages); + + expect(result.breakdown.messages).toBe(0); + expect(result.breakdown.systemPrompt).toBeGreaterThan(0); + expect(result.total).toBe(result.breakdown.systemPrompt); + }); + + it('should handle empty system prompt', () => { + const systemPrompt = ''; + const messages: InternalMessage[] = [ + { role: 'user', content: [{ type: 'text', text: 'Hello' }] }, + ]; + + const result = estimateContextTokens(systemPrompt, messages); + + expect(result.breakdown.systemPrompt).toBe(0); + expect(result.breakdown.messages).toBeGreaterThan(0); + }); +}); diff --git a/dexto/packages/core/src/context/utils.ts b/dexto/packages/core/src/context/utils.ts new file mode 100644 index 00000000..8e76f5dc --- /dev/null +++ b/dexto/packages/core/src/context/utils.ts @@ -0,0 +1,1947 @@ +import { + InternalMessage, + TextPart, + ImagePart, + FilePart, + UIResourcePart, + ContentPart, + SanitizedToolResult, + isToolMessage, +} from './types.js'; +import { isValidDisplayData, type ToolDisplayData } from '../tools/display-types.js'; +import type { IDextoLogger } from '@core/logger/v2/types.js'; +import { validateModelFileSupport } from '@core/llm/registry.js'; +import { LLMContext } from '@core/llm/types.js'; +import { safeStringify } from '@core/utils/safe-stringify.js'; +import { getFileMediaKind, getResourceKind } from './media-helpers.js'; + +// Tunable heuristics and shared constants +const MIN_BASE64_HEURISTIC_LENGTH = 512; // Below this length, treat as regular text +const MAX_TOOL_TEXT_CHARS = 8000; // Truncate overly long tool text + +type ToolBlobNamingOptions = { + toolName?: string; + toolCallId?: string; +}; + +const MIN_TOOL_INLINE_MEDIA_BYTES = 1024; + +type InlineMediaKind = 'image' | 'file'; + +type InlineMediaHint = { + index: number; + kind: InlineMediaKind; + mimeType: string; + approxBytes: number; + data: string | Buffer; + filename?: string | undefined; +}; + +export interface NormalizedToolResult { + parts: Array; + uiResources: UIResourcePart[]; + inlineMedia: InlineMediaHint[]; +} + +interface PersistToolMediaOptions { + blobStore?: import('../storage/blob/types.js').BlobStore; + toolName?: string; + toolCallId?: string; +} + +interface PersistToolMediaResult { + parts: Array; + uiResources: UIResourcePart[]; + resources?: SanitizedToolResult['resources']; +} + +function slugifyForFilename(value: string, maxLength = 48): string | null { + if (!value) return null; + const slug = value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + if (!slug) return null; + return slug.length > maxLength ? slug.slice(0, maxLength) : slug; +} + +function inferExtensionFromMime(mimeType: string | undefined, fallback: string): string { + if (!mimeType) return fallback; + const subtype = mimeType.split('/')[1]?.split(';')[0]?.split('+')[0]; + if (!subtype) return fallback; + const clean = subtype.replace(/[^a-z0-9]/gi, '').toLowerCase(); + return clean || fallback; +} + +function sanitizeExistingFilename(filename: string): string { + return filename.replace(/[^a-zA-Z0-9._-]+/g, '-'); +} + +function generateUniqueSuffix(): string { + return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`; +} + +function clonePart(part: TextPart | ImagePart | FilePart): TextPart | ImagePart | FilePart { + if (part.type === 'text') { + return { type: 'text', text: part.text }; + } + + if (part.type === 'image') { + const cloned: ImagePart = { + type: 'image', + image: part.image, + }; + if (part.mimeType) { + cloned.mimeType = part.mimeType; + } + return cloned; + } + + const cloned: FilePart = { + type: 'file', + data: part.data, + mimeType: part.mimeType, + }; + if (part.filename) { + cloned.filename = part.filename; + } + return cloned; +} + +function coerceContentToParts( + content: ContentPart[] | null +): Array { + if (content == null) { + return []; + } + + const normalized: Array = []; + for (const item of content) { + // Filter out UIResourcePart - only keep ContentPart types + if (item.type === 'ui-resource') { + continue; + } + if (item.type === 'text') { + normalized.push({ type: 'text', text: item.text }); + } else if (item.type === 'image') { + const cloned: ImagePart = { + type: 'image', + image: item.image, + }; + if (item.mimeType) { + cloned.mimeType = item.mimeType; + } + normalized.push(cloned); + } else if (item.type === 'file') { + const cloned: FilePart = { + type: 'file', + data: item.data, + mimeType: item.mimeType ?? 'application/octet-stream', + }; + if (item.filename) { + cloned.filename = item.filename; + } + normalized.push(cloned); + } + } + return normalized; +} + +function detectInlineMedia( + part: TextPart | ImagePart | FilePart, + index: number +): InlineMediaHint | null { + if (part.type === 'text') { + return null; + } + + if (part.type === 'image') { + const value = part.image; + const mimeType = part.mimeType ?? 'image/jpeg'; + if (typeof value === 'string') { + if (value.startsWith('@blob:')) return null; + if ( + value.startsWith('http://') || + value.startsWith('https://') || + value.startsWith('blob:') + ) { + return null; + } + if (isLikelyBase64String(value, 128)) { + return { + index, + kind: 'image', + mimeType, + approxBytes: base64LengthToBytes(value.length), + data: value, + }; + } + } else if (value instanceof Buffer) { + return { + index, + kind: 'image', + mimeType, + approxBytes: value.length, + data: value, + }; + } else if (value instanceof Uint8Array) { + const buffer = Buffer.from(value); + return { + index, + kind: 'image', + mimeType, + approxBytes: buffer.length, + data: buffer, + }; + } else if (value instanceof ArrayBuffer) { + const buffer = Buffer.from(new Uint8Array(value)); + return { + index, + kind: 'image', + mimeType, + approxBytes: buffer.length, + data: buffer, + }; + } + return null; + } + + const data = part.data; + const mimeType = part.mimeType ?? 'application/octet-stream'; + const filename = part.filename; + + if (typeof data === 'string') { + if (data.startsWith('@blob:')) return null; + if (data.startsWith('http://') || data.startsWith('https://') || data.startsWith('blob:')) { + return null; + } + if (data.startsWith('data:')) { + const parsed = parseDataUri(data); + if (parsed) { + return { + index, + kind: 'file', + mimeType: parsed.mediaType, + approxBytes: base64LengthToBytes(parsed.base64.length), + data: parsed.base64, + filename, + }; + } + } + if (isLikelyBase64String(data, 128)) { + return { + index, + kind: 'file', + mimeType, + approxBytes: base64LengthToBytes(data.length), + data, + filename, + }; + } + } else if (data instanceof Buffer) { + return { + index, + kind: 'file', + mimeType, + approxBytes: data.length, + data, + filename, + }; + } else if (data instanceof Uint8Array) { + const buffer = Buffer.from(data); + return { + index, + kind: 'file', + mimeType, + approxBytes: buffer.length, + data: buffer, + filename, + }; + } else if (data instanceof ArrayBuffer) { + const buffer = Buffer.from(new Uint8Array(data)); + return { + index, + kind: 'file', + mimeType, + approxBytes: buffer.length, + data: buffer, + filename, + }; + } + + return null; +} + +function buildToolBlobName( + kind: 'output' | 'image' | 'file', + mimeType: string | undefined, + options: ToolBlobNamingOptions | undefined, + preferredName?: string +): string { + if (preferredName) { + return sanitizeExistingFilename(preferredName); + } + + const toolSegment = slugifyForFilename(options?.toolName ?? '', 40); + const callSegment = slugifyForFilename(options?.toolCallId ?? '', 16); + const parts = ['tool']; + if (toolSegment) parts.push(toolSegment); + if (callSegment) parts.push(callSegment); + parts.push(kind); + const ext = inferExtensionFromMime( + mimeType, + kind === 'image' ? 'jpg' : kind === 'file' ? 'bin' : 'bin' + ); + const unique = generateUniqueSuffix(); + return `${parts.join('-')}-${unique}.${ext}`; +} + +async function resolveBlobReferenceToParts( + resourceUri: string, + resourceManager: import('../resources/index.js').ResourceManager, + logger: IDextoLogger, + allowedMediaTypes?: string[] +): Promise> { + try { + const result = await resourceManager.read(resourceUri); + + // Check if this blob type is allowed (if filtering is enabled) + if (allowedMediaTypes) { + const mimeType = result.contents[0]?.mimeType; + const metadata = result._meta as { size?: number; originalName?: string } | undefined; + + if (mimeType && !matchesAnyMimePattern(mimeType, allowedMediaTypes)) { + // Generate placeholder for filtered media + const placeholderMetadata: { + mimeType: string; + size: number; + originalName?: string; + } = { + mimeType, + size: metadata?.size ?? 0, + }; + if (metadata?.originalName) { + placeholderMetadata.originalName = metadata.originalName; + } + const placeholder = generateMediaPlaceholder(placeholderMetadata); + return [{ type: 'text', text: placeholder }]; + } + } + + const parts: Array = []; + + for (const item of result.contents ?? []) { + if (!item || typeof item !== 'object') { + continue; + } + + if (typeof (item as { text?: unknown }).text === 'string') { + parts.push({ type: 'text', text: (item as { text: string }).text }); + continue; + } + + const base64Data = + 'blob' in item && typeof item.blob === 'string' + ? item.blob + : 'data' in item && typeof (item as any).data === 'string' + ? (item as any).data + : undefined; + const mimeType = typeof item.mimeType === 'string' ? item.mimeType : undefined; + if (!base64Data || !mimeType) { + continue; + } + + const resolvedMime = mimeType ?? 'application/octet-stream'; + + if (resolvedMime.startsWith('image/')) { + // Return raw base64, NOT data URI format + // LLM APIs (Anthropic, OpenAI, etc.) expect raw base64, not data:... URIs + const imagePart: ImagePart = { + type: 'image', + image: base64Data, + mimeType: resolvedMime, + }; + parts.push(imagePart); + continue; + } + + // Return raw base64 for all file types - mimeType is provided separately + // LLM APIs expect raw base64, not data:... URIs + const filePart: FilePart = { + type: 'file', + data: base64Data, + mimeType: resolvedMime, + }; + const itemWithFilename = item as any; + if ( + typeof itemWithFilename.filename === 'string' && + itemWithFilename.filename.length > 0 + ) { + filePart.filename = itemWithFilename.filename; + } else if (typeof result._meta?.originalName === 'string') { + filePart.filename = result._meta.originalName; + } + parts.push(filePart); + } + + if (parts.length === 0) { + const fallbackName = + (typeof result._meta?.originalName === 'string' && result._meta.originalName) || + resourceUri; + parts.push({ type: 'text', text: `[Attachment: ${fallbackName}]` }); + } + + return parts; + } catch (error) { + // logger is not available in this utility function + logger.warn(`Failed to resolve blob reference ${resourceUri}: ${String(error)}`); + return [{ type: 'text', text: `[Attachment unavailable: ${resourceUri}]` }]; + } +} + +// ============= TOKEN ESTIMATION ============= +// These functions provide rough token estimates using heuristics. +// Used for context management, compaction decisions, and UI display. +// Actual token counts come from the LLM API response. + +/** + * Estimate tokens for a text string. + * Uses the common heuristic of ~4 characters per token. + */ +export function estimateStringTokens(text: string): number { + if (!text) return 0; + return Math.round(text.length / 4); +} + +/** + * Estimate tokens for an image. + * Images use a fixed token budget regardless of dimensions. + * Based on typical LLM pricing (~1000 tokens per image). + */ +export function estimateImageTokens(): number { + return 1000; +} + +/** + * Estimate tokens for a file based on its content. + * If content is available, estimates based on text length. + * Falls back to a default estimate if no content provided. + */ +export function estimateFileTokens(content?: string): number { + if (content) { + return estimateStringTokens(content); + } + // Fallback for when content is not available + return 1000; +} + +/** + * Estimate tokens for a content part (text, image, or file). + */ +export function estimateContentPartTokens(part: ContentPart): number { + if (part.type === 'text') { + return estimateStringTokens(part.text); + } + if (part.type === 'image') { + return estimateImageTokens(); + } + if (part.type === 'file') { + // File parts use a simple fallback since: + // 1. After first LLM call, we use actual token counts for the bulk of estimation + // 2. This only affects the "new messages" delta and /context display + // 3. File attachments in tool results are relatively rare + return 1000; + } + return 0; +} + +/** + * Estimate tokens for an array of messages. + * Used for telemetry/logging only - actual token counts come from the LLM API. + */ +export function estimateMessagesTokens(messages: readonly InternalMessage[]): number { + let total = 0; + for (const msg of messages) { + if (!Array.isArray(msg.content)) continue; + for (const part of msg.content) { + total += estimateContentPartTokens(part); + } + } + return total; +} + +/** + * Tool definition interface for token estimation. + * Matches the structure used by both ToolManager and getContextTokenEstimate. + */ +export interface ToolDefinition { + name?: string; + description?: string; + parameters?: unknown; +} + +/** + * Estimate tokens for tool definitions. + * Returns both total and per-tool breakdown for UI display. + */ +export function estimateToolsTokens(tools: Record): { + total: number; + perTool: Array<{ name: string; tokens: number }>; +} { + const perTool: Array<{ name: string; tokens: number }> = []; + let total = 0; + for (const [key, tool] of Object.entries(tools)) { + const toolName = tool.name || key; + const toolDescription = tool.description || ''; + const toolSchema = JSON.stringify(tool.parameters || {}); + const tokens = estimateStringTokens(toolName + toolDescription + toolSchema); + perTool.push({ name: toolName, tokens }); + total += tokens; + } + return { total, perTool }; +} + +/** + * Result of context token estimation with breakdown. + */ +export interface ContextTokenEstimate { + /** Total estimated tokens */ + total: number; + /** Breakdown by category */ + breakdown: { + systemPrompt: number; + messages: number; + tools: { + total: number; + perTool: Array<{ name: string; tokens: number }>; + }; + }; +} + +/** + * Estimate total context tokens for LLM calls. + * This is the single source of truth for context token estimation, + * used by both /context overlay and compaction pre-check. + * + * IMPORTANT: The `preparedHistory` parameter must be the result of + * `ContextManager.prepareHistory()` or `getFormattedMessagesForLLM()`. + * This ensures messages are properly filtered (compacted messages removed) + * and pruned tool outputs are replaced with placeholders. + * + * @param systemPrompt The system prompt string + * @param preparedHistory Message history AFTER filterCompacted and pruning + * @param tools Optional tool definitions - if not provided, tools are not counted + * @returns Token estimate with total and breakdown + */ +export function estimateContextTokens( + systemPrompt: string, + preparedHistory: readonly InternalMessage[], + tools?: Record +): ContextTokenEstimate { + const systemPromptTokens = estimateStringTokens(systemPrompt); + const messagesTokens = estimateMessagesTokens(preparedHistory); + const toolsEstimate = tools ? estimateToolsTokens(tools) : { total: 0, perTool: [] }; + + return { + total: systemPromptTokens + toolsEstimate.total + messagesTokens, + breakdown: { + systemPrompt: systemPromptTokens, + messages: messagesTokens, + tools: toolsEstimate, + }, + }; +} + +/** + * Extracts image data (base64 or URL) from an ImagePart or raw buffer. + * @param imagePart The image part containing image data + * @returns Base64-encoded string or URL string + */ +export function getImageData( + imagePart: { + image: string | Uint8Array | Buffer | ArrayBuffer | URL; + }, + logger: IDextoLogger +): string { + const { image } = imagePart; + if (typeof image === 'string') { + return image; + } else if (image instanceof Buffer) { + return image.toString('base64'); + } else if (image instanceof Uint8Array) { + return Buffer.from(image).toString('base64'); + } else if (image instanceof ArrayBuffer) { + return Buffer.from(new Uint8Array(image)).toString('base64'); + } else if (image instanceof URL) { + return image.toString(); + } + logger.warn(`Unexpected image data type in getImageData: ${typeof image}`); + return ''; +} + +/** + * Extracts file data (base64 or URL) from a FilePart or raw buffer. + * @param filePart The file part containing file data + * @param logger Optional logger instance + * @returns Base64-encoded string or URL string + */ +export function getFileData( + filePart: { + data: string | Uint8Array | Buffer | ArrayBuffer | URL; + }, + logger: IDextoLogger +): string { + const { data } = filePart; + if (typeof data === 'string') { + return data; + } else if (data instanceof Buffer) { + return data.toString('base64'); + } else if (data instanceof Uint8Array) { + return Buffer.from(data).toString('base64'); + } else if (data instanceof ArrayBuffer) { + return Buffer.from(new Uint8Array(data)).toString('base64'); + } else if (data instanceof URL) { + return data.toString(); + } + logger.warn(`Unexpected file data type in getFileData: ${typeof data}`); + return ''; +} + +/** + * Extracts image data with blob resolution support. + * If the image is a blob reference, resolves it from the resource manager. + * @param imagePart The image part containing image data or blob reference + * @param resourceManager Resource manager for resolving blob references + * @param logger Optional logger instance + * @returns Promise + */ +export async function getImageDataWithBlobSupport( + imagePart: { + image: string | Uint8Array | Buffer | ArrayBuffer | URL; + }, + resourceManager: import('../resources/index.js').ResourceManager, + logger: IDextoLogger +): Promise { + const { image } = imagePart; + + // Check if it's a blob reference + if (typeof image === 'string' && image.startsWith('@blob:')) { + try { + const uri = image.substring(1); // Remove @ prefix + const resourceUri = uri.startsWith('blob:') ? uri : `blob:${uri}`; + const result = await resourceManager.read(resourceUri); + + const firstContent = result.contents[0]; + if ( + firstContent && + 'blob' in firstContent && + firstContent.blob && + typeof firstContent.blob === 'string' + ) { + return firstContent.blob; + } + logger.warn(`Blob reference ${image} did not contain blob data`); + } catch (error) { + logger.warn(`Failed to resolve blob reference ${image}: ${String(error)}`); + } + } + + // Fallback to original behavior + return getImageData(imagePart, logger); +} + +/** + * Extracts file data with blob resolution support. + * If the data is a blob reference, resolves it from the resource manager. + * @param filePart The file part containing file data or blob reference + * @param resourceManager Resource manager for resolving blob references + * @returns Promise + */ +export async function getFileDataWithBlobSupport( + filePart: { + data: string | Uint8Array | Buffer | ArrayBuffer | URL; + }, + resourceManager: import('../resources/index.js').ResourceManager, + logger: IDextoLogger +): Promise { + const { data } = filePart; + + // Check if it's a blob reference + if (typeof data === 'string' && data.startsWith('@blob:')) { + try { + const uri = data.substring(1); // Remove @ prefix + const resourceUri = uri.startsWith('blob:') ? uri : `blob:${uri}`; + const result = await resourceManager.read(resourceUri); + + const firstContent = result.contents[0]; + if ( + firstContent && + 'blob' in firstContent && + firstContent.blob && + typeof firstContent.blob === 'string' + ) { + return firstContent.blob; + } + logger.warn(`Blob reference ${data} did not contain blob data`); + } catch (error) { + logger.warn(`Failed to resolve blob reference ${data}: ${String(error)}`); + } + } + + // Fallback to original behavior + return getFileData(filePart, logger); +} + +/** + * Helper: Expand blob references within a single text string. + * Returns array of parts (text segments + resolved blobs). + */ +async function expandBlobsInText( + text: string, + resourceManager: import('../resources/index.js').ResourceManager, + logger: IDextoLogger, + allowedMediaTypes?: string[] +): Promise> { + if (!text.includes('@blob:')) { + return [{ type: 'text', text }]; + } + + const blobRefPattern = /@blob:[a-f0-9]+/g; + const matches = [...text.matchAll(blobRefPattern)]; + + if (matches.length === 0) { + return [{ type: 'text', text }]; + } + + const resolvedCache = new Map>(); + const parts: Array = []; + let lastIndex = 0; + + for (const match of matches) { + const matchIndex = match.index ?? 0; + const token = match[0]; + if (matchIndex > lastIndex) { + const segment = text.slice(lastIndex, matchIndex); + if (segment.length > 0) { + parts.push({ type: 'text', text: segment }); + } + } + + const uri = token.substring(1); // Remove leading @ + const resourceUri = uri.startsWith('blob:') ? uri : `blob:${uri}`; + + let resolvedParts = resolvedCache.get(resourceUri); + if (!resolvedParts) { + resolvedParts = await resolveBlobReferenceToParts( + resourceUri, + resourceManager, + logger, + allowedMediaTypes + ); + resolvedCache.set(resourceUri, resolvedParts); + } + + if (resolvedParts.length > 0) { + parts.push(...resolvedParts.map((p) => ({ ...p }))); + } else { + parts.push({ type: 'text', text: token }); + } + + lastIndex = matchIndex + token.length; + } + + if (lastIndex < text.length) { + const trailing = text.slice(lastIndex); + if (trailing.length > 0) { + parts.push({ type: 'text', text: trailing }); + } + } + + return parts.filter((p) => p.type !== 'text' || p.text.length > 0); +} + +/** + * Resolves blob references in message content to actual data. + * Expands @blob:id references to their actual base64 content for LLM consumption. + * Can optionally filter by MIME type patterns - unsupported types are replaced with descriptive placeholders. + * + * @param content The message content that may contain blob references + * @param resourceManager Resource manager for resolving blob references + * @param allowedMediaTypes Optional array of MIME patterns (e.g., ["image/*", "application/pdf"]). + * If provided, only matching blobs are expanded; others become placeholders. + * If omitted, all blobs are expanded (legacy behavior). + * @returns Promise + */ +// Overload: null returns empty array +export async function expandBlobReferences( + content: null, + resourceManager: import('../resources/index.js').ResourceManager, + logger: IDextoLogger, + allowedMediaTypes?: string[] +): Promise; +// Overload: ContentPart[] returns ContentPart[] +export async function expandBlobReferences( + content: ContentPart[], + resourceManager: import('../resources/index.js').ResourceManager, + logger: IDextoLogger, + allowedMediaTypes?: string[] +): Promise; +// Overload: ContentPart[] | null (for InternalMessage['content']) +export async function expandBlobReferences( + content: ContentPart[] | null, + resourceManager: import('../resources/index.js').ResourceManager, + logger: IDextoLogger, + allowedMediaTypes?: string[] +): Promise; +// Implementation +export async function expandBlobReferences( + content: ContentPart[] | null, + resourceManager: import('../resources/index.js').ResourceManager, + logger: IDextoLogger, + allowedMediaTypes?: string[] +): Promise { + // Handle null/undefined content + if (content == null || !Array.isArray(content)) { + return []; + } + + const expandedParts: Array = []; + + for (const part of content) { + // UIResourcePart doesn't have blob references - pass through unchanged + if (part.type === 'ui-resource') { + expandedParts.push(part); + continue; + } + + if ( + part.type === 'image' && + typeof part.image === 'string' && + part.image.startsWith('@blob:') + ) { + const uri = part.image.substring(1); + const resourceUri = uri.startsWith('blob:') ? uri : `blob:${uri}`; + const resolved = await resolveBlobReferenceToParts( + resourceUri, + resourceManager, + logger, + allowedMediaTypes + ); + if (resolved.length > 0) { + expandedParts.push(...resolved.map((p) => ({ ...p }))); + } else { + expandedParts.push(part); + } + continue; + } + + if ( + part.type === 'file' && + typeof part.data === 'string' && + part.data.startsWith('@blob:') + ) { + const uri = part.data.substring(1); + const resourceUri = uri.startsWith('blob:') ? uri : `blob:${uri}`; + const resolved = await resolveBlobReferenceToParts( + resourceUri, + resourceManager, + logger, + allowedMediaTypes + ); + if (resolved.length > 0) { + expandedParts.push(...resolved.map((p) => ({ ...p }))); + } else { + try { + const resolvedData = await getFileDataWithBlobSupport( + part, + resourceManager, + logger + ); + expandedParts.push({ ...part, data: resolvedData }); + } catch (error) { + logger.warn(`Failed to resolve file blob reference: ${String(error)}`); + expandedParts.push(part); + } + } + continue; + } + + if (part.type === 'text' && part.text.includes('@blob:')) { + // Expand blob references in text part using helper + const expanded = await expandBlobsInText( + part.text, + resourceManager, + logger, + allowedMediaTypes + ); + expandedParts.push(...expanded); + continue; + } + + expandedParts.push(part); + } + + return expandedParts; +} + +/** + * Filters message content based on LLM capabilities. + * Removes unsupported file attachments while preserving supported content. + * Uses model-specific validation when available, falls back to provider-level. + * @param messages Array of internal messages to filter + * @param config The LLM configuration (provider and optional model) + * @returns Filtered messages with unsupported content removed + */ +export function filterMessagesByLLMCapabilities( + messages: InternalMessage[], + config: LLMContext, + logger: IDextoLogger +): InternalMessage[] { + try { + return messages.map((message) => { + // Only filter user messages with array content (multimodal) + if (message.role !== 'user' || !Array.isArray(message.content)) { + return message; + } + + const filteredContent = message.content.filter((part) => { + // Keep text and image parts + if (part.type === 'text' || part.type === 'image') { + return true; + } + + // Filter file parts based on LLM capabilities + if (part.type === 'file' && part.mimeType) { + const validation = validateModelFileSupport( + config.provider, + config.model, + part.mimeType + ); + return validation.isSupported; + } + + return true; // Keep unknown part types + }); + + // If all content was filtered out, add a placeholder text + if (filteredContent.length === 0) { + filteredContent.push({ + type: 'text', + text: `[File attachment removed - not supported by ${config.model}]`, + }); + } + + return { + ...message, + content: filteredContent, + }; + }); + } catch (error) { + // If filtering fails, return original messages to avoid breaking the flow + logger.warn(`Failed to filter messages by LLM capabilities: ${String(error)}`); + return messages; + } +} + +/** + * Detect if a string is likely a Base64 blob (not a typical sentence/text). + * Uses a length threshold and character set heuristic. + */ +export function isLikelyBase64String( + value: string, + minLength: number = MIN_BASE64_HEURISTIC_LENGTH +): boolean { + if (!value || value.length < minLength) return false; + // Fast-path for data URIs which embed base64 + if (value.startsWith('data:') && value.includes(';base64,')) return true; + // Heuristic: base64 characters only and length divisible by 4 (allow small remainder due to padding) + const b64Regex = /^[A-Za-z0-9+/=\r\n]+$/; + if (!b64Regex.test(value)) return false; + // Low whitespace / punctuation typical for base64 + const nonWordRatio = (value.match(/[^A-Za-z0-9+/=]/g)?.length || 0) / value.length; + return nonWordRatio < 0.01; +} + +/** + * Parse data URI and return { mediaType, base64 } or null if not a data URI. + */ +export function parseDataUri(value: string): { mediaType: string; base64: string } | null { + if (!value.startsWith('data:')) return null; + const commaIdx = value.indexOf(','); + if (commaIdx === -1) return null; + const meta = value.slice(5, commaIdx); // skip 'data:' + if (!/;base64$/i.test(meta)) return null; + const mediaType = meta.replace(/;base64$/i, '') || 'application/octet-stream'; + const base64 = value.slice(commaIdx + 1); + return { mediaType, base64 }; +} + +// Re-export browser-safe helpers for convenience (already imported above) +export { getFileMediaKind, getResourceKind }; + +/** + * Check if a MIME type matches a pattern with wildcard support. + * Supports exact matches and wildcard patterns: + * - "image/png" matches "image/png" exactly + * - "image/star" (where star is asterisk) matches "image/png", "image/jpeg", etc. + * - Single asterisk or "asterisk/asterisk" matches everything + * + * @param mimeType The MIME type to check (e.g., "image/png") + * @param pattern The pattern to match against (e.g., "image/asterisk") + * @returns true if the MIME type matches the pattern + */ +export function matchesMimePattern(mimeType: string | undefined, pattern: string): boolean { + if (!mimeType) return false; + + // Normalize to lowercase for case-insensitive comparison + const normalizedMime = mimeType.toLowerCase().trim(); + const normalizedPattern = pattern.toLowerCase().trim(); + + // Match everything + if (normalizedPattern === '*' || normalizedPattern === '*/*') { + return true; + } + + // Exact match + if (normalizedMime === normalizedPattern) { + return true; + } + + // Wildcard pattern (e.g., "image/*") + if (normalizedPattern.endsWith('/*')) { + const patternType = normalizedPattern.slice(0, -2); // Remove "/*" + const mimeType = normalizedMime.split('/')[0]; // Get type part + return mimeType === patternType; + } + + return false; +} + +/** + * Check if a MIME type matches any pattern in an array of patterns. + * + * @param mimeType The MIME type to check + * @param patterns Array of MIME patterns to match against + * @returns true if the MIME type matches any pattern + */ +export function matchesAnyMimePattern(mimeType: string | undefined, patterns: string[]): boolean { + return patterns.some((pattern) => matchesMimePattern(mimeType, pattern)); +} + +/** + * Convert supported file types to MIME type patterns. + * Used to translate LLM registry file types to MIME patterns for filtering. + * + * @param fileTypes Array of supported file types from LLM registry (e.g., ['image', 'pdf', 'audio']) + * @returns Array of MIME type patterns (e.g., ['image/*', 'application/pdf', 'audio/*']) + */ +export function fileTypesToMimePatterns(fileTypes: string[], logger: IDextoLogger): string[] { + const patterns: string[] = []; + for (const fileType of fileTypes) { + switch (fileType) { + case 'image': + patterns.push('image/*'); + break; + case 'pdf': + patterns.push('application/pdf'); + break; + case 'audio': + patterns.push('audio/*'); + break; + case 'video': + patterns.push('video/*'); + break; + default: + // Unknown file type - skip it + logger.warn(`Unknown file type in registry: ${fileType}`); + } + } + return patterns; +} + +/** + * Generate a descriptive placeholder for filtered media. + * Returns a clean, LLM-readable reference like: [Video: demo.mp4 (5.2 MB)] + * + * @param metadata Blob metadata containing MIME type, size, and original name + * @returns Formatted placeholder string + */ +function generateMediaPlaceholder(metadata: { + mimeType: string; + size: number; + originalName?: string; +}): string { + // Determine media type label + let typeLabel = 'File'; + if (metadata.mimeType.startsWith('video/')) typeLabel = 'Video'; + else if (metadata.mimeType.startsWith('audio/')) typeLabel = 'Audio'; + else if (metadata.mimeType.startsWith('image/')) typeLabel = 'Image'; + else if (metadata.mimeType === 'application/pdf') typeLabel = 'PDF'; + + // Format size in human-readable format + const formatSize = (bytes: number): string => { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]; + }; + + const size = formatSize(metadata.size); + const name = metadata.originalName || 'unknown'; + + return `[${typeLabel}: ${name} (${size})]`; +} + +/** + * Recursively sanitize objects by replacing suspiciously-large base64 strings + * with placeholders to avoid blowing up the context window. + */ +function sanitizeDeepObject(obj: unknown, logger: IDextoLogger): unknown { + if (obj == null) return obj; + if (typeof obj === 'string') { + if (isLikelyBase64String(obj)) { + // Replace with short placeholder; do not keep raw data + const approxBytes = Math.floor((obj.length * 3) / 4); + logger.debug( + `sanitizeDeepObject: replaced large base64 string (~${approxBytes} bytes) with placeholder` + ); + return `[binary data omitted ~${approxBytes} bytes]`; + } + return obj; + } + if (Array.isArray(obj)) return obj.map((x) => sanitizeDeepObject(x, logger)); + if (typeof obj === 'object') { + const out: Record = {}; + for (const [k, v] of Object.entries(obj as Record)) { + out[k] = sanitizeDeepObject(v, logger); + } + return out; + } + return obj; +} + +export async function normalizeToolResult( + result: unknown, + logger: IDextoLogger +): Promise { + const content = await sanitizeToolResultToContentWithBlobs( + result, + logger, + undefined, + undefined + ); + + // Separate UI resources from other parts since they need special handling + const uiResources: UIResourcePart[] = []; + const otherContent: InternalMessage['content'] = []; + + if (Array.isArray(content)) { + for (const item of content) { + if (item && typeof item === 'object' && 'type' in item && item.type === 'ui-resource') { + uiResources.push(item as UIResourcePart); + } else { + otherContent.push(item); + } + } + } else { + // If content is not an array (string or other), pass through as-is + (otherContent as unknown[]).push(content); + } + + if (uiResources.length > 0) { + logger.debug( + `normalizeToolResult: extracted ${uiResources.length} UI resource(s): ${uiResources.map((r) => r.uri).join(', ')}` + ); + } + + const parts = coerceContentToParts(otherContent as InternalMessage['content']); + const inlineMedia: InlineMediaHint[] = []; + + parts.forEach((part, index) => { + const hint = detectInlineMedia(part, index); + if (hint) { + inlineMedia.push(hint); + } + }); + + return { + parts, + uiResources, + inlineMedia, + }; +} + +function shouldPersistInlineMedia(hint: InlineMediaHint): boolean { + const kind = getFileMediaKind(hint.mimeType); + if (kind === 'audio' || kind === 'video') { + return true; + } + return hint.approxBytes >= MIN_TOOL_INLINE_MEDIA_BYTES; +} + +export async function persistToolMedia( + normalized: NormalizedToolResult, + options: PersistToolMediaOptions, + logger: IDextoLogger +): Promise { + const parts = normalized.parts.map((part) => clonePart(part)); + const blobStore = options.blobStore; + const namingOptions: ToolBlobNamingOptions | undefined = + options.toolName || options.toolCallId + ? { + ...(options.toolName ? { toolName: options.toolName } : {}), + ...(options.toolCallId ? { toolCallId: options.toolCallId } : {}), + } + : undefined; + + // Track stored blobs for annotation + const storedBlobs: Array<{ uri: string; kind: string; mimeType: string; filename?: string }> = + []; + + if (blobStore) { + for (const hint of normalized.inlineMedia) { + if (!shouldPersistInlineMedia(hint)) { + continue; + } + + try { + const originalName = + hint.filename ?? + buildToolBlobName( + hint.kind === 'image' ? 'image' : 'file', + hint.mimeType, + namingOptions + ); + + const blobRef = await blobStore.store(hint.data, { + mimeType: hint.mimeType, + originalName, + source: 'tool', + }); + + const resourceUri = blobRef.uri; + + if (hint.kind === 'image') { + parts[hint.index] = createBlobImagePart(resourceUri, blobRef.metadata.mimeType); + } else { + const resolvedMimeType = blobRef.metadata.mimeType || hint.mimeType; + const filename = blobRef.metadata.originalName ?? hint.filename; + parts[hint.index] = createBlobFilePart(resourceUri, resolvedMimeType, filename); + } + + // Track for annotation + storedBlobs.push({ + uri: resourceUri, + kind: hint.kind, + mimeType: blobRef.metadata.mimeType, + ...(blobRef.metadata.originalName && { + filename: blobRef.metadata.originalName, + }), + }); + } catch (error) { + logger.warn( + `Failed to persist tool media: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + } + + // Add text annotations for stored blobs so the agent knows the references + // IMPORTANT: Use "resource_ref:" prefix (not "@blob:") to avoid expansion by expandBlobsInText() + // The @blob: pattern triggers base64 expansion which would duplicate the image data + if (storedBlobs.length > 0) { + const annotations = storedBlobs + .map((blob) => { + const label = blob.filename || blob.kind; + // Use resource_ref: prefix - agent should use this with get_shareable_url tool + // Format: resource_ref:blob:abc123 (can be used as "@blob:abc123" or "blob:abc123" in tool calls) + return `[Stored resource_ref:${blob.uri} (${label}, ${blob.mimeType})]`; + }) + .join('\n'); + + // Add annotation as a text part at the end + parts.push({ type: 'text', text: annotations }); + logger.debug(`Added blob reference annotations for ${storedBlobs.length} resource(s)`); + } + + const resources = extractResourceDescriptors(parts); + + return { + parts, + uiResources: normalized.uiResources, + ...(resources ? { resources } : {}), + }; +} + +/** + * Convert an arbitrary tool result into safe InternalMessage content with optional blob storage. + * - Automatically stores large media in blob store and returns resource references + * - Converts data URIs and base64 blobs to media/file parts or blob references + * - Removes huge binary blobs inside objects + * - Truncates extremely long raw text + */ +export async function sanitizeToolResultToContentWithBlobs( + result: unknown, + logger: IDextoLogger, + blobStore?: import('../storage/blob/types.js').BlobStore, + namingOptions?: ToolBlobNamingOptions +): Promise { + try { + // Case 1: string outputs + if (typeof result === 'string') { + // Data URI + const dataUri = parseDataUri(result); + if (dataUri) { + const mediaType = dataUri.mediaType; + logger.debug( + `sanitizeToolResultToContentWithBlobs: detected data URI (${mediaType})` + ); + + // Check if we should store as blob based on size + const approxSize = Math.floor((dataUri.base64.length * 3) / 4); + const shouldStoreAsBlob = blobStore && approxSize > 1024; // Store blobs > 1KB + + if (shouldStoreAsBlob) { + try { + logger.debug( + `Storing data URI as blob (${approxSize} bytes, ${mediaType})` + ); + const blobRef = await blobStore.store(result, { + mimeType: mediaType, + source: 'tool', + originalName: buildToolBlobName('output', mediaType, namingOptions), + }); + logger.debug(`Stored blob: ${blobRef.uri} (${approxSize} bytes)`); + + if (mediaType.startsWith('image/')) { + return [createBlobImagePart(blobRef.uri, mediaType)]; + } + return [createBlobFilePart(blobRef.uri, mediaType, undefined)]; + } catch (error) { + logger.warn( + `Failed to store blob, falling back to inline: ${String(error)}` + ); + // Fall through to original behavior + } + } + + // Original behavior: return as structured part + if (mediaType.startsWith('image/')) { + return [{ type: 'image', image: dataUri.base64, mimeType: mediaType }]; + } + return [ + { + type: 'file', + data: dataUri.base64, + mimeType: mediaType, + }, + ]; + } + + // Long text: truncate with ellipsis to keep context sane + if (result.length > MAX_TOOL_TEXT_CHARS) { + const head = result.slice(0, 4000); + const tail = result.slice(-1000); + logger.debug( + `sanitizeToolResultToContentWithBlobs: truncating long text tool output (len=${result.length})` + ); + return [ + { + type: 'text', + text: `${head}\n... [${result.length - 5000} chars omitted] ...\n${tail}`, + }, + ]; + } + return [{ type: 'text', text: result }]; + } + + // Case 2: array of parts or mixed + if (Array.isArray(result)) { + const parts: Array = []; + for (const item of result as unknown[]) { + if (item == null) continue; + + // Process each item recursively + const processedItem = await sanitizeToolResultToContentWithBlobs( + item, + logger, + blobStore, + namingOptions + ); + + if (Array.isArray(processedItem)) { + parts.push( + ...(processedItem as Array< + TextPart | ImagePart | FilePart | UIResourcePart + >) + ); + } + } + return parts as InternalMessage['content']; + } + + // Case 3: object — attempt to infer media, otherwise stringify safely + if (result && typeof result === 'object') { + const anyObj = result as Record; + + // Handle MCP tool results with nested content array + if ('content' in anyObj && Array.isArray(anyObj.content)) { + logger.debug( + `Processing MCP tool result with ${anyObj.content.length} content items` + ); + const processedContent = []; + + for (const item of anyObj.content) { + if (item && typeof item === 'object') { + // Handle MCP-UI resource type (ui:// URIs for interactive content) + if (item.type === 'resource' && item.resource) { + const resource = item.resource; + const resourceUri = resource.uri as string | undefined; + + // Check if this is a UI resource (uri starts with ui://) + if (resourceUri && resourceUri.startsWith('ui://')) { + logger.debug( + `Detected MCP-UI resource: ${resourceUri} (${resource.mimeType})` + ); + // Extract metadata - @mcp-ui/server puts metadata in _meta field + const resourceMeta = resource._meta || {}; + const title = resourceMeta.title || resource.title; + const preferredSize = + resourceMeta.preferredSize || resource.preferredSize; + + const uiPart: UIResourcePart = { + type: 'ui-resource', + uri: resourceUri, + mimeType: resource.mimeType || 'text/html', + content: resource.text, + blob: resource.blob, + metadata: { + title, + preferredSize, + }, + }; + // Clean up undefined metadata fields + if (!uiPart.metadata?.title && !uiPart.metadata?.preferredSize) { + delete uiPart.metadata; + } + processedContent.push(uiPart); + continue; + } + } + + // Handle MCP resource type (embedded resources) + if (item.type === 'resource' && item.resource) { + const resource = item.resource; + if (resource.text && resource.mimeType) { + const fileData = resource.text; + const mimeType = resource.mimeType; + + // Check if we should store as blob + const approxSize = + typeof fileData === 'string' + ? Math.floor((fileData.length * 3) / 4) + : 0; + const shouldStoreAsBlob = blobStore && approxSize > 1024; + + if (shouldStoreAsBlob) { + try { + logger.debug( + `Storing MCP resource as blob (${approxSize} bytes, ${mimeType})` + ); + const blobRef = await blobStore.store(fileData, { + mimeType, + source: 'tool', + originalName: buildToolBlobName( + mimeType.startsWith('image/') ? 'image' : 'file', + mimeType, + namingOptions, + resource.title + ), + }); + logger.debug( + `Stored MCP resource blob: ${blobRef.uri} (${approxSize} bytes)` + ); + if (mimeType.startsWith('image/')) { + processedContent.push( + createBlobImagePart(blobRef.uri, mimeType) + ); + } else { + processedContent.push( + createBlobFilePart( + blobRef.uri, + mimeType, + resource.title + ) + ); + } + continue; + } catch (error) { + logger.warn( + `Failed to store MCP resource blob, falling back to inline: ${String(error)}` + ); + } + } + + // Fall back to original structure based on MIME type + if (mimeType.startsWith('image/')) { + processedContent.push({ + type: 'image', + image: fileData, + mimeType, + }); + } else if (mimeType.startsWith('video/')) { + processedContent.push({ + type: 'file', + data: fileData, + mimeType, + filename: resource.title, + }); + } else { + processedContent.push({ + type: 'file', + data: fileData, + mimeType, + filename: resource.title, + }); + } + continue; + } + } + + // Handle legacy data field (for backwards compatibility) + if ('data' in item && item.mimeType) { + const fileData = getFileData({ data: item.data }, logger); + const mimeType = item.mimeType; + + // Check if we should store as blob + const approxSize = + typeof fileData === 'string' + ? Math.floor((fileData.length * 3) / 4) + : 0; + const shouldStoreAsBlob = blobStore && approxSize > 1024; + + if (shouldStoreAsBlob) { + try { + logger.debug( + `Storing MCP content item as blob (${approxSize} bytes, ${mimeType})` + ); + const blobRef = await blobStore.store(fileData, { + mimeType, + source: 'tool', + originalName: buildToolBlobName( + item.type === 'image' ? 'image' : 'file', + mimeType, + namingOptions, + item.filename + ), + }); + logger.debug( + `Stored MCP blob: ${blobRef.uri} (${approxSize} bytes)` + ); + if (item.type === 'image') { + processedContent.push( + createBlobImagePart(blobRef.uri, mimeType) + ); + } else { + processedContent.push( + createBlobFilePart(blobRef.uri, mimeType, item.filename) + ); + } + continue; + } catch (error) { + logger.warn( + `Failed to store MCP blob, falling back to inline: ${String(error)}` + ); + } + } + + // Fall back to original structure + if (item.type === 'image') { + processedContent.push({ + type: 'image', + image: fileData, + mimeType, + }); + } else { + processedContent.push({ + type: 'file', + data: fileData, + mimeType, + filename: item.filename, + }); + } + continue; + } + } + + // Non-media content, keep as-is + processedContent.push(item); + } + + return processedContent; + } + + // Common shapes: { image, mimeType? } or { data, mimeType } + if ('image' in anyObj) { + const imageData = getImageData({ image: anyObj.image }, logger); + const mimeType = anyObj.mimeType || 'image/jpeg'; + + // Check if we should store as blob + const approxSize = + typeof imageData === 'string' ? Math.floor((imageData.length * 3) / 4) : 0; + const shouldStoreAsBlob = blobStore && approxSize > 1024; + + if (shouldStoreAsBlob) { + try { + const blobRef = await blobStore.store(imageData, { + mimeType, + source: 'tool', + originalName: buildToolBlobName('image', mimeType, namingOptions), + }); + logger.debug( + `Stored tool image as blob: ${blobRef.uri} (${approxSize} bytes)` + ); + return [createBlobImagePart(blobRef.uri, mimeType)]; + } catch (error) { + logger.warn( + `Failed to store image blob, falling back to inline: ${String(error)}` + ); + } + } + + return [ + { + type: 'image', + image: imageData, + mimeType, + }, + ]; + } + + if ('data' in anyObj && anyObj.mimeType) { + const fileData = getFileData({ data: anyObj.data }, logger); + const mimeType = anyObj.mimeType; + + // Check if we should store as blob + const approxSize = + typeof fileData === 'string' ? Math.floor((fileData.length * 3) / 4) : 0; + const shouldStoreAsBlob = blobStore && approxSize > 1024; + + if (shouldStoreAsBlob) { + try { + const blobRef = await blobStore.store(fileData, { + mimeType, + source: 'tool', + originalName: buildToolBlobName( + 'file', + mimeType, + namingOptions, + anyObj.filename + ), + }); + logger.debug( + `Stored tool file as blob: ${blobRef.uri} (${approxSize} bytes)` + ); + return [createBlobFilePart(blobRef.uri, mimeType, anyObj.filename)]; + } catch (error) { + logger.warn( + `Failed to store file blob, falling back to inline: ${String(error)}` + ); + } + } + + return [ + { + type: 'file', + data: fileData, + mimeType, + filename: anyObj.filename, + }, + ]; + } + + // Generic object: remove huge base64 fields and stringify + const cleaned = sanitizeDeepObject(anyObj, logger); + return [{ type: 'text', text: safeStringify(cleaned) }]; + } + + // Fallback + return [{ type: 'text', text: safeStringify(result ?? '') }]; + } catch (err) { + logger.warn( + `sanitizeToolResultToContentWithBlobs failed, falling back to string: ${String(err)}` + ); + try { + return [{ type: 'text', text: safeStringify(result ?? '') }]; + } catch { + return [{ type: 'text', text: String(result ?? '') }]; + } + } +} + +// Deprecated: Use getResourceKind instead. Kept for internal backwards compatibility during migration. +function inferResourceKind(mimeType: string | undefined): 'image' | 'audio' | 'video' | 'binary' { + return getResourceKind(mimeType); +} + +function createBlobImagePart(uri: string, mimeType?: string): ImagePart { + return { + type: 'image', + image: `@${uri}`, + ...(mimeType ? { mimeType } : {}), + }; +} + +function createBlobFilePart(uri: string, mimeType: string, filename?: string): FilePart { + return { + type: 'file', + data: `@${uri}`, + mimeType, + ...(filename ? { filename } : {}), + }; +} + +function extractResourceDescriptors( + parts: Array +): SanitizedToolResult['resources'] { + const resources: NonNullable = []; + + for (const part of parts) { + if ( + part.type === 'image' && + typeof part.image === 'string' && + part.image.startsWith('@blob:') + ) { + resources.push({ + uri: part.image.substring(1), + kind: 'image', + mimeType: part.mimeType ?? 'image/jpeg', + }); + } + + if ( + part.type === 'file' && + typeof part.data === 'string' && + part.data.startsWith('@blob:') + ) { + resources.push({ + uri: part.data.substring(1), + kind: inferResourceKind(part.mimeType), + mimeType: part.mimeType, + ...(part.filename ? { filename: part.filename } : {}), + }); + } + } + + return resources.length > 0 ? resources : undefined; +} + +export async function sanitizeToolResult( + result: unknown, + options: { + blobStore?: import('../storage/blob/types.js').BlobStore; + toolName: string; + toolCallId: string; + success: boolean; + }, + logger: IDextoLogger +): Promise { + // Extract _display from tool result before normalization (if present) + // Strip it from the payload to avoid duplicating large display data in LLM content + let display: ToolDisplayData | undefined; + let resultForNormalization = result; + + if (result && typeof result === 'object' && '_display' in result) { + const { _display: rawDisplay, ...rest } = result as Record; + if (isValidDisplayData(rawDisplay)) { + display = rawDisplay; + logger.debug( + `sanitizeToolResult: extracted display data (type=${display.type}) for ${options.toolName}` + ); + } + // Always strip _display from payload sent to LLM, even if invalid + resultForNormalization = rest; + } + + const normalized = await normalizeToolResult(resultForNormalization, logger); + const persisted = await persistToolMedia( + normalized, + { + ...(options.blobStore ? { blobStore: options.blobStore } : {}), + toolName: options.toolName, + toolCallId: options.toolCallId, + }, + logger + ); + + const fallbackContent: TextPart[] = [{ type: 'text', text: '' }]; + // Combine regular parts with UI resources + const allContent: Array = [ + ...persisted.parts, + ...persisted.uiResources, + ]; + const content = allContent.length > 0 ? allContent : fallbackContent; + + if (persisted.uiResources.length > 0) { + logger.debug( + `sanitizeToolResult: including ${persisted.uiResources.length} UI resource(s) in final content for ${options.toolName}` + ); + } + + return { + content, + ...(persisted.resources ? { resources: persisted.resources } : {}), + meta: { + toolName: options.toolName, + toolCallId: options.toolCallId, + success: options.success, + ...(display ? { display } : {}), + }, + }; +} + +/** + * Produce a short textual summary for tool content, to be used with providers + * that only accept text for tool messages (e.g., OpenAI/Anthropic tool role). + */ +export function summarizeToolContentForText(content: InternalMessage['content']): string { + if (!Array.isArray(content)) return String(content || ''); + const parts: string[] = []; + for (const p of content) { + if (p.type === 'text') { + parts.push(p.text); + } else if (p.type === 'image') { + // Try estimating size + let bytes = 0; + if (typeof p.image === 'string') bytes = Math.floor((p.image.length * 3) / 4); + else if (p.image instanceof ArrayBuffer) bytes = p.image.byteLength; + else if (p.image instanceof Uint8Array) bytes = p.image.length; + else if (p.image instanceof Buffer) bytes = p.image.length; + parts.push(`[image ${p.mimeType || 'image'} ~${Math.ceil(bytes / 1024)}KB]`); + } else if (p.type === 'file') { + let bytes = 0; + if (typeof p.data === 'string') bytes = Math.floor((p.data.length * 3) / 4); + else if (p.data instanceof ArrayBuffer) bytes = p.data.byteLength; + else if (p.data instanceof Uint8Array) bytes = p.data.length; + else if (p.data instanceof Buffer) bytes = p.data.length; + const label = p.filename ? `${p.filename}` : `${p.mimeType || 'file'}`; + parts.push(`[file ${label} ~${Math.ceil(bytes / 1024)}KB]`); + } + } + const summary = parts.join('\n'); + // Avoid passing enormous text anyway + return summary.slice(0, 4000); +} + +// Helper: estimate base64 byte length from string length +function base64LengthToBytes(charLength: number): number { + // 4 base64 chars -> 3 bytes; ignore padding for approximation + return Math.floor((charLength * 3) / 4); +} + +/** + * Convert arbitrary tool content to safe text for providers that only accept textual tool messages. + * - If content is an array of parts, summarize it. + * - If content is a string that looks like base64/data URI, replace with a short placeholder. + * - Otherwise pass text through. + */ +export function toTextForToolMessage(content: InternalMessage['content']): string { + if (Array.isArray(content)) { + return summarizeToolContentForText(content); + } + if (typeof content === 'string') { + return isLikelyBase64String(content) ? '[binary data omitted]' : content; + } + return String(content ?? ''); +} + +/** + * Filter history to exclude messages before the most recent summary. + * This implements read-time compression for inline compaction. + * + * Used by: + * - TurnExecutor for inline compaction during agentic turns (overflow handling) + * - DextoAgent.getContextStats() for accurate token/message counts + * + * When a summary message exists (with metadata.isSummary === true or + * metadata.isSessionSummary === true), this function returns only the + * summary message and everything after it. This effectively hides old + * messages from the LLM while preserving them in storage. + * + * @param history The full conversation history + * @returns Filtered history starting from the most recent summary (or full history if no summary) + */ +export function filterCompacted(history: readonly InternalMessage[]): InternalMessage[] { + // Find the most recent summary message (search backwards for efficiency) + // Check for both old isSummary marker and new isSessionSummary marker + let summaryIndex = -1; + for (let i = history.length - 1; i >= 0; i--) { + const msg = history[i]; + if (msg?.metadata?.isSummary === true || msg?.metadata?.isSessionSummary === true) { + summaryIndex = i; + break; + } + } + + // If no summary found, return full history (slice returns mutable copy) + if (summaryIndex === -1) { + return history.slice(); + } + + // Get the summary message (we know it exists since we found the index) + const summaryMessage = history[summaryIndex]!; + + // Get the count of messages that were summarized (stored in metadata) + // The preserved messages are between the summarized portion and the summary + // Clamp to valid range: 0 <= originalMessageCount <= summaryIndex + // For legacy summaries without metadata, default to summaryIndex (no preserved messages) + const rawCount = summaryMessage.metadata?.originalMessageCount; + const originalMessageCount = + typeof rawCount === 'number' && rawCount >= 0 && rawCount <= summaryIndex + ? rawCount + : summaryIndex; + + // Layout after compaction: + // [summarized..., preserved..., summary, afterSummary...] + // ^-- indices 0 to (originalMessageCount-1) + // ^-- indices originalMessageCount to (summaryIndex-1) + // ^-- index summaryIndex + // ^-- indices (summaryIndex+1) onwards + + // Get preserved messages (messages between summarized portion and summary) + const preservedMessages = history.slice(originalMessageCount, summaryIndex); + + // Get any messages added after the summary (rare but possible) + const messagesAfterSummary = history.slice(summaryIndex + 1); + + // Return: summary + preserved + afterSummary + return [summaryMessage, ...preservedMessages, ...messagesAfterSummary]; +} + +/** + * Format tool output for display, respecting compactedAt marker. + * If a tool message has been compacted (pruned), return a placeholder. + * + * @param message The tool message to format + * @returns The content string or placeholder if compacted + */ +export function formatToolOutputForDisplay(message: InternalMessage): string { + if (isToolMessage(message) && message.compactedAt) { + return '[Old tool result content cleared]'; + } + + if (typeof message.content === 'string') { + return message.content; + } + + if (Array.isArray(message.content)) { + // Extract text parts + return message.content + .filter((part): part is TextPart => part.type === 'text') + .map((part) => part.text) + .join('\n'); + } + + return '[no content]'; +} diff --git a/dexto/packages/core/src/errors/DextoBaseError.ts b/dexto/packages/core/src/errors/DextoBaseError.ts new file mode 100644 index 00000000..3164785a --- /dev/null +++ b/dexto/packages/core/src/errors/DextoBaseError.ts @@ -0,0 +1,22 @@ +import { randomUUID } from 'crypto'; + +/** + * Abstract base class for all Dexto errors + * Provides common functionality like trace ID generation and JSON serialization + */ +export abstract class DextoBaseError extends Error { + public readonly traceId: string; + + constructor(message: string, traceId?: string) { + super(message); + this.traceId = traceId || randomUUID(); + // Ensure the name is set to the actual class name + this.name = this.constructor.name; + } + + /** + * Convert error to JSON representation + * Must be implemented by subclasses + */ + abstract toJSON(): Record; +} diff --git a/dexto/packages/core/src/errors/DextoRuntimeError.ts b/dexto/packages/core/src/errors/DextoRuntimeError.ts new file mode 100644 index 00000000..d3b38d9b --- /dev/null +++ b/dexto/packages/core/src/errors/DextoRuntimeError.ts @@ -0,0 +1,35 @@ +import { DextoBaseError } from './DextoBaseError.js'; +import { ErrorScope } from './types.js'; +import { ErrorType } from './types.js'; +import type { DextoErrorCode } from './types.js'; + +/** + * Runtime error class for single-issue errors + * Provides structured error information with scope, type, and recovery guidance + */ +export class DextoRuntimeError extends DextoBaseError { + constructor( + public readonly code: DextoErrorCode | string, + public readonly scope: ErrorScope | string, + public readonly type: ErrorType, + message: string, + public readonly context?: C, + public readonly recovery?: string | string[], + traceId?: string + ) { + super(message, traceId); + this.name = 'DextoRuntimeError'; + } + + toJSON() { + return { + code: this.code, + message: this.message, + scope: this.scope, + type: this.type, + context: this.context, + recovery: this.recovery, + traceId: this.traceId, + }; + } +} diff --git a/dexto/packages/core/src/errors/DextoValidationError.ts b/dexto/packages/core/src/errors/DextoValidationError.ts new file mode 100644 index 00000000..37236517 --- /dev/null +++ b/dexto/packages/core/src/errors/DextoValidationError.ts @@ -0,0 +1,111 @@ +import { DextoBaseError } from './DextoBaseError.js'; +import type { Issue } from './types.js'; + +/** + * Validation error class for handling multiple validation issues + * Similar to ZodError, provides first-class access to all validation issues + */ +export class DextoValidationError extends DextoBaseError { + public readonly issues: Issue[]; + + constructor(issues: Issue[]) { + const message = DextoValidationError.formatMessage(issues); + super(message); + this.name = 'DextoValidationError'; + this.issues = issues; + } + + /** + * Format multiple issues into a readable error message + */ + private static formatMessage(issues: Issue[]): string { + if (issues.length === 0) { + return 'Validation failed'; + } + + if (issues.length === 1) { + return issues[0]!.message; // We know it exists after length check + } + + const errors = issues.filter((i) => i.severity === 'error'); + const warnings = issues.filter((i) => i.severity === 'warning'); + + const parts: string[] = []; + if (errors.length > 0) { + parts.push(`${errors.length} error${errors.length > 1 ? 's' : ''}`); + } + if (warnings.length > 0) { + parts.push(`${warnings.length} warning${warnings.length > 1 ? 's' : ''}`); + } + + return `Validation failed with ${parts.join(' and ')}`; + } + + /** + * Get only error-severity issues + */ + get errors(): Issue[] { + return this.issues.filter((i) => i.severity === 'error'); + } + + /** + * Get only warning-severity issues + */ + get warnings(): Issue[] { + return this.issues.filter((i) => i.severity === 'warning'); + } + + /** + * Check if there are any error-severity issues + */ + hasErrors(): boolean { + return this.errors.length > 0; + } + + /** + * Check if there are any warning-severity issues + */ + hasWarnings(): boolean { + return this.warnings.length > 0; + } + + /** + * Get the first error-severity issue (if any) + * Useful for getting the primary error when multiple exist + */ + get firstError(): Issue | undefined { + return this.errors[0]; + } + + /** + * Get the first warning-severity issue (if any) + */ + get firstWarning(): Issue | undefined { + return this.warnings[0]; + } + + /** + * Format issues for display + * Returns an object with categorized issues for easy logging + */ + format(): { errors: string[]; warnings: string[] } { + return { + errors: this.errors.map((e) => `[${e.code}] ${e.message}`), + warnings: this.warnings.map((w) => `[${w.code}] ${w.message}`), + }; + } + + /** + * Convert to JSON representation + */ + toJSON(): Record { + return { + name: this.name, + message: this.message, + issues: this.issues, + traceId: this.traceId, + errorCount: this.errors.length, + warningCount: this.warnings.length, + }; + } +} diff --git a/dexto/packages/core/src/errors/index.ts b/dexto/packages/core/src/errors/index.ts new file mode 100644 index 00000000..ff5ba062 --- /dev/null +++ b/dexto/packages/core/src/errors/index.ts @@ -0,0 +1,11 @@ +/** + * Main entry point for the error management system + * Exports core types and utilities for error handling + */ + +export { DextoBaseError } from './DextoBaseError.js'; +export { DextoRuntimeError } from './DextoRuntimeError.js'; +export { DextoValidationError } from './DextoValidationError.js'; +export { ErrorScope, ErrorType } from './types.js'; +export type { Issue, Severity, DextoErrorCode } from './types.js'; +export { ensureOk } from './result-bridge.js'; diff --git a/dexto/packages/core/src/errors/result-bridge.ts b/dexto/packages/core/src/errors/result-bridge.ts new file mode 100644 index 00000000..fdd0fbed --- /dev/null +++ b/dexto/packages/core/src/errors/result-bridge.ts @@ -0,0 +1,36 @@ +import type { Result } from '../utils/result.js'; +import { DextoValidationError } from './DextoValidationError.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; + +/** + * Bridge function to convert Result pattern to validation exceptions + * Used at public API boundaries for validation flows + * + * Note: Runtime errors are thrown directly, not through Result pattern + * + * @param result - The Result to check (typically from validation functions) + * @param logger - Logger instance for logging errors + * @returns The data if successful + * @throws DextoValidationError if the result contains validation issues + * + * @example + * ```typescript + * // Validation flow + * const result = validateInputForLLM(input, config); + * const data = ensureOk(result, logger); // Throws DextoValidationError if validation failed + * + * // LLM config validation + * const configResult = resolveAndValidateLLMConfig(current, updates); + * const validatedConfig = ensureOk(configResult, logger); + * ``` + */ +export function ensureOk(result: Result, logger: IDextoLogger): T { + if (result.ok) { + return result.data; + } + + const issueMessages = result.issues.map((i) => i.message).join('; '); + logger.error(`ensureOk: validation failed - ${issueMessages}`); + // Result pattern is used for validation - throw validation error + throw new DextoValidationError(result.issues); +} diff --git a/dexto/packages/core/src/errors/types.ts b/dexto/packages/core/src/errors/types.ts new file mode 100644 index 00000000..18964cf6 --- /dev/null +++ b/dexto/packages/core/src/errors/types.ts @@ -0,0 +1,90 @@ +import type { AgentErrorCode } from '@core/agent/error-codes.js'; +// ConfigErrorCode has been moved to @dexto/agent-management +// Import from there if needed for error type unions +import type { ContextErrorCode } from '@core/context/error-codes.js'; +import type { LLMErrorCode } from '@core/llm/error-codes.js'; +import type { MCPErrorCode } from '@core/mcp/error-codes.js'; +import type { SessionErrorCode } from '@core/session/error-codes.js'; +import type { StorageErrorCode } from '@core/storage/error-codes.js'; +import type { SystemPromptErrorCode } from '@core/systemPrompt/error-codes.js'; +import type { ToolErrorCode } from '@core/tools/error-codes.js'; +import type { ResourceErrorCode } from '@core/resources/error-codes.js'; +import type { PromptErrorCode } from '@core/prompts/error-codes.js'; +import type { ApprovalErrorCode } from '@core/approval/error-codes.js'; +import type { MemoryErrorCode } from '@core/memory/error-codes.js'; +import type { PluginErrorCode } from '@core/plugins/error-codes.js'; +import type { TelemetryErrorCode } from '@core/telemetry/error-codes.js'; + +/** + * Error scopes representing functional domains in the system + * Each scope owns its validation and error logic + */ +export enum ErrorScope { + LLM = 'llm', // LLM operations, model compatibility, input validation for LLMs + AGENT = 'agent', // Agent lifecycle, configuration + CONFIG = 'config', // Configuration file operations, parsing, validation + CONTEXT = 'context', // Context management, message validation, token processing + SESSION = 'session', // Session lifecycle, management, and state + MCP = 'mcp', // MCP server connections and protocol + TOOLS = 'tools', // Tool execution and authorization + STORAGE = 'storage', // Storage backend operations + LOGGER = 'logger', // Logging system operations, transports, and configuration + SYSTEM_PROMPT = 'system_prompt', // System prompt contributors and file processing + RESOURCE = 'resource', // Resource management (MCP/internal) discovery and access + PROMPT = 'prompt', // Prompt management, resolution, and providers + MEMORY = 'memory', // Memory management and storage + PLUGIN = 'plugin', // Plugin loading, validation, and execution + TELEMETRY = 'telemetry', // Telemetry initialization and export operations +} + +/** + * Error types that map directly to HTTP status codes + * Each type represents the nature of the error + */ +export enum ErrorType { + USER = 'user', // 400 - bad input, config errors, validation failures + PAYMENT_REQUIRED = 'payment_required', // 402 - insufficient credits, billing issue + FORBIDDEN = 'forbidden', // 403 - permission denied, unauthorized + NOT_FOUND = 'not_found', // 404 - resource doesn't exist (session, file, etc.) + TIMEOUT = 'timeout', // 408 - operation timed out + CONFLICT = 'conflict', // 409 - resource conflict, concurrent operation + RATE_LIMIT = 'rate_limit', // 429 - too many requests + SYSTEM = 'system', // 500 - bugs, internal failures, unexpected states + THIRD_PARTY = 'third_party', // 502 - upstream provider failures, API errors + UNKNOWN = 'unknown', // 500 - unclassified errors, fallback +} + +/** + * Union type for all error codes across domains + * Provides type safety for error handling + * Note: ConfigErrorCode has been moved to @dexto/agent-management + */ +export type DextoErrorCode = + | LLMErrorCode + | AgentErrorCode + | ContextErrorCode + | SessionErrorCode + | MCPErrorCode + | ToolErrorCode + | StorageErrorCode + | SystemPromptErrorCode + | ResourceErrorCode + | PromptErrorCode + | ApprovalErrorCode + | MemoryErrorCode + | PluginErrorCode + | TelemetryErrorCode; + +/** Severity of an issue */ +export type Severity = 'error' | 'warning'; + +/** Generic issue type for validation results */ +export interface Issue { + code: DextoErrorCode | string; + message: string; + scope: ErrorScope | string; // Domain that generated this issue + type: ErrorType; // HTTP status mapping + severity: Severity; + path?: Array; + context?: C; +} diff --git a/dexto/packages/core/src/events/index.test.ts b/dexto/packages/core/src/events/index.test.ts new file mode 100644 index 00000000..bbe7924a --- /dev/null +++ b/dexto/packages/core/src/events/index.test.ts @@ -0,0 +1,113 @@ +import { describe, it, expect, vi } from 'vitest'; +import { AgentEventBus } from './index.js'; + +describe('EventBus AbortController Support', () => { + it('should remove event listener when signal is aborted', () => { + const eventBus = new AgentEventBus(); + const abortController = new AbortController(); + const listener = vi.fn(); + + // Add listener with abort signal + eventBus.on('session:reset', listener, { signal: abortController.signal }); + + // Emit event - should be received + eventBus.emit('session:reset', { sessionId: 'test' }); + expect(listener).toHaveBeenCalledTimes(1); + + // Abort the signal + abortController.abort(); + + // Emit event again - should not be received + eventBus.emit('session:reset', { sessionId: 'test' }); + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('should not add listener if signal is already aborted', () => { + const eventBus = new AgentEventBus(); + const abortController = new AbortController(); + const listener = vi.fn(); + + // Abort signal first + abortController.abort(); + + // Try to add listener with aborted signal + eventBus.on('session:reset', listener, { signal: abortController.signal }); + + // Emit event - should not be received + eventBus.emit('session:reset', { sessionId: 'test' }); + expect(listener).not.toHaveBeenCalled(); + }); + + it('should work with once() and abort signal', () => { + const eventBus = new AgentEventBus(); + const abortController = new AbortController(); + const listener = vi.fn(); + + // Add once listener with abort signal + eventBus.once('session:reset', listener, { signal: abortController.signal }); + + // Abort the signal before emitting + abortController.abort(); + + // Emit event - should not be received + eventBus.emit('session:reset', { sessionId: 'test' }); + expect(listener).not.toHaveBeenCalled(); + }); + + it('should work without signal (backward compatibility)', () => { + const eventBus = new AgentEventBus(); + const listener = vi.fn(); + + // Add listener without signal (old way) + eventBus.on('session:reset', listener); + + // Emit event - should be received + eventBus.emit('session:reset', { sessionId: 'test' }); + expect(listener).toHaveBeenCalledTimes(1); + + // Remove manually + eventBus.off('session:reset', listener); + + // Emit event again - should not be received + eventBus.emit('session:reset', { sessionId: 'test' }); + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('should handle multiple listeners with different signals', () => { + const eventBus = new AgentEventBus(); + const controller1 = new AbortController(); + const controller2 = new AbortController(); + const listener1 = vi.fn(); + const listener2 = vi.fn(); + const listener3 = vi.fn(); + + // Add listeners with different signals + eventBus.on('session:reset', listener1, { signal: controller1.signal }); + eventBus.on('session:reset', listener2, { signal: controller2.signal }); + eventBus.on('session:reset', listener3); // No signal + + // Emit event - all should receive + eventBus.emit('session:reset', { sessionId: 'test' }); + expect(listener1).toHaveBeenCalledTimes(1); + expect(listener2).toHaveBeenCalledTimes(1); + expect(listener3).toHaveBeenCalledTimes(1); + + // Abort first signal + controller1.abort(); + + // Emit event - only listener2 and listener3 should receive + eventBus.emit('session:reset', { sessionId: 'test' }); + expect(listener1).toHaveBeenCalledTimes(1); // Still 1 + expect(listener2).toHaveBeenCalledTimes(2); + expect(listener3).toHaveBeenCalledTimes(2); + + // Abort second signal + controller2.abort(); + + // Emit event - only listener3 should receive + eventBus.emit('session:reset', { sessionId: 'test' }); + expect(listener1).toHaveBeenCalledTimes(1); // Still 1 + expect(listener2).toHaveBeenCalledTimes(2); // Still 2 + expect(listener3).toHaveBeenCalledTimes(3); + }); +}); diff --git a/dexto/packages/core/src/events/index.ts b/dexto/packages/core/src/events/index.ts new file mode 100644 index 00000000..4302dce0 --- /dev/null +++ b/dexto/packages/core/src/events/index.ts @@ -0,0 +1,902 @@ +import { EventEmitter } from 'events'; +import type { LLMProvider } from '../llm/types.js'; +import { ValidatedAgentConfig } from '../agent/schemas.js'; +import type { ApprovalRequest, ApprovalResponse } from '../approval/types.js'; +import type { SanitizedToolResult } from '../context/types.js'; + +/** + * LLM finish reason - why the LLM stopped generating + * + * Superset of Vercel AI SDK's LanguageModelV3FinishReason with app-specific additions. + */ +export type LLMFinishReason = + // From Vercel AI SDK (LanguageModelV3FinishReason) + | 'stop' // Normal completion + | 'tool-calls' // Stopped to execute tool calls (more steps coming) + | 'length' // Hit token/length limit + | 'content-filter' // Content filter violation stopped the model + | 'error' // Error occurred + | 'other' // Other reason + | 'unknown' // Model has not transmitted a finish reason + // App-specific additions + | 'cancelled' // User cancelled + | 'max-steps'; // Hit max steps limit + +/** + * Agent-level event names - events that occur at the agent/global level + */ +export const AGENT_EVENT_NAMES = [ + 'session:reset', + 'session:created', + 'session:title-updated', + 'session:override-set', + 'session:override-cleared', + 'mcp:server-connected', + 'mcp:server-added', + 'mcp:server-removed', + 'mcp:server-restarted', + 'mcp:server-updated', + 'mcp:resource-updated', + 'mcp:prompts-list-changed', + 'mcp:tools-list-changed', + 'tools:available-updated', + 'llm:switched', + 'state:changed', + 'state:exported', + 'state:reset', + 'resource:cache-invalidated', + 'approval:request', + 'approval:response', + 'run:invoke', +] as const; + +/** + * Session-level event names - events that occur within individual sessions + */ +export const SESSION_EVENT_NAMES = [ + 'llm:thinking', + 'llm:chunk', + 'llm:response', + 'llm:tool-call', + 'llm:tool-result', + 'llm:error', + 'llm:switched', + 'llm:unsupported-input', + 'tool:running', + 'context:compacting', + 'context:compacted', + 'context:pruned', + 'message:queued', + 'message:dequeued', + 'message:removed', + 'run:complete', +] as const; + +/** + * All event names combined for backward compatibility + */ +export const EVENT_NAMES = [...AGENT_EVENT_NAMES, ...SESSION_EVENT_NAMES] as const; + +/** + * Event Visibility Tiers + * + * These define which events are exposed through different APIs: + * - STREAMING_EVENTS: Exposed via DextoAgent.stream() for real-time chat UIs + * - INTEGRATION_EVENTS: Exposed via webhooks, A2A, and monitoring systems + * - Internal events: Only available via direct EventBus access + */ + +/** + * Tier 1: Streaming Events + * + * Events exposed via DextoAgent.stream() for real-time streaming. + * These are the most commonly used events for building chat UIs and + * represent the core user-facing event stream. + */ +export const STREAMING_EVENTS = [ + // LLM events (session-scoped, forwarded to agent bus with sessionId) + 'llm:thinking', + 'llm:chunk', + 'llm:response', + 'llm:tool-call', + 'llm:tool-result', + 'llm:error', + 'llm:unsupported-input', + + // Tool execution events + 'tool:running', + + // Context management events + 'context:compacting', + 'context:compacted', + 'context:pruned', + + // Message queue events (for mid-task user guidance) + 'message:queued', + 'message:dequeued', + + // Run lifecycle events + 'run:complete', + + // Session metadata + 'session:title-updated', + + // Approval events (needed for tool confirmation in streaming UIs) + 'approval:request', + 'approval:response', + + // Service events (extensible pattern for non-core services) + 'service:event', +] as const; + +/** + * Tier 2: Integration Events + * + * Events exposed via webhooks, A2A subscriptions, and monitoring systems. + * Includes all streaming events plus lifecycle and state management events + * useful for external integrations. + */ +export const INTEGRATION_EVENTS = [ + ...STREAMING_EVENTS, + + // Session lifecycle + 'session:created', + 'session:reset', + + // MCP lifecycle + 'mcp:server-connected', + 'mcp:server-restarted', + 'mcp:tools-list-changed', + 'mcp:prompts-list-changed', + + // Tools + 'tools:available-updated', + + // LLM provider switching + 'llm:switched', + + // State management + 'state:changed', +] as const; + +/** + * Tier 3: Internal Events + * + * Events only exposed via direct AgentEventBus access for advanced use cases. + * These are implementation details that may change between versions. + * + * Internal events include: + * - resource:cache-invalidated + * - state:exported + * - state:reset + * - mcp:server-added + * - mcp:server-removed + * - mcp:server-updated + * - mcp:resource-updated + * - session:override-set + * - session:override-cleared + */ + +export type StreamingEventName = (typeof STREAMING_EVENTS)[number]; +export type IntegrationEventName = (typeof INTEGRATION_EVENTS)[number]; +export type InternalEventName = Exclude; + +/** + * Type helper to extract events by name from AgentEventMap + */ +export type AgentEventByName = { + name: T; +} & AgentEventMap[T]; + +/** + * Union type of all streaming events with their payloads + * Automatically derived from STREAMING_EVENTS const to stay in sync. + * Uses 'name' property (not 'type') to avoid collision with payload fields like ApprovalRequest.type + * These are the events that the message-stream API actually returns. + */ +export type StreamingEvent = { + [K in StreamingEventName]: { name: K } & AgentEventMap[K]; +}[StreamingEventName]; + +/** + * Union type of all integration events with their payloads + */ +export type IntegrationEvent = + | StreamingEvent + | ({ name: 'session:created' } & AgentEventMap['session:created']) + | ({ name: 'session:reset' } & AgentEventMap['session:reset']) + | ({ name: 'mcp:server-connected' } & AgentEventMap['mcp:server-connected']) + | ({ name: 'mcp:server-restarted' } & AgentEventMap['mcp:server-restarted']) + | ({ name: 'mcp:tools-list-changed' } & AgentEventMap['mcp:tools-list-changed']) + | ({ name: 'mcp:prompts-list-changed' } & AgentEventMap['mcp:prompts-list-changed']) + | ({ name: 'tools:available-updated' } & AgentEventMap['tools:available-updated']) + | ({ name: 'llm:switched' } & AgentEventMap['llm:switched']) + | ({ name: 'state:changed' } & AgentEventMap['state:changed']); + +/** + * Combined event map for the agent bus - includes agent events and session events with sessionId + * This is what the global agent event bus uses to aggregate all events + */ +export interface AgentEventMap { + // Session events + /** Fired when session conversation is reset */ + 'session:reset': { + sessionId: string; + }; + + /** Fired when a new session is created and should become active */ + 'session:created': { + sessionId: string | null; // null means clear without creating (deferred creation) + switchTo: boolean; // Whether UI should switch to this session + }; + + /** Fired when a session's human-friendly title is updated */ + 'session:title-updated': { + sessionId: string; + title: string; + }; + + /** Fired when session override is set */ + 'session:override-set': { + sessionId: string; + override: any; // SessionOverride type + }; + + /** Fired when session override is cleared */ + 'session:override-cleared': { + sessionId: string; + }; + + // MCP events + /** Fired when MCP server connection succeeds or fails */ + 'mcp:server-connected': { + name: string; + success: boolean; + error?: string; + }; + + /** Fired when MCP server is added to runtime state */ + 'mcp:server-added': { + serverName: string; + config: any; // McpServerConfig type + }; + + /** Fired when MCP server is removed from runtime state */ + 'mcp:server-removed': { + serverName: string; + }; + + /** Fired when MCP server is restarted */ + 'mcp:server-restarted': { + serverName: string; + }; + + /** Fired when MCP server is updated in runtime state */ + 'mcp:server-updated': { + serverName: string; + config: any; // McpServerConfig type + }; + + /** Fired when MCP server resource is updated */ + 'mcp:resource-updated': { + serverName: string; + resourceUri: string; + }; + + /** Fired when MCP server prompts list changes */ + 'mcp:prompts-list-changed': { + serverName: string; + prompts: string[]; + }; + + /** Fired when MCP server tools list changes */ + 'mcp:tools-list-changed': { + serverName: string; + tools: string[]; + }; + + // Tools events + /** Fired when available tools list updates */ + 'tools:available-updated': { + tools: string[]; + source: 'mcp' | 'builtin'; + }; + + /** + * Agent run is being invoked externally (e.g., by scheduler, A2A, API). + * Fired BEFORE agent.stream()/run() is called. + * UI can use this to display the incoming prompt and set up streaming subscriptions. + */ + 'run:invoke': { + /** The session this run will execute in */ + sessionId: string; + /** The prompt/content being sent */ + content: import('../context/types.js').ContentPart[]; + /** Source of the invocation */ + source: 'scheduler' | 'a2a' | 'api' | 'external'; + /** Optional metadata about the invocation */ + metadata?: Record; + }; + + // LLM events (forwarded from session bus with sessionId added) + /** LLM service started thinking */ + 'llm:thinking': { + sessionId: string; + }; + + /** LLM service sent a streaming chunk */ + 'llm:chunk': { + chunkType: 'text' | 'reasoning'; + content: string; + isComplete?: boolean; + sessionId: string; + }; + + /** LLM service final response */ + 'llm:response': { + content: string; + reasoning?: string; + provider?: LLMProvider; + model?: string; + tokenUsage?: { + inputTokens?: number; + outputTokens?: number; + reasoningTokens?: number; + totalTokens?: number; + cacheReadTokens?: number; + cacheWriteTokens?: number; + }; + /** Estimated input tokens before LLM call (for analytics/calibration) */ + estimatedInputTokens?: number; + /** Finish reason: 'tool-calls' means more steps coming, others indicate completion */ + finishReason?: LLMFinishReason; + sessionId: string; + }; + + /** LLM service requested a tool call */ + 'llm:tool-call': { + toolName: string; + args: Record; + callId?: string; + sessionId: string; + }; + + /** LLM service returned a tool result */ + 'llm:tool-result': { + toolName: string; + callId?: string; + success: boolean; + /** Sanitized result - present when success=true */ + sanitized?: SanitizedToolResult; + rawResult?: unknown; + /** Error message - present when success=false */ + error?: string; + /** Whether this tool required user approval */ + requireApproval?: boolean; + /** The approval status (only present if requireApproval is true) */ + approvalStatus?: 'approved' | 'rejected'; + sessionId: string; + }; + + /** Tool execution actually started (after approval if needed) */ + 'tool:running': { + toolName: string; + toolCallId: string; + sessionId: string; + }; + + /** LLM service error */ + 'llm:error': { + error: Error; + context?: string; + recoverable?: boolean; + /** Tool call ID if error occurred during tool execution */ + toolCallId?: string; + sessionId: string; + }; + + /** LLM service switched */ + 'llm:switched': { + newConfig: any; // LLMConfig type + historyRetained?: boolean; + sessionIds: string[]; // Array of affected session IDs + }; + + /** LLM service unsupported input */ + 'llm:unsupported-input': { + errors: string[]; + provider: LLMProvider; + model?: string; + fileType?: string; + details?: any; + sessionId: string; + }; + + /** Context compaction is starting */ + 'context:compacting': { + /** Estimated tokens that triggered compaction */ + estimatedTokens: number; + sessionId: string; + }; + + /** Context was compacted during multi-step tool calling */ + 'context:compacted': { + /** Actual input tokens from API that triggered compaction */ + originalTokens: number; + /** Estimated tokens after compaction (simple length/4 heuristic) */ + compactedTokens: number; + originalMessages: number; + compactedMessages: number; + strategy: string; + reason: 'overflow' | 'manual'; + sessionId: string; + }; + + /** Old tool outputs were pruned (marked with compactedAt) to save tokens */ + 'context:pruned': { + prunedCount: number; + savedTokens: number; + sessionId: string; + }; + + /** Context was manually cleared via /clear command */ + 'context:cleared': { + sessionId: string; + }; + + /** User message was queued during agent execution */ + 'message:queued': { + position: number; + id: string; + sessionId: string; + }; + + /** Queued messages were dequeued and injected into context */ + 'message:dequeued': { + count: number; + ids: string[]; + coalesced: boolean; + /** Combined content of all dequeued messages (for UI display) */ + content: import('../context/types.js').ContentPart[]; + sessionId: string; + }; + + /** Queued message was removed from queue */ + 'message:removed': { + id: string; + sessionId: string; + }; + + /** Agent run completed (all steps done, no queued messages remaining) */ + 'run:complete': { + /** How the run ended */ + finishReason: LLMFinishReason; + /** Number of steps executed */ + stepCount: number; + /** Total wall-clock duration of the run in milliseconds */ + durationMs: number; + /** Error that caused termination (only if finishReason === 'error') */ + error?: Error; + sessionId: string; + }; + + // State events + /** Fired when agent runtime state changes */ + 'state:changed': { + field: string; // keyof AgentRuntimeState + oldValue: any; + newValue: any; + sessionId?: string; + }; + + /** Fired when agent state is exported as config */ + 'state:exported': { + config: ValidatedAgentConfig; + }; + + /** Fired when agent state is reset to baseline */ + 'state:reset': { + toConfig: any; // AgentConfig type + }; + + // Resource events + /** Fired when resource cache should be invalidated */ + 'resource:cache-invalidated': { + resourceUri?: string; + serverName: string; + action: 'updated' | 'server_connected' | 'server_removed' | 'blob_stored'; + }; + + // Approval events - use ApprovalRequest directly + // No transformation needed since we use 'name' (not 'type') as SSE discriminant + /** Fired when user approval is requested (generalized approval system) */ + 'approval:request': ApprovalRequest; + + /** Fired when user approval response is received */ + 'approval:response': ApprovalResponse; + + /** + * Extensible service event for non-core/additive services. + * Allows services like agent-spawner, process-tools, etc. to emit events + * without polluting the core event namespace. + */ + 'service:event': { + /** Service identifier (e.g., 'agent-spawner', 'process-tools') */ + service: string; + /** Event type within the service (e.g., 'progress', 'stdout') */ + event: string; + /** Links this event to a parent tool call */ + toolCallId?: string; + /** Session this event belongs to */ + sessionId: string; + /** Arbitrary event data - service-specific payload */ + data: Record; + }; +} + +/** + * Session-level events - these occur within individual sessions without session context + * (since they're already scoped to a session) + */ +export interface SessionEventMap { + /** LLM service started thinking */ + 'llm:thinking': void; + + /** LLM service sent a streaming chunk */ + 'llm:chunk': { + chunkType: 'text' | 'reasoning'; + content: string; + isComplete?: boolean; + }; + + /** LLM service final response */ + 'llm:response': { + content: string; + reasoning?: string; + provider?: LLMProvider; + model?: string; + tokenUsage?: { + inputTokens?: number; + outputTokens?: number; + reasoningTokens?: number; + totalTokens?: number; + cacheReadTokens?: number; + cacheWriteTokens?: number; + }; + /** Estimated input tokens before LLM call (for analytics/calibration) */ + estimatedInputTokens?: number; + /** Finish reason: 'tool-calls' means more steps coming, others indicate completion */ + finishReason?: LLMFinishReason; + }; + + /** LLM service requested a tool call */ + 'llm:tool-call': { + toolName: string; + args: Record; + callId?: string; + }; + + /** LLM service returned a tool result */ + 'llm:tool-result': { + toolName: string; + callId?: string; + success: boolean; + /** Sanitized result - present when success=true */ + sanitized?: SanitizedToolResult; + rawResult?: unknown; + /** Error message - present when success=false */ + error?: string; + /** Whether this tool required user approval */ + requireApproval?: boolean; + /** The approval status (only present if requireApproval is true) */ + approvalStatus?: 'approved' | 'rejected'; + }; + + /** Tool execution actually started (after approval if needed) */ + 'tool:running': { + toolName: string; + toolCallId: string; + }; + + /** LLM service error */ + 'llm:error': { + error: Error; + context?: string; + recoverable?: boolean; + /** Tool call ID if error occurred during tool execution */ + toolCallId?: string; + }; + + /** LLM service switched */ + 'llm:switched': { + newConfig: any; // LLMConfig type + historyRetained?: boolean; + }; + + /** LLM service unsupported input */ + 'llm:unsupported-input': { + errors: string[]; + provider: LLMProvider; + model?: string; + fileType?: string; + details?: any; + }; + + /** Context compaction is starting */ + 'context:compacting': { + /** Estimated tokens that triggered compaction */ + estimatedTokens: number; + }; + + /** Context was compacted during multi-step tool calling */ + 'context:compacted': { + /** Actual input tokens from API that triggered compaction */ + originalTokens: number; + /** Estimated tokens after compaction (simple length/4 heuristic) */ + compactedTokens: number; + originalMessages: number; + compactedMessages: number; + strategy: string; + reason: 'overflow' | 'manual'; + }; + + /** Old tool outputs were pruned (marked with compactedAt) to save tokens */ + 'context:pruned': { + prunedCount: number; + savedTokens: number; + }; + + /** User message was queued during agent execution */ + 'message:queued': { + position: number; + id: string; + }; + + /** Queued messages were dequeued and injected into context */ + 'message:dequeued': { + count: number; + ids: string[]; + coalesced: boolean; + /** Combined content of all dequeued messages (for UI display) */ + content: import('../context/types.js').ContentPart[]; + }; + + /** Queued message was removed from queue */ + 'message:removed': { + id: string; + }; + + /** Agent run completed (all steps done, no queued messages remaining) */ + 'run:complete': { + /** How the run ended */ + finishReason: LLMFinishReason; + /** Number of steps executed */ + stepCount: number; + /** Total wall-clock duration of the run in milliseconds */ + durationMs: number; + /** Error that caused termination (only if finishReason === 'error') */ + error?: Error; + }; +} + +export type AgentEventName = keyof AgentEventMap; +export type SessionEventName = keyof SessionEventMap; +export type EventName = keyof AgentEventMap; + +/** + * Compile-time checks to ensure event name arrays and maps stay synchronized + */ +type _AgentEventNamesInMap = (typeof AGENT_EVENT_NAMES)[number] extends keyof AgentEventMap + ? true + : never; +type _SessionEventNamesInMap = (typeof SESSION_EVENT_NAMES)[number] extends SessionEventName + ? true + : never; +type _EventNamesInMap = (typeof EVENT_NAMES)[number] extends EventName ? true : never; + +const _checkAgentEventNames: _AgentEventNamesInMap = true; +const _checkSessionEventNames: _SessionEventNamesInMap = true; +const _checkEventNames: _EventNamesInMap = true; + +// Explicitly mark compile-time checks as used to avoid linter warnings +void _checkAgentEventNames; +void _checkSessionEventNames; +void _checkEventNames; + +/** + * Runtime arrays of event names for iteration, validation, etc. + */ +export const AgentEventNames: readonly AgentEventName[] = Object.freeze([...AGENT_EVENT_NAMES]); +export const SessionEventNames: readonly SessionEventName[] = Object.freeze([ + ...SESSION_EVENT_NAMES, +]); +export const EventNames: readonly EventName[] = Object.freeze([...EVENT_NAMES]); + +/** + * Generic typed EventEmitter base class using composition instead of inheritance + * This provides full compile-time type safety by not extending EventEmitter + * + * Exported for extension by packages like multi-agent-server that need custom event buses. + */ +export class BaseTypedEventEmitter> { + // Wrapped EventEmitter instance + private _emitter = new EventEmitter(); + + // Store listeners with their abort controllers for cleanup + // Maps AbortSignal -> Event Name -> Set of listener functions + private _abortListeners = new WeakMap>>(); + + /** + * Emit an event with type-safe payload + */ + emit( + event: K, + ...args: TEventMap[K] extends void ? [] : [TEventMap[K]] + ): boolean { + return this._emitter.emit(event as string, ...args); + } + + /** + * Subscribe to an event with type-safe listener + */ + on( + event: K, + listener: TEventMap[K] extends void ? () => void : (payload: TEventMap[K]) => void, + options?: { signal?: AbortSignal } + ): this { + // If signal is already aborted, don't add the listener + if (options?.signal?.aborted) { + return this; + } + + // Add the listener + this._emitter.on(event as string, listener); + + // Set up abort handling if signal is provided + if (options?.signal) { + const signal = options.signal; + + // Track this listener for cleanup using Map -> Set structure + if (!this._abortListeners.has(signal)) { + this._abortListeners.set(signal, new Map()); + } + const eventMap = this._abortListeners.get(signal)!; + if (!eventMap.has(event)) { + eventMap.set(event, new Set()); + } + eventMap.get(event)!.add(listener as Function); + + // Set up abort handler + const abortHandler = () => { + this.off(event, listener); + + // Clean up tracking + const eventMap = this._abortListeners.get(signal); + if (eventMap) { + const listenerSet = eventMap.get(event); + if (listenerSet) { + listenerSet.delete(listener as Function); + if (listenerSet.size === 0) { + eventMap.delete(event); + } + } + if (eventMap.size === 0) { + this._abortListeners.delete(signal); + } + } + }; + + signal.addEventListener('abort', abortHandler, { once: true }); + } + + return this; + } + + /** + * Subscribe to an event once with type-safe listener + */ + once( + event: K, + listener: TEventMap[K] extends void ? () => void : (payload: TEventMap[K]) => void, + options?: { signal?: AbortSignal } + ): this { + // If signal is already aborted, don't add the listener + if (options?.signal?.aborted) { + return this; + } + + // Create a wrapper that handles both once and abort cleanup + const onceWrapper = (...args: any[]) => { + // Clean up abort tracking before calling the original listener + if (options?.signal) { + const eventMap = this._abortListeners.get(options.signal); + if (eventMap) { + const listenerSet = eventMap.get(event); + if (listenerSet) { + listenerSet.delete(onceWrapper); + if (listenerSet.size === 0) { + eventMap.delete(event); + } + } + if (eventMap.size === 0) { + this._abortListeners.delete(options.signal); + } + } + } + (listener as any)(...args); + }; + + // Add the wrapped listener + this._emitter.once(event as string, onceWrapper); + + // Set up abort handling if signal is provided + if (options?.signal) { + const signal = options.signal; + + // Track this listener for cleanup using Map -> Set structure + if (!this._abortListeners.has(signal)) { + this._abortListeners.set(signal, new Map()); + } + const eventMap = this._abortListeners.get(signal)!; + if (!eventMap.has(event)) { + eventMap.set(event, new Set()); + } + eventMap.get(event)!.add(onceWrapper); + + // Set up abort handler + const abortHandler = () => { + this.off(event, onceWrapper); + + // Clean up tracking + const eventMap = this._abortListeners.get(signal); + if (eventMap) { + const listenerSet = eventMap.get(event); + if (listenerSet) { + listenerSet.delete(onceWrapper); + if (listenerSet.size === 0) { + eventMap.delete(event); + } + } + if (eventMap.size === 0) { + this._abortListeners.delete(signal); + } + } + }; + + signal.addEventListener('abort', abortHandler, { once: true }); + } + + return this; + } + + /** + * Unsubscribe from an event + */ + off( + event: K, + listener: TEventMap[K] extends void ? () => void : (payload: TEventMap[K]) => void + ): this { + this._emitter.off(event as string, listener); + return this; + } +} + +/** + * Agent-level typed event emitter for global agent events + */ +export class AgentEventBus extends BaseTypedEventEmitter {} + +/** + * Session-level typed event emitter for session-scoped events + */ +export class SessionEventBus extends BaseTypedEventEmitter {} + +/** + * Combined typed event emitter for backward compatibility + */ +export class TypedEventEmitter extends BaseTypedEventEmitter {} + +/** + * Global shared event bus (backward compatibility) + */ +export const eventBus = new TypedEventEmitter(); diff --git a/dexto/packages/core/src/image/define-image.ts b/dexto/packages/core/src/image/define-image.ts new file mode 100644 index 00000000..108658f8 --- /dev/null +++ b/dexto/packages/core/src/image/define-image.ts @@ -0,0 +1,212 @@ +/** + * Image definition helper + * + * Provides type-safe API for defining base images. + */ + +import type { ImageDefinition } from './types.js'; + +/** + * Define a Dexto base image. + * + * This function provides type checking and validation for image definitions. + * Use this in your dexto.image.ts file. + * + * @example + * ```typescript + * // dexto.image.ts + * import { defineImage } from '@dexto/core'; + * import { localBlobProvider } from './providers/blob.js'; + * + * export default defineImage({ + * name: 'local', + * version: '1.0.0', + * description: 'Local development base image', + * target: 'local-development', + * + * providers: { + * blobStore: { + * providers: [localBlobProvider], + * }, + * }, + * + * defaults: { + * storage: { + * blob: { type: 'local', storePath: './data/blobs' }, + * }, + * }, + * + * constraints: ['filesystem-required', 'offline-capable'], + * }); + * ``` + * + * @param definition - Image definition object + * @returns The same definition (for type inference) + */ +export function defineImage(definition: ImageDefinition): ImageDefinition { + // Validation + if (!definition.name) { + throw new Error('Image definition must have a name'); + } + if (!definition.version) { + throw new Error('Image definition must have a version'); + } + if (!definition.description) { + throw new Error('Image definition must have a description'); + } + + // Validate provider categories have at least one of: providers or register + for (const [category, config] of Object.entries(definition.providers)) { + if (!config) continue; + if (!config.providers && !config.register) { + throw new Error( + `Provider category '${category}' must have either 'providers' array or 'register' function` + ); + } + } + + return definition; +} + +/** + * Helper to create a provider category configuration. + * + * @example + * ```typescript + * import { defineProviderCategory } from '@dexto/core'; + * + * const blobStore = defineProviderCategory({ + * providers: [localBlobProvider, s3BlobProvider], + * }); + * ``` + */ +export function defineProviderCategory(config: { + providers?: any[]; + register?: () => void | Promise; +}) { + if (!config.providers && !config.register) { + throw new Error('Provider category must have either providers or register function'); + } + return config; +} + +/** + * Validate an image definition. + * Throws if the definition is invalid. + * + * Used by bundler to validate images before building. + */ +export function validateImageDefinition(definition: ImageDefinition): void { + // Basic validation + if (!definition.name || typeof definition.name !== 'string') { + throw new Error('Image name must be a non-empty string'); + } + + if (!definition.version || typeof definition.version !== 'string') { + throw new Error('Image version must be a non-empty string'); + } + + if (!definition.description || typeof definition.description !== 'string') { + throw new Error('Image description must be a non-empty string'); + } + + // Validate version format (basic semver check) + const versionRegex = /^\d+\.\d+\.\d+(-[a-zA-Z0-9.-]+)?$/; + if (!versionRegex.test(definition.version)) { + throw new Error( + `Image version '${definition.version}' is not valid semver. Expected format: x.y.z` + ); + } + + // Validate target if provided + const validTargets = [ + 'local-development', + 'cloud-production', + 'edge-serverless', + 'embedded-iot', + 'enterprise', + 'custom', + ]; + if (definition.target && !validTargets.includes(definition.target)) { + throw new Error( + `Invalid target '${definition.target}'. Valid targets: ${validTargets.join(', ')}` + ); + } + + // Validate provider categories + // Allow empty providers if extending a base image (providers inherited from base) + const hasProviders = + definition.providers && + Object.values(definition.providers).some((config) => config !== undefined); + + if (!hasProviders && !definition.extends) { + throw new Error( + 'Image must either define at least one provider category or extend a base image' + ); + } + + for (const [category, config] of Object.entries(definition.providers)) { + if (!config) continue; + + if (!config.providers && !config.register) { + throw new Error( + `Provider category '${category}' must have either 'providers' array or 'register' function` + ); + } + + if (config.providers && !Array.isArray(config.providers)) { + throw new Error(`Provider category '${category}' providers must be an array`); + } + + if (config.register && typeof config.register !== 'function') { + throw new Error(`Provider category '${category}' register must be a function`); + } + } + + // Validate constraints if provided + const validConstraints = [ + 'filesystem-required', + 'network-required', + 'offline-capable', + 'serverless-compatible', + 'cold-start-optimized', + 'low-memory', + 'edge-compatible', + 'browser-compatible', + ]; + + if (definition.constraints) { + if (!Array.isArray(definition.constraints)) { + throw new Error('Image constraints must be an array'); + } + + for (const constraint of definition.constraints) { + if (!validConstraints.includes(constraint)) { + throw new Error( + `Invalid constraint '${constraint}'. Valid constraints: ${validConstraints.join(', ')}` + ); + } + } + } + + // Validate utils if provided + if (definition.utils) { + for (const [name, path] of Object.entries(definition.utils)) { + if (typeof path !== 'string') { + throw new Error(`Utility '${name}' path must be a string`); + } + if (!path.startsWith('./')) { + throw new Error( + `Utility '${name}' path must be relative (start with './'). Got: ${path}` + ); + } + } + } + + // Validate extends if provided + if (definition.extends) { + if (typeof definition.extends !== 'string') { + throw new Error('Image extends must be a string (parent image name)'); + } + } +} diff --git a/dexto/packages/core/src/image/index.ts b/dexto/packages/core/src/image/index.ts new file mode 100644 index 00000000..7546e61f --- /dev/null +++ b/dexto/packages/core/src/image/index.ts @@ -0,0 +1,68 @@ +/** + * Base Image Infrastructure + * + * Provides types and helpers for defining Dexto base images. + * Base images are pre-configured backend surfaces that bundle providers, + * utilities, and defaults for specific deployment targets. + * + * @example Creating a base image + * ```typescript + * // dexto.image.ts + * import { defineImage } from '@dexto/core'; + * + * export default defineImage({ + * name: 'local', + * version: '1.0.0', + * description: 'Local development base image', + * target: 'local-development', + * + * providers: { + * blobStore: { + * providers: [localBlobProvider], + * }, + * database: { + * register: async () => { + * const { sqliteProvider } = await import('./providers/database.js'); + * databaseRegistry.register(sqliteProvider); + * }, + * }, + * }, + * + * defaults: { + * storage: { + * blob: { type: 'local', storePath: './data/blobs' }, + * database: { type: 'sqlite', path: './data/agent.db' }, + * }, + * }, + * + * constraints: ['filesystem-required', 'offline-capable'], + * }); + * ``` + * + * @example Using a base image + * ```typescript + * // my-app/src/index.ts + * import { createAgent, enrichConfigForLocal } from '@dexto/image-local'; + * + * const config = enrichConfigForLocal(rawConfig); + * const agent = createAgent(config); // Providers already registered! + * ``` + */ + +// Core types +export type { + ImageProvider, + ProviderMetadata, + ProviderRegistrationFn, + ProviderCategoryConfig, + ImageDefinition, + ImageTarget, + ImageConstraint, + ImageDefaults, + ImageMetadata, + ImageBuildResult, + ImageBuildOptions, +} from './types.js'; + +// Definition helpers +export { defineImage, defineProviderCategory, validateImageDefinition } from './define-image.js'; diff --git a/dexto/packages/core/src/image/types.ts b/dexto/packages/core/src/image/types.ts new file mode 100644 index 00000000..783a8ffb --- /dev/null +++ b/dexto/packages/core/src/image/types.ts @@ -0,0 +1,278 @@ +/** + * Dexto Base Image Definition + * + * Base images are pre-configured backend surfaces that bundle providers, + * utilities, and defaults for specific deployment targets. + * + * Like Alpine Linux or Ubuntu, but for AI agents. + */ + +import type { z } from 'zod'; + +/** + * Generic provider interface that all provider types should extend. + * Provides common structure for type-safe provider registration. + * + * Note: This is a simplified interface for image definitions. + * Actual provider implementations should use the specific provider + * interfaces from their respective modules (e.g., BlobStoreProvider). + */ +export interface ImageProvider { + /** Unique type identifier for this provider (e.g., 'sqlite', 'local', 's3') */ + type: TType; + /** Zod schema for validating provider configuration */ + configSchema: z.ZodType; + /** Factory function to create provider instance */ + create: (config: any, deps: any) => any; + /** Optional metadata about the provider */ + metadata?: ProviderMetadata; +} + +/** + * Metadata about a provider's characteristics and requirements + */ +export interface ProviderMetadata { + /** Human-readable display name */ + displayName?: string; + /** Brief description of what this provider does */ + description?: string; + /** Whether this provider requires network connectivity */ + requiresNetwork?: boolean; + /** Whether this provider requires filesystem access */ + requiresFilesystem?: boolean; + /** Persistence level of storage providers */ + persistenceLevel?: 'ephemeral' | 'persistent'; + /** Platforms this provider is compatible with */ + platforms?: ('node' | 'browser' | 'edge' | 'worker')[]; +} + +/** + * Registry function that registers providers on module initialization. + * Called automatically when the image is imported. + */ +export type ProviderRegistrationFn = () => void | Promise; + +/** + * Configuration for a single provider category in an image. + * Supports both direct provider objects and registration functions. + */ +export interface ProviderCategoryConfig { + /** Direct provider objects to register */ + providers?: ImageProvider[]; + /** Registration function for complex initialization */ + register?: ProviderRegistrationFn; +} + +/** + * Complete image definition structure. + * This is what dexto.image.ts exports. + */ +export interface ImageDefinition { + /** Unique name for this image (e.g., 'local', 'cloud', 'edge') */ + name: string; + /** Semantic version of this image */ + version: string; + /** Brief description of this image's purpose and target environment */ + description: string; + /** Target deployment environment (for documentation and validation) */ + target?: ImageTarget; + + /** + * Provider categories to register. + * Each category can include direct providers or a registration function. + */ + providers: { + /** Blob storage providers (e.g., local filesystem, S3, R2) */ + blobStore?: ProviderCategoryConfig; + /** Database providers (e.g., SQLite, PostgreSQL, D1) */ + database?: ProviderCategoryConfig; + /** Cache providers (e.g., in-memory, Redis, KV) */ + cache?: ProviderCategoryConfig; + /** Custom tool providers (e.g., datetime helpers, API integrations) */ + customTools?: ProviderCategoryConfig; + /** Plugin providers (e.g., audit logging, content filtering) */ + plugins?: ProviderCategoryConfig; + /** Compression strategy providers (e.g., sliding window, summarization) */ + compression?: ProviderCategoryConfig; + }; + + /** + * Default configuration values. + * Used when agent config doesn't specify values. + * Merged with agent config during agent creation. + */ + defaults?: ImageDefaults; + + /** + * Runtime constraints this image requires. + * Used for validation and error messages. + */ + constraints?: ImageConstraint[]; + + /** + * Utilities exported by this image. + * Maps utility name to file path (relative to image root). + * + * Example: + * { + * configEnrichment: './utils/config.js', + * lifecycle: './utils/lifecycle.js' + * } + */ + utils?: Record; + + /** + * Selective named exports from packages. + * Allows re-exporting specific types and values from dependencies. + * + * Example: + * { + * '@dexto/core': ['logger', 'createAgentCard', 'type DextoAgent'], + * '@dexto/utils': ['formatDate', 'parseConfig'] + * } + */ + exports?: Record; + + /** + * Parent image to extend (for image inheritance). + * Optional: enables creating specialized images from base images. + */ + extends?: string; + + /** + * Bundled plugin paths. + * Absolute paths to plugin directories containing .dexto-plugin or .claude-plugin manifests. + * These plugins are automatically discovered alongside user/project plugins. + * + * Example: + * ```typescript + * import { PLUGIN_PATH as planToolsPluginPath } from '@dexto/tools-plan'; + * + * bundledPlugins: [planToolsPluginPath] + * ``` + */ + bundledPlugins?: string[]; +} + +/** + * Target deployment environments for images. + * Helps users choose the right image for their use case. + */ +export type ImageTarget = + | 'local-development' + | 'cloud-production' + | 'edge-serverless' + | 'embedded-iot' + | 'enterprise' + | 'custom'; + +/** + * Runtime constraints that an image requires. + * Used for validation and helpful error messages. + */ +export type ImageConstraint = + | 'filesystem-required' + | 'network-required' + | 'offline-capable' + | 'serverless-compatible' + | 'cold-start-optimized' + | 'low-memory' + | 'edge-compatible' + | 'browser-compatible'; + +/** + * Default configuration values provided by an image. + * These are used when agent config doesn't specify values. + */ +export interface ImageDefaults { + /** Default storage configuration */ + storage?: { + database?: { + type: string; + [key: string]: any; + }; + blob?: { + type: string; + [key: string]: any; + }; + cache?: { + type: string; + [key: string]: any; + }; + }; + /** Default logging configuration */ + logging?: { + level?: 'debug' | 'info' | 'warn' | 'error'; + fileLogging?: boolean; + [key: string]: any; + }; + /** Default LLM configuration */ + llm?: { + provider?: string; + model?: string; + [key: string]: any; + }; + /** Default tool configuration */ + tools?: { + internalTools?: string[]; + [key: string]: any; + }; + /** Other default values */ + [key: string]: any; +} + +/** + * Metadata about a built image (generated by bundler). + * Included in the compiled image output. + */ +export interface ImageMetadata { + /** Image name */ + name: string; + /** Image version */ + version: string; + /** Description */ + description: string; + /** Target environment */ + target?: ImageTarget; + /** Runtime constraints */ + constraints: ImageConstraint[]; + /** Build timestamp */ + builtAt: string; + /** Core version this image was built for */ + coreVersion: string; + /** Base image this extends (if any) */ + extends?: string; + /** Bundled plugin paths (absolute paths to plugin directories) */ + bundledPlugins?: string[]; +} + +/** + * Result of building an image. + * Contains the generated code and metadata. + */ +export interface ImageBuildResult { + /** Generated JavaScript code for the image entry point */ + code: string; + /** Generated TypeScript definitions */ + types: string; + /** Image metadata */ + metadata: ImageMetadata; + /** Warnings encountered during build */ + warnings?: string[]; +} + +/** + * Options for building an image. + */ +export interface ImageBuildOptions { + /** Path to dexto.image.ts file */ + imagePath: string; + /** Output directory for built image */ + outDir: string; + /** Whether to generate source maps */ + sourcemap?: boolean; + /** Whether to minify output */ + minify?: boolean; + /** Additional validation rules */ + strict?: boolean; +} diff --git a/dexto/packages/core/src/index.browser.ts b/dexto/packages/core/src/index.browser.ts new file mode 100644 index 00000000..0cb2331e --- /dev/null +++ b/dexto/packages/core/src/index.browser.ts @@ -0,0 +1,108 @@ +// Browser-safe root exports for @dexto/core +// Export only what's actually used by client packages (webui, cli, client-sdk) + +// Runtime utilities (actually used by client packages) +export { toError } from './utils/error-conversion.js'; // Used by webui package +export { zodToIssues } from './utils/result.js'; // Used by client-sdk package +export { ErrorScope, ErrorType } from './errors/types.js'; // Used by client-sdk package + +// Type-only exports (used as types, no runtime overhead) +export type { Issue, Severity, DextoErrorCode } from './errors/types.js'; + +// Context/message types (used by webui package) +export type { + InternalMessage, + SystemMessage, + UserMessage, + AssistantMessage, + ToolMessage, + TextPart, + FilePart, + ImageData, + FileData, + UIResourcePart, + ContentPart, + ToolCall, + ToolApprovalStatus, +} from './context/types.js'; +// Note: ImagePart not exported - only used internally in core package + +// Message type guards (used by CLI and webui packages) +export { + isSystemMessage, + isUserMessage, + isAssistantMessage, + isToolMessage, + isTextPart, + isImagePart, + isFilePart, + isUIResourcePart, +} from './context/types.js'; + +// Context utilities (used by webui package for media kind detection) +export { getFileMediaKind, getResourceKind } from './context/media-helpers.js'; + +// LLM types (used by client packages) +export type { LLMProvider } from './llm/types.js'; +export { LLM_PROVIDERS } from './llm/types.js'; + +// MCP types and constants (used by webui) +export type { McpServerType, McpConnectionMode } from './mcp/schemas.js'; +export { + MCP_SERVER_TYPES, + MCP_CONNECTION_MODES, + DEFAULT_MCP_CONNECTION_MODE, +} from './mcp/schemas.js'; + +// Storage types and constants (used by webui) +export type { CacheType, DatabaseType } from './storage/schemas.js'; +export { CACHE_TYPES, DATABASE_TYPES } from './storage/schemas.js'; + +// Tool confirmation types and constants (used by webui) +export type { ToolConfirmationMode, AllowedToolsStorageType } from './tools/schemas.js'; +export { + TOOL_CONFIRMATION_MODES, + ALLOWED_TOOLS_STORAGE_TYPES, + DEFAULT_TOOL_CONFIRMATION_MODE, + DEFAULT_ALLOWED_TOOLS_STORAGE, +} from './tools/schemas.js'; + +// Approval types and constants (used by webui) +export { ApprovalStatus, ApprovalType, DenialReason } from './approval/types.js'; +export type { ApprovalRequest, ApprovalResponse } from './approval/types.js'; + +// Session types (used by CLI package) +export type { SessionMetadata } from './session/session-manager.js'; + +// Agent types (used by webui for form configuration) +export type { AgentConfig, ValidatedAgentConfig } from './agent/schemas.js'; + +// System prompt types and constants (used by webui) +export { PROMPT_GENERATOR_SOURCES } from './systemPrompt/registry.js'; +export type { ContributorConfig, SystemPromptConfig } from './systemPrompt/schemas.js'; + +// Search types (used by client-sdk package) +export type { + SearchOptions, + SearchResult, + SessionSearchResult, + SearchResponse, + SessionSearchResponse, +} from './search/types.js'; + +// Event types (used by client-sdk package) +export type { AgentEventMap, SessionEventMap } from './events/index.js'; + +// LLM registry types (used by client-sdk package) +export type { ModelInfo, ProviderInfo } from './llm/registry.js'; +export type { SupportedFileType } from './llm/types.js'; + +// Resource types and utilities (used by webui package) +// Note: Only export browser-safe reference parsing functions, NOT ResourceManager +// (ResourceManager requires logger which has Node.js dependencies) +export type { ResourceMetadata } from './resources/types.js'; +export type { ResourceReference } from './resources/reference-parser.js'; +export { + parseResourceReferences, + resolveResourceReferences, +} from './resources/reference-parser.js'; diff --git a/dexto/packages/core/src/index.ts b/dexto/packages/core/src/index.ts new file mode 100644 index 00000000..e19ecc54 --- /dev/null +++ b/dexto/packages/core/src/index.ts @@ -0,0 +1,95 @@ +/** + * @dexto/core - Main entry point + * + * This package is designed for server-side use (Node.js). + * For browser/client usage, use server components/actions or the API. + * + * The package.json conditional exports handle environment routing: + * - Browser: Routes to index.browser.ts (minimal safe exports) + * - Node: Routes to this file (full exports) + * + * TODO: Break down into subpath exports for better tree-shaking + * Consider adding exports like: + * - @dexto/core/telemetry - Telemetry utilities + * - @dexto/core/llm - LLM services and factories + * - @dexto/core/session - Session management (currently internal) + * - @dexto/core/tools - Tool system + * This would allow: + * 1. Better tree-shaking (only import what you need) + * 2. Cleaner public API boundaries + * 3. Reduced bundle sizes for packages that only need specific functionality + * 4. Avoid pulling in OpenTelemetry decorators for packages that don't need instrumentation + */ + +// Core Agent +export * from './agent/index.js'; + +// Configuration +// Config loading has been moved to @dexto/agent-management +// Import from '@dexto/agent-management' instead: +// - loadAgentConfig +// - ConfigError +// - ConfigErrorCode + +// Errors +export * from './errors/index.js'; + +// Events +export * from './events/index.js'; + +// LLM +export * from './llm/index.js'; + +// Search +export * from './search/index.js'; + +// Logger +export * from './logger/index.js'; + +// MCP +export * from './mcp/index.js'; + +// Session +export * from './session/index.js'; + +// Storage +export * from './storage/index.js'; + +// System Prompt +export * from './systemPrompt/index.js'; + +// Tools +export * from './tools/index.js'; + +// Context +export * from './context/index.js'; +export { getFileMediaKind, getResourceKind } from './context/index.js'; + +// Prompts +export * from './prompts/index.js'; + +// Utils +export * from './utils/index.js'; + +// Resources +export * from './resources/index.js'; + +// Approval (User Approval System) +export * from './approval/index.js'; + +// Memory +export * from './memory/index.js'; + +// Plugins +export * from './plugins/index.js'; + +// Telemetry +export * from './telemetry/index.js'; + +// Providers +export * from './providers/index.js'; + +// Base Image Infrastructure +export * from './image/index.js'; + +// Note: Blob types, schemas, and errors are exported from './storage/index.js' diff --git a/dexto/packages/core/src/llm/error-codes.ts b/dexto/packages/core/src/llm/error-codes.ts new file mode 100644 index 00000000..bf511121 --- /dev/null +++ b/dexto/packages/core/src/llm/error-codes.ts @@ -0,0 +1,38 @@ +/** + * LLM-specific error codes + * Includes configuration, validation, and runtime errors for LLM operations + */ +export enum LLMErrorCode { + // Configuration errors + API_KEY_MISSING = 'llm_api_key_missing', + API_KEY_INVALID = 'llm_api_key_invalid', // Too short, wrong format + API_KEY_CANDIDATE_MISSING = 'llm_api_key_candidate_missing', + BASE_URL_MISSING = 'llm_base_url_missing', + BASE_URL_INVALID = 'llm_base_url_invalid', + CONFIG_MISSING = 'llm_config_missing', // Required config (e.g., GOOGLE_VERTEX_PROJECT) + + // Model/Provider compatibility + MODEL_INCOMPATIBLE = 'llm_model_incompatible', + MODEL_UNKNOWN = 'llm_model_unknown', + PROVIDER_UNSUPPORTED = 'llm_provider_unsupported', + + // Input validation (formerly generic "validation") + INPUT_FILE_UNSUPPORTED = 'llm_input_file_unsupported', + INPUT_IMAGE_UNSUPPORTED = 'llm_input_image_unsupported', + INPUT_TEXT_INVALID = 'llm_input_text_invalid', + + // Limits + TOKENS_EXCEEDED = 'llm_tokens_exceeded', + RATE_LIMIT_EXCEEDED = 'llm_rate_limit_exceeded', + INSUFFICIENT_CREDITS = 'llm_insufficient_credits', + + // Operations + SWITCH_FAILED = 'llm_switch_failed', + GENERATION_FAILED = 'llm_generation_failed', + + // Input validation (moved from agent) + SWITCH_INPUT_MISSING = 'llm_switch_input_missing', // At least model or provider must be specified + + // Schema validation + REQUEST_INVALID_SCHEMA = 'llm_request_invalid_schema', +} diff --git a/dexto/packages/core/src/llm/errors.ts b/dexto/packages/core/src/llm/errors.ts new file mode 100644 index 00000000..acef10ec --- /dev/null +++ b/dexto/packages/core/src/llm/errors.ts @@ -0,0 +1,142 @@ +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { ErrorScope } from '@core/errors/types.js'; +import { ErrorType } from '../errors/types.js'; +import { LLMErrorCode } from './error-codes.js'; +// Use types solely from types.ts to avoid duplication +import { getSupportedProviders } from './registry.js'; +import type { LLMProvider } from './types.js'; + +/** + * LLM runtime error factory methods + * Creates properly typed errors for LLM runtime operations + * + * Note: Validation errors (missing API keys, invalid models, etc.) are handled + * by DextoValidationError through Zod schema validation + */ +export class LLMError { + // Runtime model/provider lookup errors + static unknownModel(provider: LLMProvider, model: string) { + return new DextoRuntimeError( + LLMErrorCode.MODEL_UNKNOWN, + ErrorScope.LLM, + ErrorType.USER, + `Unknown model '${model}' for provider '${provider}'`, + { provider, model } + ); + } + + static baseUrlMissing(provider: LLMProvider) { + return new DextoRuntimeError( + LLMErrorCode.BASE_URL_MISSING, + ErrorScope.LLM, + ErrorType.USER, + `Provider '${provider}' requires a baseURL (set config.baseURL or OPENAI_BASE_URL environment variable)`, + { provider } + ); + } + + static missingConfig(provider: LLMProvider, configName: string) { + return new DextoRuntimeError( + LLMErrorCode.CONFIG_MISSING, + ErrorScope.LLM, + ErrorType.USER, + `Provider '${provider}' requires ${configName}`, + { provider, configName } + ); + } + + static unsupportedProvider(provider: string) { + const availableProviders = getSupportedProviders(); + return new DextoRuntimeError( + LLMErrorCode.PROVIDER_UNSUPPORTED, + ErrorScope.LLM, + ErrorType.USER, + `Provider '${provider}' is not supported. Available providers: ${availableProviders.join(', ')}`, + { provider, availableProviders } + ); + } + + /** + * Runtime error when API key is missing for a provider that requires it. + * This occurs when relaxed validation allowed the app to start without an API key, + * and the user then tries to use the LLM functionality. + */ + static apiKeyMissing(provider: LLMProvider, envVar: string) { + return new DextoRuntimeError( + LLMErrorCode.API_KEY_MISSING, + ErrorScope.LLM, + ErrorType.USER, + `API key required for provider '${provider}'`, + { provider, envVar }, + `Set the ${envVar} environment variable or configure it in Settings` + ); + } + + static modelProviderUnknown(model: string) { + const availableProviders = getSupportedProviders(); + return new DextoRuntimeError( + LLMErrorCode.MODEL_UNKNOWN, + ErrorScope.LLM, + ErrorType.USER, + `Unknown model '${model}' - could not infer provider. Available providers: ${availableProviders.join(', ')}`, + { model, availableProviders }, + 'Specify the provider explicitly or use a recognized model name' + ); + } + + // Runtime service errors + + static rateLimitExceeded(provider: LLMProvider, retryAfter?: number) { + return new DextoRuntimeError( + LLMErrorCode.RATE_LIMIT_EXCEEDED, + ErrorScope.LLM, + ErrorType.RATE_LIMIT, + `Rate limit exceeded for ${provider}`, + { + details: { provider, retryAfter }, + recovery: retryAfter + ? `Wait ${retryAfter} seconds before retrying` + : 'Wait before retrying or upgrade your plan', + } + ); + } + + /** + * Error when Dexto account has insufficient credits. + * Returned as 402 from the gateway with code INSUFFICIENT_CREDITS. + */ + static insufficientCredits(balance?: number) { + const balanceStr = balance !== undefined ? `$${balance.toFixed(2)}` : 'low'; + return new DextoRuntimeError( + LLMErrorCode.INSUFFICIENT_CREDITS, + ErrorScope.LLM, + ErrorType.FORBIDDEN, + `Insufficient Dexto credits. Balance: ${balanceStr}`, + { balance }, + 'Run `dexto billing` to check your balance' + ); + } + + // Runtime operation errors + static generationFailed(error: string, provider: LLMProvider, model: string) { + return new DextoRuntimeError( + LLMErrorCode.GENERATION_FAILED, + ErrorScope.LLM, + ErrorType.THIRD_PARTY, + `Generation failed: ${error}`, + { details: { error, provider, model } } + ); + } + + // Switch operation errors (runtime checks not covered by Zod) + static switchInputMissing() { + return new DextoRuntimeError( + LLMErrorCode.SWITCH_INPUT_MISSING, + ErrorScope.LLM, + ErrorType.USER, + 'At least model or provider must be specified for LLM switch', + {}, + 'Provide either a model name, provider, or both' + ); + } +} diff --git a/dexto/packages/core/src/llm/executor/provider-options.ts b/dexto/packages/core/src/llm/executor/provider-options.ts new file mode 100644 index 00000000..dbdad2c2 --- /dev/null +++ b/dexto/packages/core/src/llm/executor/provider-options.ts @@ -0,0 +1,131 @@ +/** + * Provider-specific options builder for Vercel AI SDK's streamText/generateText. + * + * Centralizes provider-specific configuration that requires explicit opt-in: + * - Anthropic: cacheControl for prompt caching, sendReasoning for extended thinking + * - Bedrock/Vertex Claude: Same as Anthropic (Claude models on these platforms) + * - Google: thinkingConfig for Gemini thinking models + * - OpenAI: reasoningEffort for o1/o3/codex/gpt-5 models + * + * Caching notes: + * - Anthropic: Requires explicit cacheControl option (we enable it) + * - OpenAI: Automatic for prompts ≥1024 tokens (no config needed) + * - Google: Implicit caching automatic for Gemini 2.5+ (≥1024 tokens for Flash, + * ≥2048 for Pro). Explicit caching requires pre-created cachedContent IDs. + * All providers return cached token counts in the response (cachedInputTokens). + */ + +import type { LLMProvider } from '../types.js'; +import { isReasoningCapableModel } from '../registry.js'; + +export type ReasoningEffort = 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; + +export interface ProviderOptionsConfig { + provider: LLMProvider; + model: string; + reasoningEffort?: ReasoningEffort | undefined; +} + +/** + * Build provider-specific options for streamText/generateText. + * + * @param config Provider, model, and optional reasoning effort configuration + * @returns Provider options object or undefined if no special options needed + */ +export function buildProviderOptions( + config: ProviderOptionsConfig +): Record> | undefined { + const { provider, model, reasoningEffort } = config; + const modelLower = model.toLowerCase(); + + // Anthropic: Enable prompt caching and reasoning streaming + if (provider === 'anthropic') { + return { + anthropic: { + // Enable prompt caching - saves money and improves latency + cacheControl: { type: 'ephemeral' }, + // Stream reasoning/thinking content when model supports it + sendReasoning: true, + }, + }; + } + + // Bedrock: Enable caching and reasoning for Claude models + if (provider === 'bedrock' && modelLower.includes('claude')) { + return { + bedrock: { + cacheControl: { type: 'ephemeral' }, + sendReasoning: true, + }, + }; + } + + // Vertex: Enable caching and reasoning for Claude models + if (provider === 'vertex' && modelLower.includes('claude')) { + return { + 'vertex-anthropic': { + cacheControl: { type: 'ephemeral' }, + sendReasoning: true, + }, + }; + } + + // Google: Enable thinking for models that support it + // Note: Google automatically enables thinking for thinking models, + // but we explicitly enable includeThoughts to receive the reasoning + if (provider === 'google' || (provider === 'vertex' && !modelLower.includes('claude'))) { + return { + google: { + thinkingConfig: { + // Include thoughts in the response for transparency + includeThoughts: true, + }, + }, + }; + } + + // OpenAI: Set reasoning effort for reasoning-capable models + // Use config value if provided, otherwise auto-detect based on model + if (provider === 'openai') { + const effectiveEffort = reasoningEffort ?? getDefaultReasoningEffort(model); + if (effectiveEffort) { + return { + openai: { + reasoningEffort: effectiveEffort, + }, + }; + } + } + + return undefined; +} + +/** + * Determine the default reasoning effort for OpenAI models. + * + * OpenAI reasoning effort levels (from lowest to highest): + * - 'none': No reasoning, fastest responses + * - 'low': Minimal reasoning, fast responses + * - 'medium': Balanced reasoning (OpenAI's recommended daily driver) + * - 'high': Thorough reasoning for complex tasks + * - 'xhigh': Extra high reasoning for quality-critical, non-latency-sensitive tasks + * + * Default strategy: + * - Reasoning-capable models (codex, o1, o3, gpt-5): 'medium' - OpenAI's recommended default + * - Other models: undefined (no reasoning effort needed) + * + * @param model The model name + * @returns Reasoning effort level or undefined if not applicable + */ +export function getDefaultReasoningEffort( + model: string +): Exclude | undefined { + // Use the centralized registry function for capability detection + if (isReasoningCapableModel(model)) { + // 'medium' is OpenAI's recommended daily driver for reasoning models + return 'medium'; + } + + // Other models don't need explicit reasoning effort + return undefined; +} diff --git a/dexto/packages/core/src/llm/executor/stream-processor.test.ts b/dexto/packages/core/src/llm/executor/stream-processor.test.ts new file mode 100644 index 00000000..a0202c95 --- /dev/null +++ b/dexto/packages/core/src/llm/executor/stream-processor.test.ts @@ -0,0 +1,1293 @@ +import { describe, test, expect, vi, beforeEach, afterEach } from 'vitest'; +import { StreamProcessor } from './stream-processor.js'; +import type { StreamProcessorConfig } from './stream-processor.js'; +import type { ContextManager } from '../../context/manager.js'; +import type { SessionEventBus } from '../../events/index.js'; +import type { ResourceManager } from '../../resources/index.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; + +/** + * Creates a mock async generator that yields events + */ +function createMockStream(events: Array>) { + return { + fullStream: (async function* () { + for (const event of events) { + yield event; + } + })(), + }; +} + +/** + * Creates mock dependencies for StreamProcessor + */ +function createMocks() { + const emittedEvents: Array<{ name: string; payload: unknown }> = []; + + const mockContextManager = { + addAssistantMessage: vi.fn().mockResolvedValue(undefined), + appendAssistantText: vi.fn().mockResolvedValue(undefined), + updateAssistantMessage: vi.fn().mockResolvedValue(undefined), + addToolCall: vi.fn().mockResolvedValue(undefined), + addToolResult: vi.fn().mockResolvedValue(undefined), + getHistory: vi.fn().mockResolvedValue([{ id: 'msg-1', role: 'assistant', content: '' }]), + } as unknown as ContextManager; + + const mockEventBus = { + emit: vi.fn((name: string, payload: unknown) => { + emittedEvents.push({ name, payload }); + }), + } as unknown as SessionEventBus; + + const mockResourceManager = { + getBlobStore: vi.fn().mockReturnValue({}), + } as unknown as ResourceManager; + + const mockLogger = { + createChild: vi.fn().mockReturnThis(), + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + } as unknown as IDextoLogger; + + const mockAbortController = new AbortController(); + + const config: StreamProcessorConfig = { + provider: 'openai', + model: 'gpt-4', + }; + + return { + contextManager: mockContextManager, + eventBus: mockEventBus, + resourceManager: mockResourceManager, + logger: mockLogger, + abortController: mockAbortController, + config, + emittedEvents, + }; +} + +describe('StreamProcessor', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('Text Accumulation', () => { + test('accumulates text from multiple text-delta events', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Hello' }, + { type: 'text-delta', text: ' world' }, + { type: 'text-delta', text: '!' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + expect(result.text).toBe('Hello world!'); + }); + + test('returns accumulated text in result', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Test response' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + expect(result.text).toBe('Test response'); + expect(result.finishReason).toBe('stop'); + }); + + test('includes accumulated text in llm:response event', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Response content' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + const responseEvent = mocks.emittedEvents.find((e) => e.name === 'llm:response'); + expect(responseEvent).toBeDefined(); + expect((responseEvent?.payload as { content: string }).content).toBe( + 'Response content' + ); + }); + + test('creates assistant message on first text-delta', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Hello' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + expect(mocks.contextManager.addAssistantMessage).toHaveBeenCalledWith('', [], {}); + }); + + test('appends text to assistant message for each delta', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Hello' }, + { type: 'text-delta', text: ' world' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + expect(mocks.contextManager.appendAssistantText).toHaveBeenCalledTimes(2); + expect(mocks.contextManager.appendAssistantText).toHaveBeenNthCalledWith( + 1, + 'msg-1', + 'Hello' + ); + expect(mocks.contextManager.appendAssistantText).toHaveBeenNthCalledWith( + 2, + 'msg-1', + ' world' + ); + }); + }); + + describe('Chunk Emission', () => { + test('streaming=true: emits llm:chunk for each text-delta', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true // streaming = true + ); + + const events = [ + { type: 'text-delta', text: 'Hello' }, + { type: 'text-delta', text: ' world' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + const chunkEvents = mocks.emittedEvents.filter((e) => e.name === 'llm:chunk'); + expect(chunkEvents).toHaveLength(2); + expect((chunkEvents[0]?.payload as { content: string }).content).toBe('Hello'); + expect((chunkEvents[1]?.payload as { content: string }).content).toBe(' world'); + }); + + test('streaming=false: does NOT emit llm:chunk events', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + false // streaming = false + ); + + const events = [ + { type: 'text-delta', text: 'Hello' }, + { type: 'text-delta', text: ' world' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + const chunkEvents = mocks.emittedEvents.filter((e) => e.name === 'llm:chunk'); + expect(chunkEvents).toHaveLength(0); + }); + + test('still accumulates text when streaming=false', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + false // streaming = false + ); + + const events = [ + { type: 'text-delta', text: 'Hello' }, + { type: 'text-delta', text: ' world' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + expect(result.text).toBe('Hello world'); + }); + }); + + describe('Reasoning Delta Handling', () => { + test('accumulates reasoning-delta separately from text', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'reasoning-delta', text: 'Thinking...' }, + { type: 'text-delta', text: 'Answer' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + // Text should only contain non-reasoning content + expect(result.text).toBe('Answer'); + }); + + test('includes reasoning in llm:response event', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'reasoning-delta', text: 'Let me think...' }, + { type: 'reasoning-delta', text: ' about this.' }, + { type: 'text-delta', text: 'Here is my answer' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + const responseEvent = mocks.emittedEvents.find((e) => e.name === 'llm:response'); + expect(responseEvent).toBeDefined(); + expect((responseEvent?.payload as { reasoning: string }).reasoning).toBe( + 'Let me think... about this.' + ); + }); + + test('emits reasoning chunks when streaming=true', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'reasoning-delta', text: 'Thinking...' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + const chunkEvents = mocks.emittedEvents.filter((e) => e.name === 'llm:chunk'); + expect(chunkEvents).toHaveLength(1); + expect((chunkEvents[0]?.payload as { chunkType: string }).chunkType).toBe('reasoning'); + }); + }); + + describe('Tool Call Handling', () => { + test('creates assistant message if not exists', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'test_tool', + input: { arg: 'value' }, + }, + { + type: 'finish', + finishReason: 'tool-calls', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + expect(mocks.contextManager.addAssistantMessage).toHaveBeenCalled(); + }); + + test('records tool call to context', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'test_tool', + input: { arg: 'value' }, + }, + { + type: 'finish', + finishReason: 'tool-calls', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + expect(mocks.contextManager.addToolCall).toHaveBeenCalledWith('msg-1', { + id: 'call-1', + type: 'function', + function: { + name: 'test_tool', + arguments: JSON.stringify({ arg: 'value' }), + }, + }); + }); + + test('does not persist providerMetadata for OpenAI tool calls (avoids OpenAI Responses replay issues)', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'test_tool', + input: { arg: 'value' }, + providerMetadata: { openai: { some: 'metadata' } }, + }, + { + type: 'finish', + finishReason: 'tool-calls', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + expect(mocks.contextManager.addToolCall).toHaveBeenCalledWith('msg-1', { + id: 'call-1', + type: 'function', + function: { + name: 'test_tool', + arguments: JSON.stringify({ arg: 'value' }), + }, + }); + }); + + test('persists providerMetadata for Google tool calls (required for round-tripping thought signatures)', async () => { + const mocks = createMocks(); + const config: StreamProcessorConfig = { ...mocks.config, provider: 'google' }; + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + config, + mocks.logger, + true + ); + + const events = [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'test_tool', + input: { arg: 'value' }, + providerMetadata: { google: { thoughtSignature: 'sig-1' } }, + }, + { + type: 'finish', + finishReason: 'tool-calls', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + expect(mocks.contextManager.addToolCall).toHaveBeenCalledWith('msg-1', { + id: 'call-1', + type: 'function', + function: { + name: 'test_tool', + arguments: JSON.stringify({ arg: 'value' }), + }, + providerOptions: { google: { thoughtSignature: 'sig-1' } }, + }); + }); + + test('persists tool call to context (llm:tool-call emitted by ToolManager)', async () => { + // NOTE: llm:tool-call is now emitted from ToolManager.executeTool() instead of StreamProcessor. + // This ensures correct event ordering - llm:tool-call arrives before approval:request. + // This test verifies StreamProcessor still persists tool calls to context. + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'test_tool', + input: { arg: 'value' }, + }, + { + type: 'finish', + finishReason: 'tool-calls', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + // Verify tool call was persisted to context + expect(mocks.contextManager.addToolCall).toHaveBeenCalledWith( + expect.any(String), // assistant message ID + { + id: 'call-1', + type: 'function', + function: { + name: 'test_tool', + arguments: JSON.stringify({ arg: 'value' }), + }, + } + ); + }); + }); + + describe('Tool Result Handling', () => { + test('persists sanitized and truncated tool result via contextManager', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'test_tool', + output: { result: 'raw output' }, + }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + // Verify addToolResult was called with sanitized result containing meta.success + expect(mocks.contextManager.addToolResult).toHaveBeenCalledWith( + 'call-1', + 'test_tool', + expect.objectContaining({ + content: expect.arrayContaining([expect.objectContaining({ type: 'text' })]), + meta: expect.objectContaining({ + success: true, // Success status is in sanitizedResult.meta + }), + }), + undefined // No approval metadata for this call + ); + }); + + test('emits llm:tool-result with success=true', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'test_tool', + output: { result: 'success' }, + }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + const toolResultEvent = mocks.emittedEvents.find((e) => e.name === 'llm:tool-result'); + expect(toolResultEvent).toBeDefined(); + expect((toolResultEvent?.payload as { success: boolean }).success).toBe(true); + expect((toolResultEvent?.payload as { toolName: string }).toolName).toBe('test_tool'); + expect((toolResultEvent?.payload as { callId: string }).callId).toBe('call-1'); + }); + + test('stores tool result with success status for rehydration', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { + type: 'tool-result', + toolCallId: 'call-rehydrate', + toolName: 'storage_tool', + output: { stored: true, id: 'doc-123' }, + }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + // Verify addToolResult is called with success in meta for storage/rehydration + expect(mocks.contextManager.addToolResult).toHaveBeenCalledWith( + 'call-rehydrate', + 'storage_tool', + expect.objectContaining({ + meta: expect.objectContaining({ + success: true, // Success status is in sanitizedResult.meta + }), + }), + undefined // No approval metadata for this call + ); + }); + + test('passes approval metadata separately from success status', async () => { + const mocks = createMocks(); + + // Create approval metadata to pass via constructor + const approvalMetadata = new Map< + string, + { requireApproval: boolean; approvalStatus?: 'approved' | 'rejected' } + >([['call-with-approval', { requireApproval: true, approvalStatus: 'approved' }]]); + + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true, + approvalMetadata + ); + + const events = [ + { + type: 'tool-result', + toolCallId: 'call-with-approval', + toolName: 'approved_tool', + output: { result: 'approved execution' }, + }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + // Verify success is in meta, approval metadata passed separately + expect(mocks.contextManager.addToolResult).toHaveBeenCalledWith( + 'call-with-approval', + 'approved_tool', + expect.objectContaining({ + meta: expect.objectContaining({ + success: true, // Success status is in sanitizedResult.meta + }), + }), + expect.objectContaining({ + requireApproval: true, + approvalStatus: 'approved', + }) + ); + }); + }); + + describe('Finish Event Handling', () => { + test('captures finishReason', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Done' }, + { + type: 'finish', + finishReason: 'length', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + expect(result.finishReason).toBe('length'); + }); + + test('captures token usage including reasoning tokens', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Response' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { + inputTokens: 100, + outputTokens: 50, + totalTokens: 150, + reasoningTokens: 20, + }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + expect(result.usage).toEqual({ + inputTokens: 100, + outputTokens: 50, + totalTokens: 150, + reasoningTokens: 20, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }); + }); + + test('subtracts cached input tokens from inputTokens when cachedInputTokens is present', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Response' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { + inputTokens: 1000, + outputTokens: 50, + totalTokens: 1050, + cachedInputTokens: 900, + }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + expect(result.usage).toEqual({ + inputTokens: 100, + outputTokens: 50, + totalTokens: 1050, + cacheReadTokens: 900, + cacheWriteTokens: 0, + }); + }); + + test('avoids double-counting cache write tokens when only cachedInputTokens are present', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Response' }, + { + type: 'finish', + finishReason: 'stop', + providerMetadata: { + anthropic: { + cacheCreationInputTokens: 100, + cacheReadInputTokens: 900, + }, + }, + totalUsage: { + inputTokens: 1100, + outputTokens: 50, + totalTokens: 1150, + cachedInputTokens: 900, + }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + expect(result.usage).toEqual({ + inputTokens: 100, + outputTokens: 50, + totalTokens: 1150, + cacheReadTokens: 900, + cacheWriteTokens: 100, + }); + }); + + test('updates assistant message with usage', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Response' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + expect(mocks.contextManager.updateAssistantMessage).toHaveBeenCalledWith('msg-1', { + tokenUsage: { + inputTokens: 100, + outputTokens: 50, + totalTokens: 150, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + }); + }); + + test('persists reasoning text to assistant message', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'reasoning-delta', text: 'Let me think...' }, + { type: 'reasoning-delta', text: ' about this carefully.' }, + { type: 'text-delta', text: 'Here is my answer' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + // Verify reasoning is persisted to the assistant message + expect(mocks.contextManager.updateAssistantMessage).toHaveBeenCalledWith( + 'msg-1', + expect.objectContaining({ + reasoning: 'Let me think... about this carefully.', + }) + ); + }); + + test('persists reasoning metadata (providerMetadata) to assistant message', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const providerMetadata = { openai: { itemId: 'item-123' } }; + const events = [ + { type: 'reasoning-delta', text: 'Thinking...', providerMetadata }, + { type: 'text-delta', text: 'Answer' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + // Verify reasoning metadata is persisted + expect(mocks.contextManager.updateAssistantMessage).toHaveBeenCalledWith( + 'msg-1', + expect.objectContaining({ + reasoning: 'Thinking...', + reasoningMetadata: providerMetadata, + }) + ); + }); + + test('emits llm:response with content, usage, provider, model', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Final response' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + const responseEvent = mocks.emittedEvents.find((e) => e.name === 'llm:response'); + expect(responseEvent).toBeDefined(); + expect(responseEvent?.payload).toMatchObject({ + content: 'Final response', + provider: 'openai', + model: 'gpt-4', + tokenUsage: { + inputTokens: 100, + outputTokens: 50, + totalTokens: 150, + }, + }); + }); + }); + + describe('Error Event Handling', () => { + test('emits llm:error with Error object', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const testError = new Error('Test error'); + const events = [{ type: 'error', error: testError }]; + + await processor.process(() => createMockStream(events) as never); + + const errorEvent = mocks.emittedEvents.find((e) => e.name === 'llm:error'); + expect(errorEvent).toBeDefined(); + expect((errorEvent?.payload as { error: Error }).error).toBe(testError); + }); + + test('wraps non-Error in Error', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [{ type: 'error', error: 'String error message' }]; + + await processor.process(() => createMockStream(events) as never); + + const errorEvent = mocks.emittedEvents.find((e) => e.name === 'llm:error'); + expect(errorEvent).toBeDefined(); + expect((errorEvent?.payload as { error: Error }).error).toBeInstanceOf(Error); + expect((errorEvent?.payload as { error: Error }).error.message).toBe( + 'String error message' + ); + }); + }); + + describe('Abort Signal', () => { + test('completes with accumulated content when abort signal fires mid-stream', async () => { + const mocks = createMocks(); + + // Create a stream that will yield events then abort + const slowStream = { + fullStream: (async function* () { + yield { type: 'text-delta', text: 'Hello' }; + // Abort signal fires but stream continues (SDK behavior) + mocks.abortController.abort(); + yield { type: 'text-delta', text: ' world' }; + })(), + }; + + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + // Resolves with accumulated content + const result = await processor.process(() => slowStream as never); + expect(result.text).toBe('Hello world'); + }); + + test('emits partial response with cancelled finish reason when stream emits abort event', async () => { + const mocks = createMocks(); + + // Create a stream that emits an abort event (SDK-level cancellation) + const abortingStream = { + fullStream: (async function* () { + yield { type: 'text-delta', text: 'Hello' }; + // Stream itself signals abort (e.g., network disconnect, timeout) + yield { type: 'abort' }; + })(), + }; + + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + // Emits partial response with cancelled finish reason + const result = await processor.process(() => abortingStream as never); + expect(result.text).toBe('Hello'); + expect(result.finishReason).toBe('cancelled'); + }); + }); + + describe('Edge Cases', () => { + test('handles empty stream (only finish event)', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 0, totalTokens: 10 }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + expect(result.text).toBe(''); + expect(result.finishReason).toBe('stop'); + }); + + test('handles multiple tool calls in sequence', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'tool_a', + input: { a: 1 }, + }, + { + type: 'tool-call', + toolCallId: 'call-2', + toolName: 'tool_b', + input: { b: 2 }, + }, + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'tool_a', + output: 'result a', + }, + { + type: 'tool-result', + toolCallId: 'call-2', + toolName: 'tool_b', + output: 'result b', + }, + { + type: 'finish', + finishReason: 'tool-calls', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + await processor.process(() => createMockStream(events) as never); + + // NOTE: llm:tool-call is now emitted from ToolManager.executeTool() instead of StreamProcessor. + // StreamProcessor still emits llm:tool-result events. + const toolResultEvents = mocks.emittedEvents.filter( + (e) => e.name === 'llm:tool-result' + ); + + expect(toolResultEvents).toHaveLength(2); + + // Verify both tool calls were persisted to context + expect(mocks.contextManager.addToolCall).toHaveBeenCalledTimes(2); + }); + + test('handles interleaved text and tool calls', async () => { + const mocks = createMocks(); + const processor = new StreamProcessor( + mocks.contextManager, + mocks.eventBus, + mocks.resourceManager, + mocks.abortController.signal, + mocks.config, + mocks.logger, + true + ); + + const events = [ + { type: 'text-delta', text: 'Let me check ' }, + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'lookup', + input: {}, + }, + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'lookup', + output: 'found', + }, + { type: 'text-delta', text: 'the answer is 42' }, + { + type: 'finish', + finishReason: 'stop', + totalUsage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + ]; + + const result = await processor.process(() => createMockStream(events) as never); + + expect(result.text).toBe('Let me check the answer is 42'); + }); + }); +}); diff --git a/dexto/packages/core/src/llm/executor/stream-processor.ts b/dexto/packages/core/src/llm/executor/stream-processor.ts new file mode 100644 index 00000000..10da8267 --- /dev/null +++ b/dexto/packages/core/src/llm/executor/stream-processor.ts @@ -0,0 +1,647 @@ +import { StreamTextResult, ToolSet as VercelToolSet } from 'ai'; +import { ContextManager } from '../../context/manager.js'; +import { SessionEventBus, LLMFinishReason } from '../../events/index.js'; +import { ResourceManager } from '../../resources/index.js'; +import { truncateToolResult } from './tool-output-truncator.js'; +import { StreamProcessorResult } from './types.js'; +import { sanitizeToolResult } from '../../context/utils.js'; +import type { SanitizedToolResult } from '../../context/types.js'; +import { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { LLMProvider, TokenUsage } from '../types.js'; + +type UsageLike = { + inputTokens?: number | undefined; + outputTokens?: number | undefined; + totalTokens?: number | undefined; + reasoningTokens?: number | undefined; + cachedInputTokens?: number | undefined; + inputTokenDetails?: { + noCacheTokens?: number | undefined; + cacheReadTokens?: number | undefined; + cacheWriteTokens?: number | undefined; + }; +}; + +export interface StreamProcessorConfig { + provider: LLMProvider; + model: string; + /** Estimated input tokens before LLM call (for analytics/calibration) */ + estimatedInputTokens?: number; +} + +export class StreamProcessor { + private assistantMessageId: string | null = null; + private actualTokens: TokenUsage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; + private finishReason: LLMFinishReason = 'unknown'; + private reasoningText: string = ''; + private reasoningMetadata: Record | undefined; + private accumulatedText: string = ''; + private logger: IDextoLogger; + private hasStepUsage = false; + /** + * Track pending tool calls (added to context but no result yet). + * On cancel/abort, we add synthetic "cancelled" results to maintain tool_use/tool_result pairing. + */ + private pendingToolCalls: Map = new Map(); + + /** + * @param contextManager Context manager for message persistence + * @param eventBus Event bus for emitting events + * @param resourceManager Resource manager for blob storage + * @param abortSignal Abort signal for cancellation + * @param config Provider/model configuration + * @param logger Logger instance + * @param streaming If true, emits llm:chunk events. Default true. + * @param approvalMetadata Map of tool call IDs to approval metadata + */ + constructor( + private contextManager: ContextManager, + private eventBus: SessionEventBus, + private resourceManager: ResourceManager, + private abortSignal: AbortSignal, + private config: StreamProcessorConfig, + logger: IDextoLogger, + private streaming: boolean = true, + private approvalMetadata?: Map< + string, + { requireApproval: boolean; approvalStatus?: 'approved' | 'rejected' } + > + ) { + this.logger = logger.createChild(DextoLogComponent.EXECUTOR); + } + + async process( + streamFn: () => StreamTextResult + ): Promise { + const stream = streamFn(); + + try { + for await (const event of stream.fullStream) { + // Don't call throwIfAborted() here - let Vercel SDK handle abort gracefully + // and emit 'abort' event which we handle below in the switch + + switch (event.type) { + case 'text-delta': + if (!this.assistantMessageId) { + // Create assistant message on first text delta if not exists + this.assistantMessageId = await this.contextManager + .addAssistantMessage('', [], {}) + .then(() => { + return this.getLastMessageId(); + }); + } + + await this.contextManager.appendAssistantText( + this.assistantMessageId!, + event.text + ); + + // Accumulate text for return value + this.accumulatedText += event.text; + + // Only emit chunks in streaming mode + if (this.streaming) { + this.eventBus.emit('llm:chunk', { + chunkType: 'text', + content: event.text, + }); + } + break; + + case 'reasoning-delta': + // Handle reasoning delta (extended thinking from Claude, etc.) + this.reasoningText += event.text; + + // Capture provider metadata for round-tripping (e.g., OpenAI itemId, Gemini thought signatures) + // This must be passed back to the provider on subsequent requests + if (event.providerMetadata) { + this.reasoningMetadata = event.providerMetadata; + } + + // Only emit chunks in streaming mode + if (this.streaming) { + this.eventBus.emit('llm:chunk', { + chunkType: 'reasoning', + content: event.text, + }); + } + break; + + case 'tool-call': { + // Create tool call record + if (!this.assistantMessageId) { + this.assistantMessageId = await this.createAssistantMessage(); + } + + // Extract providerMetadata for round-tripping (e.g., Gemini 3 thought signatures) + // These are opaque tokens that must be passed back to maintain model state + const toolCall: Parameters[1] = { + id: event.toolCallId, + type: 'function', + function: { + name: event.toolName, + arguments: JSON.stringify(event.input), + }, + }; + // IMPORTANT: Only persist providerMetadata for providers that require round-tripping + // (e.g., Gemini thought signatures). OpenAI Responses metadata can cause invalid + // follow-up requests (function_call item references missing required reasoning items). + const shouldPersistProviderMetadata = + this.config.provider === 'google' || + (this.config.provider as string) === 'vertex'; + + if (shouldPersistProviderMetadata && event.providerMetadata) { + toolCall.providerOptions = { + ...event.providerMetadata, + } as Record; + } + + await this.contextManager.addToolCall(this.assistantMessageId!, toolCall); + + // Track pending tool call for abort handling + this.pendingToolCalls.set(event.toolCallId, { + toolName: event.toolName, + }); + + // NOTE: llm:tool-call is now emitted from ToolManager.executeTool() instead. + // This ensures correct event ordering - llm:tool-call arrives before approval:request. + // See tool-manager.ts for detailed explanation of the timing issue. + break; + } + + case 'tool-result': { + // PERSISTENCE HAPPENS HERE + const rawResult = event.output; + + // Log raw tool output for debugging + this.logger.debug('Tool result received', { + toolName: event.toolName, + toolCallId: event.toolCallId, + rawResult, + }); + + // Sanitize + const sanitized = await sanitizeToolResult( + rawResult, + { + blobStore: this.resourceManager.getBlobStore(), + toolName: event.toolName, + toolCallId: event.toolCallId, + success: true, + }, + this.logger + ); + + // Truncate + const truncated = truncateToolResult(sanitized); + + // Get approval metadata for this tool call + const approval = this.approvalMetadata?.get(event.toolCallId); + + // Persist to history (success status comes from truncated.meta.success) + await this.contextManager.addToolResult( + event.toolCallId, + event.toolName, + truncated, // Includes meta.success from sanitization + approval // Only approval metadata if present + ); + + this.eventBus.emit('llm:tool-result', { + toolName: event.toolName, + callId: event.toolCallId, + success: true, + sanitized: truncated, + rawResult: rawResult, + ...(approval?.requireApproval !== undefined && { + requireApproval: approval.requireApproval, + }), + ...(approval?.approvalStatus !== undefined && { + approvalStatus: approval.approvalStatus, + }), + }); + + // Clean up approval metadata after use + this.approvalMetadata?.delete(event.toolCallId); + // Remove from pending (tool completed successfully) + this.pendingToolCalls.delete(event.toolCallId); + break; + } + + case 'finish-step': + // Track token usage from completed steps for partial runs + // TODO: Token usage for cancelled mid-step responses is unavailable. + // LLM providers only send token counts in their final response chunk. + // If we abort mid-stream, that chunk never arrives. The tokens are + // still billed by the provider, but we can't report them. + if (event.usage) { + const providerMetadata = this.getProviderMetadata(event); + const stepUsage = this.normalizeUsage(event.usage, providerMetadata); + + // Accumulate usage across steps + this.actualTokens = { + inputTokens: + (this.actualTokens.inputTokens ?? 0) + + (stepUsage.inputTokens ?? 0), + outputTokens: + (this.actualTokens.outputTokens ?? 0) + + (stepUsage.outputTokens ?? 0), + totalTokens: + (this.actualTokens.totalTokens ?? 0) + + (stepUsage.totalTokens ?? 0), + ...(stepUsage.reasoningTokens !== undefined && { + reasoningTokens: + (this.actualTokens.reasoningTokens ?? 0) + + stepUsage.reasoningTokens, + }), + // Cache tokens + cacheReadTokens: + (this.actualTokens.cacheReadTokens ?? 0) + + (stepUsage.cacheReadTokens ?? 0), + cacheWriteTokens: + (this.actualTokens.cacheWriteTokens ?? 0) + + (stepUsage.cacheWriteTokens ?? 0), + }; + this.hasStepUsage = true; + } + break; + + case 'finish': { + this.finishReason = event.finishReason; + + const providerMetadata = this.getProviderMetadata(event); + const fallbackUsage = this.normalizeUsage( + event.totalUsage, + providerMetadata + ); + const usage = this.hasStepUsage ? { ...this.actualTokens } : fallbackUsage; + + // Backfill usage fields from fallback when step usage reported zeros/undefined. + // This handles edge cases where providers send partial usage in finish-step + // events but complete usage in the final finish event (e.g., Anthropic sends + // cache tokens in providerMetadata rather than usage object). + if (this.hasStepUsage) { + // Backfill input/output tokens if step usage was zero but fallback has values. + // This is defensive - most providers report these consistently, but we log + // when backfill occurs to detect any providers with this edge case. + const fallbackInput = fallbackUsage.inputTokens ?? 0; + if ((usage.inputTokens ?? 0) === 0 && fallbackInput > 0) { + this.logger.debug( + 'Backfilling inputTokens from fallback usage (step reported 0)', + { stepValue: usage.inputTokens, fallbackValue: fallbackInput } + ); + usage.inputTokens = fallbackInput; + } + const fallbackOutput = fallbackUsage.outputTokens ?? 0; + if ((usage.outputTokens ?? 0) === 0 && fallbackOutput > 0) { + this.logger.debug( + 'Backfilling outputTokens from fallback usage (step reported 0)', + { stepValue: usage.outputTokens, fallbackValue: fallbackOutput } + ); + usage.outputTokens = fallbackOutput; + } + const fallbackCacheRead = fallbackUsage.cacheReadTokens ?? 0; + if ((usage.cacheReadTokens ?? 0) === 0 && fallbackCacheRead > 0) { + usage.cacheReadTokens = fallbackCacheRead; + } + const fallbackCacheWrite = fallbackUsage.cacheWriteTokens ?? 0; + if ((usage.cacheWriteTokens ?? 0) === 0 && fallbackCacheWrite > 0) { + usage.cacheWriteTokens = fallbackCacheWrite; + } + const fallbackTotalTokens = fallbackUsage.totalTokens ?? 0; + if ((usage.totalTokens ?? 0) === 0 && fallbackTotalTokens > 0) { + usage.totalTokens = fallbackTotalTokens; + } + if ( + usage.reasoningTokens === undefined && + fallbackUsage.reasoningTokens !== undefined + ) { + usage.reasoningTokens = fallbackUsage.reasoningTokens; + } + } + + this.actualTokens = usage; + + // Log complete LLM response for debugging + this.logger.info('LLM response complete', { + finishReason: event.finishReason, + contentLength: this.accumulatedText.length, + content: this.accumulatedText, + ...(this.reasoningText && { + reasoningLength: this.reasoningText.length, + reasoning: this.reasoningText, + }), + usage, + provider: this.config.provider, + model: this.config.model, + }); + + // Finalize assistant message with usage in reasoning + if (this.assistantMessageId) { + await this.contextManager.updateAssistantMessage( + this.assistantMessageId, + { + tokenUsage: usage, + // Persist reasoning text and metadata for round-tripping + ...(this.reasoningText && { reasoning: this.reasoningText }), + ...(this.reasoningMetadata && { + reasoningMetadata: this.reasoningMetadata, + }), + } + ); + } + + // Skip empty responses when tools are being called + // The meaningful response will come after tool execution completes + const hasContent = this.accumulatedText || this.reasoningText; + if (this.finishReason !== 'tool-calls' || hasContent) { + this.eventBus.emit('llm:response', { + content: this.accumulatedText, + ...(this.reasoningText && { reasoning: this.reasoningText }), + provider: this.config.provider, + model: this.config.model, + tokenUsage: usage, + ...(this.config.estimatedInputTokens !== undefined && { + estimatedInputTokens: this.config.estimatedInputTokens, + }), + finishReason: this.finishReason, + }); + } + break; + } + + case 'tool-error': { + // Tool execution failed - emit error event with tool context + this.logger.error('Tool execution failed', { + toolName: event.toolName, + toolCallId: event.toolCallId, + error: event.error, + }); + + const errorMessage = + event.error instanceof Error + ? event.error.message + : String(event.error); + + // CRITICAL: Must persist error result to history to maintain tool_use/tool_result pairing + // Without this, the conversation history has tool_use without tool_result, + // causing "tool_use ids were found without tool_result blocks" API errors + const errorResult: SanitizedToolResult = { + content: [{ type: 'text', text: `Error: ${errorMessage}` }], + meta: { + toolName: event.toolName, + toolCallId: event.toolCallId, + success: false, + }, + }; + + await this.contextManager.addToolResult( + event.toolCallId, + event.toolName, + errorResult, + undefined // No approval metadata for errors + ); + + this.eventBus.emit('llm:tool-result', { + toolName: event.toolName, + callId: event.toolCallId, + success: false, + error: errorMessage, + }); + + this.eventBus.emit('llm:error', { + error: + event.error instanceof Error + ? event.error + : new Error(String(event.error)), + context: `Tool execution failed: ${event.toolName}`, + toolCallId: event.toolCallId, + recoverable: true, // Tool errors are typically recoverable + }); + // Remove from pending (tool failed but result was persisted) + this.pendingToolCalls.delete(event.toolCallId); + break; + } + + case 'error': { + const err = + event.error instanceof Error + ? event.error + : new Error(String(event.error)); + this.logger.error(`LLM error: ${err.toString()}}`); + this.eventBus.emit('llm:error', { + error: err, + }); + break; + } + + case 'abort': + // Vercel SDK emits 'abort' when the stream is cancelled + this.logger.debug('Stream aborted, emitting partial response'); + this.finishReason = 'cancelled'; + + // Persist cancelled results for any pending tool calls + await this.persistCancelledToolResults(); + + this.eventBus.emit('llm:response', { + content: this.accumulatedText, + ...(this.reasoningText && { reasoning: this.reasoningText }), + provider: this.config.provider, + model: this.config.model, + tokenUsage: this.actualTokens, + ...(this.config.estimatedInputTokens !== undefined && { + estimatedInputTokens: this.config.estimatedInputTokens, + }), + finishReason: 'cancelled', + }); + + // Return immediately - stream will close after abort event + return { + text: this.accumulatedText, + finishReason: 'cancelled', + usage: this.actualTokens, + }; + } + } + } catch (error) { + // Check if this is an abort error (intentional cancellation) + // Note: DOMException extends Error in Node.js 17+, so the first check covers it + const isAbortError = + (error instanceof Error && error.name === 'AbortError') || this.abortSignal.aborted; + + if (isAbortError) { + // Emit final response with accumulated content on cancellation + this.logger.debug('Stream cancelled, emitting partial response'); + this.finishReason = 'cancelled'; + + // Persist cancelled results for any pending tool calls + await this.persistCancelledToolResults(); + + this.eventBus.emit('llm:response', { + content: this.accumulatedText, + ...(this.reasoningText && { reasoning: this.reasoningText }), + provider: this.config.provider, + model: this.config.model, + tokenUsage: this.actualTokens, + ...(this.config.estimatedInputTokens !== undefined && { + estimatedInputTokens: this.config.estimatedInputTokens, + }), + finishReason: 'cancelled', + }); + + // Don't throw - cancellation is intentional, not an error + return { + text: this.accumulatedText, + finishReason: 'cancelled', + usage: this.actualTokens, + }; + } + + // Non-abort errors are real failures + this.logger.error('Stream processing failed', { error }); + + // Emit error event so UI knows about the failure + this.eventBus.emit('llm:error', { + error: error instanceof Error ? error : new Error(String(error)), + context: 'StreamProcessor', + recoverable: false, + }); + + throw error; + } + + return { + text: this.accumulatedText, + finishReason: this.finishReason, + usage: this.actualTokens, + }; + } + + private getCacheTokensFromProviderMetadata( + providerMetadata: Record | undefined + ): { cacheReadTokens: number; cacheWriteTokens: number } { + const anthropicMeta = providerMetadata?.['anthropic'] as Record | undefined; + const bedrockMeta = providerMetadata?.['bedrock'] as + | { usage?: Record } + | undefined; + + const cacheWriteTokens = + anthropicMeta?.['cacheCreationInputTokens'] ?? + bedrockMeta?.usage?.['cacheWriteInputTokens'] ?? + 0; + const cacheReadTokens = + anthropicMeta?.['cacheReadInputTokens'] ?? + bedrockMeta?.usage?.['cacheReadInputTokens'] ?? + 0; + + return { cacheReadTokens, cacheWriteTokens }; + } + + private normalizeUsage( + usage: UsageLike | undefined, + providerMetadata?: Record + ): TokenUsage { + const inputTokensRaw = usage?.inputTokens ?? 0; + const outputTokens = usage?.outputTokens ?? 0; + const totalTokens = usage?.totalTokens ?? 0; + const reasoningTokens = usage?.reasoningTokens; + const cachedInputTokens = usage?.cachedInputTokens; + const inputTokenDetails = usage?.inputTokenDetails; + + const providerCache = this.getCacheTokensFromProviderMetadata(providerMetadata); + const cacheReadTokens = + inputTokenDetails?.cacheReadTokens ?? + cachedInputTokens ?? + providerCache.cacheReadTokens ?? + 0; + const cacheWriteTokens = + inputTokenDetails?.cacheWriteTokens ?? providerCache.cacheWriteTokens ?? 0; + + const needsCacheWriteAdjustment = + inputTokenDetails === undefined && + cachedInputTokens !== undefined && + providerCache.cacheWriteTokens > 0; + const noCacheTokens = + inputTokenDetails?.noCacheTokens ?? + (cachedInputTokens !== undefined + ? inputTokensRaw - + cachedInputTokens - + (needsCacheWriteAdjustment ? providerCache.cacheWriteTokens : 0) + : inputTokensRaw); + + return { + inputTokens: Math.max(0, noCacheTokens), + outputTokens, + totalTokens, + ...(reasoningTokens !== undefined && { reasoningTokens }), + cacheReadTokens, + cacheWriteTokens, + }; + } + + private getProviderMetadata( + event: Record + ): Record | undefined { + const metadata = + 'providerMetadata' in event + ? (event as { providerMetadata?: Record }).providerMetadata + : undefined; + if (!metadata || typeof metadata !== 'object') { + return undefined; + } + return metadata; + } + + private async createAssistantMessage(): Promise { + await this.contextManager.addAssistantMessage('', [], {}); + return this.getLastMessageId(); + } + + private async getLastMessageId(): Promise { + const history = await this.contextManager.getHistory(); + const last = history[history.length - 1]; + if (!last || !last.id) throw new Error('Failed to get last message ID'); + return last.id; + } + + /** + * Persist synthetic "cancelled" results for all pending tool calls. + * This maintains the tool_use/tool_result pairing required by LLM APIs. + * Called on abort/cancel to prevent "tool_use ids were found without tool_result" errors. + */ + private async persistCancelledToolResults(): Promise { + if (this.pendingToolCalls.size === 0) return; + + this.logger.debug( + `Persisting cancelled results for ${this.pendingToolCalls.size} pending tool call(s)` + ); + + for (const [toolCallId, { toolName }] of this.pendingToolCalls) { + const cancelledResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'Cancelled by user' }], + meta: { + toolName, + toolCallId, + success: false, + }, + }; + + await this.contextManager.addToolResult( + toolCallId, + toolName, + cancelledResult, + undefined // No approval metadata for cancelled tools + ); + + // Emit tool-result event so CLI/WebUI can update UI + this.eventBus.emit('llm:tool-result', { + toolName, + callId: toolCallId, + success: false, + error: 'Cancelled by user', + }); + } + + this.pendingToolCalls.clear(); + } +} diff --git a/dexto/packages/core/src/llm/executor/tool-output-truncator.test.ts b/dexto/packages/core/src/llm/executor/tool-output-truncator.test.ts new file mode 100644 index 00000000..ee457ea8 --- /dev/null +++ b/dexto/packages/core/src/llm/executor/tool-output-truncator.test.ts @@ -0,0 +1,76 @@ +import { describe, it, expect } from 'vitest'; +import { truncateStringOutput, truncateToolResult } from './tool-output-truncator.js'; +import { SanitizedToolResult } from '../../context/types.js'; + +describe('tool-output-truncator', () => { + describe('truncateStringOutput', () => { + it('should not truncate if output is within limit', () => { + const output = 'short output'; + const result = truncateStringOutput(output, { maxChars: 100 }); + expect(result.truncated).toBe(false); + expect(result.output).toBe(output); + expect(result.originalLength).toBe(output.length); + }); + + it('should truncate if output exceeds limit', () => { + const output = 'long output that exceeds the limit'; + const maxChars = 10; + const result = truncateStringOutput(output, { maxChars }); + expect(result.truncated).toBe(true); + expect(result.output).toContain('[Output truncated'); + expect(result.output.startsWith('long outpu')).toBe(true); + expect(result.originalLength).toBe(output.length); + }); + + it('should use default limit if not provided', () => { + // Use input significantly larger than default limit (120,000) + // so the truncation saves more chars than the appended message adds + const output = 'a'.repeat(150000); + const result = truncateStringOutput(output); + expect(result.truncated).toBe(true); + // Output should be ~120,000 + truncation message (~104 chars) = ~120,104 + // which is less than original 150,000 + expect(result.output.length).toBeLessThan(output.length); + expect(result.output).toContain('[Output truncated'); + }); + }); + + describe('truncateToolResult', () => { + it('should truncate text parts in SanitizedToolResult', () => { + const longText = 'a'.repeat(200); + const toolResult: SanitizedToolResult = { + content: [ + { type: 'text', text: longText }, + { type: 'image', image: 'data:image/png;base64,...' }, + ], + meta: { toolName: 'test', toolCallId: '123', success: true }, + }; + + const result = truncateToolResult(toolResult, { maxChars: 100 }); + + const firstPart = result.content[0]; + expect(firstPart).toBeDefined(); + if (firstPart) { + expect(firstPart.type).toBe('text'); + } + + if (firstPart && firstPart.type === 'text') { + expect(firstPart.text).toContain('[Output truncated'); + expect(firstPart.text.length).toBeLessThan(longText.length); + } + + // Should preserve other parts + expect(result.content[1]).toEqual(toolResult.content[1]); + }); + + it('should not modify result if no truncation needed', () => { + const toolResult: SanitizedToolResult = { + content: [{ type: 'text', text: 'short' }], + meta: { toolName: 'test', toolCallId: '123', success: true }, + }; + + const result = truncateToolResult(toolResult, { maxChars: 100 }); + expect(result).toEqual(toolResult); + }); + }); +}); diff --git a/dexto/packages/core/src/llm/executor/tool-output-truncator.ts b/dexto/packages/core/src/llm/executor/tool-output-truncator.ts new file mode 100644 index 00000000..d557d87c --- /dev/null +++ b/dexto/packages/core/src/llm/executor/tool-output-truncator.ts @@ -0,0 +1,77 @@ +import { SanitizedToolResult } from '../../context/types.js'; + +// Constants - configurable per agent +export const DEFAULT_MAX_TOOL_OUTPUT_CHARS = 120_000; // ~30K tokens +export const DEFAULT_MAX_FILE_LINES = 2000; +export const DEFAULT_MAX_LINE_LENGTH = 2000; + +export interface TruncationOptions { + maxChars?: number; +} + +export interface TruncationResult { + output: string; + truncated: boolean; + originalLength: number; +} + +/** + * Truncates a string tool output to prevent context overflow. + * Appends a warning message if truncated. + */ +export function truncateStringOutput( + output: string, + options: TruncationOptions = {} +): TruncationResult { + const maxChars = options.maxChars ?? DEFAULT_MAX_TOOL_OUTPUT_CHARS; + + if (output.length <= maxChars) { + return { + output, + truncated: false, + originalLength: output.length, + }; + } + + const truncatedOutput = + output.slice(0, maxChars) + + `\n\n[Output truncated - exceeded maximum length of ${maxChars} characters. Total length was ${output.length} characters.]`; + + return { + output: truncatedOutput, + truncated: true, + originalLength: output.length, + }; +} + +/** + * Truncates a SanitizedToolResult. + * Currently only truncates text parts. + * + * @param result The sanitized tool result to truncate + * @param options Truncation options + * @returns The truncated result + */ +export function truncateToolResult( + result: SanitizedToolResult, + options: TruncationOptions = {} +): SanitizedToolResult { + const newContent = result.content.map((part) => { + if (part.type === 'text') { + const { output, truncated } = truncateStringOutput(part.text, options); + if (truncated) { + return { ...part, text: output }; + } + } + return part; + }); + + return { + ...result, + content: newContent, + meta: { + ...result.meta, + // We could add a flag here if we wanted to track truncation in metadata + }, + }; +} diff --git a/dexto/packages/core/src/llm/executor/turn-executor.integration.test.ts b/dexto/packages/core/src/llm/executor/turn-executor.integration.test.ts new file mode 100644 index 00000000..52229190 --- /dev/null +++ b/dexto/packages/core/src/llm/executor/turn-executor.integration.test.ts @@ -0,0 +1,901 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { TurnExecutor } from './turn-executor.js'; +import { ContextManager } from '../../context/manager.js'; +import { ToolManager } from '../../tools/tool-manager.js'; +import { SessionEventBus, AgentEventBus } from '../../events/index.js'; +import { ResourceManager } from '../../resources/index.js'; +import { MessageQueueService } from '../../session/message-queue.js'; +import { SystemPromptManager } from '../../systemPrompt/manager.js'; +import { VercelMessageFormatter } from '../formatters/vercel.js'; +import { MemoryHistoryProvider } from '../../session/history/memory.js'; +import { MCPManager } from '../../mcp/manager.js'; +import { ApprovalManager } from '../../approval/manager.js'; +import { createLogger } from '../../logger/factory.js'; +import { createStorageManager, StorageManager } from '../../storage/storage-manager.js'; +import { MemoryManager } from '../../memory/index.js'; +import { SystemPromptConfigSchema } from '../../systemPrompt/schemas.js'; +import type { LanguageModel, ModelMessage } from 'ai'; +import type { LLMContext } from '../types.js'; +import type { ValidatedLLMConfig } from '../schemas.js'; +import type { ValidatedStorageConfig } from '../../storage/schemas.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; + +// Only mock the AI SDK's streamText/generateText - everything else is real +vi.mock('ai', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + streamText: vi.fn(), + generateText: vi.fn(), + }; +}); + +vi.mock('@opentelemetry/api', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + trace: { + ...actual.trace, + getActiveSpan: vi.fn(() => null), + }, + }; +}); + +import { streamText, generateText } from 'ai'; + +/** + * Helper to create mock stream results that simulate Vercel AI SDK responses + */ +function createMockStream(options: { + text?: string; + finishReason?: string; + usage?: { inputTokens: number; outputTokens: number; totalTokens: number }; + providerMetadata?: Record; + toolCalls?: Array<{ toolCallId: string; toolName: string; args: Record }>; + reasoning?: string; +}) { + const events: Array<{ type: string; [key: string]: unknown }> = []; + + // Add reasoning delta if present + if (options.reasoning) { + for (const char of options.reasoning) { + events.push({ type: 'reasoning-delta', text: char }); + } + } + + // Add text delta events + if (options.text) { + for (const char of options.text) { + events.push({ type: 'text-delta', text: char }); + } + } + + // Add tool call events + if (options.toolCalls) { + for (const tc of options.toolCalls) { + events.push({ + type: 'tool-call', + toolCallId: tc.toolCallId, + toolName: tc.toolName, + args: tc.args, + }); + } + } + + // Add finish event + events.push({ + type: 'finish', + finishReason: options.finishReason ?? 'stop', + totalUsage: options.usage ?? { inputTokens: 100, outputTokens: 50, totalTokens: 150 }, + ...(options.providerMetadata && { providerMetadata: options.providerMetadata }), + }); + + return { + fullStream: (async function* () { + for (const event of events) { + yield event; + } + })(), + }; +} + +/** + * Creates a mock LanguageModel + */ +function createMockModel(): LanguageModel { + return { + modelId: 'test-model', + provider: 'test-provider', + specificationVersion: 'v1', + doStream: vi.fn(), + doGenerate: vi.fn(), + } as unknown as LanguageModel; +} + +describe('TurnExecutor Integration Tests', () => { + let executor: TurnExecutor; + let contextManager: ContextManager; + let toolManager: ToolManager; + let sessionEventBus: SessionEventBus; + let agentEventBus: AgentEventBus; + let resourceManager: ResourceManager; + let messageQueue: MessageQueueService; + let logger: IDextoLogger; + let historyProvider: MemoryHistoryProvider; + let mcpManager: MCPManager; + let approvalManager: ApprovalManager; + let storageManager: StorageManager; + + const sessionId = 'test-session'; + const llmContext: LLMContext = { provider: 'openai', model: 'gpt-4' }; + + beforeEach(async () => { + vi.clearAllMocks(); + + // Create real logger + logger = createLogger({ + config: { + level: 'warn', // Use warn to reduce noise in tests + transports: [{ type: 'console', colorize: false }], + }, + agentId: 'test-agent', + }); + + // Create real event buses + agentEventBus = new AgentEventBus(); + sessionEventBus = new SessionEventBus(); + + // Create real storage manager with in-memory backends + // Cast to ValidatedStorageConfig since we know test data is valid (avoids schema parsing overhead) + const storageConfig = { + cache: { type: 'in-memory' }, + database: { type: 'in-memory' }, + blob: { + type: 'in-memory', + maxBlobSize: 10 * 1024 * 1024, + maxTotalSize: 100 * 1024 * 1024, + }, + } as unknown as ValidatedStorageConfig; + storageManager = await createStorageManager(storageConfig, logger); + + // Create real MCP manager + mcpManager = new MCPManager(logger); + + // Create real resource manager with proper wiring + resourceManager = new ResourceManager( + mcpManager, + { + internalResourcesConfig: { enabled: false, resources: [] }, + blobStore: storageManager.getBlobStore(), + }, + logger + ); + await resourceManager.initialize(); + + // Create real history provider + historyProvider = new MemoryHistoryProvider(logger); + + // Create real memory manager and system prompt manager + const memoryManager = new MemoryManager(storageManager.getDatabase(), logger); + const systemPromptConfig = SystemPromptConfigSchema.parse('You are a helpful assistant.'); + const systemPromptManager = new SystemPromptManager( + systemPromptConfig, + '/tmp', // configDir - not used with inline prompts + memoryManager, + undefined, // memoriesConfig + logger + ); + + // Create real context manager with Vercel formatter + const formatter = new VercelMessageFormatter(logger); + // Cast to ValidatedLLMConfig since we know test data is valid + const llmConfig = { + provider: 'openai', + model: 'gpt-4', + apiKey: 'test-api-key', + maxInputTokens: 100000, + maxOutputTokens: 4096, + temperature: 0.7, + maxIterations: 10, + } as unknown as ValidatedLLMConfig; + + contextManager = new ContextManager( + llmConfig, + formatter, + systemPromptManager, + 100000, + historyProvider, + sessionId, + resourceManager, + logger + ); + + // Create real approval manager + approvalManager = new ApprovalManager( + { + toolConfirmation: { mode: 'auto-approve', timeout: 120000 }, + elicitation: { enabled: false, timeout: 120000 }, + }, + logger + ); + + // Create real tool manager (minimal setup - no internal tools) + const mockAllowedToolsProvider = { + isToolAllowed: vi.fn().mockResolvedValue(false), + allowTool: vi.fn(), + disallowTool: vi.fn(), + }; + + toolManager = new ToolManager( + mcpManager, + approvalManager, + mockAllowedToolsProvider, + 'auto-approve', + agentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsServices: {}, internalToolsConfig: [] }, + logger + ); + await toolManager.initialize(); + + // Create real message queue + messageQueue = new MessageQueueService(sessionEventBus, logger); + + // Default streamText mock - simple text response + vi.mocked(streamText).mockImplementation( + () => + createMockStream({ text: 'Hello!', finishReason: 'stop' }) as unknown as ReturnType< + typeof streamText + > + ); + + // Create executor with real components + executor = new TurnExecutor( + createMockModel(), + toolManager, + contextManager, + sessionEventBus, + resourceManager, + sessionId, + { maxSteps: 10, maxOutputTokens: 4096, temperature: 0.7 }, + llmContext, + logger, + messageQueue + ); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + logger.destroy(); + }); + + describe('Basic Execution Flow', () => { + it('should execute and return result with real context manager', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + const result = await executor.execute({ mcpManager }, true); + + expect(result.finishReason).toBe('stop'); + expect(result.text).toBe('Hello!'); + expect(result.usage).toEqual({ + inputTokens: 100, + outputTokens: 50, + totalTokens: 150, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }); + }); + + it('should persist assistant response to history', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + const history = await contextManager.getHistory(); + expect(history.length).toBeGreaterThanOrEqual(2); + + const assistantMessages = history.filter((m) => m.role === 'assistant'); + expect(assistantMessages.length).toBeGreaterThan(0); + }); + + it('should emit events through real event bus', async () => { + const thinkingHandler = vi.fn(); + const responseHandler = vi.fn(); + const runCompleteHandler = vi.fn(); + + sessionEventBus.on('llm:thinking', thinkingHandler); + sessionEventBus.on('llm:response', responseHandler); + sessionEventBus.on('run:complete', runCompleteHandler); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + expect(thinkingHandler).toHaveBeenCalled(); + expect(responseHandler).toHaveBeenCalled(); + expect(runCompleteHandler).toHaveBeenCalledWith( + expect.objectContaining({ + finishReason: 'stop', + stepCount: 0, + }) + ); + }); + + it('should emit chunk events when streaming', async () => { + const chunkHandler = vi.fn(); + sessionEventBus.on('llm:chunk', chunkHandler); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + expect(chunkHandler).toHaveBeenCalled(); + expect(chunkHandler.mock.calls.some((call) => call[0].chunkType === 'text')).toBe(true); + }); + + it('should not emit chunk events when not streaming', async () => { + const chunkHandler = vi.fn(); + sessionEventBus.on('llm:chunk', chunkHandler); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, false); + + expect(chunkHandler).not.toHaveBeenCalled(); + }); + }); + + describe('Multi-Step Tool Execution', () => { + it('should continue looping on tool-calls finish reason', async () => { + let callCount = 0; + vi.mocked(streamText).mockImplementation(() => { + callCount++; + if (callCount < 3) { + return createMockStream({ + text: `Step ${callCount}`, + finishReason: 'tool-calls', + toolCalls: [ + { toolCallId: `call-${callCount}`, toolName: 'test_tool', args: {} }, + ], + }) as unknown as ReturnType; + } + return createMockStream({ + text: 'Final response', + finishReason: 'stop', + }) as unknown as ReturnType; + }); + + await contextManager.addUserMessage([{ type: 'text', text: 'Do something' }]); + const result = await executor.execute({ mcpManager }, true); + + expect(result.finishReason).toBe('stop'); + expect(result.stepCount).toBe(2); + expect(callCount).toBe(3); + }); + + it('should stop at maxSteps limit', async () => { + vi.mocked(streamText).mockImplementation( + () => + createMockStream({ + text: 'Tool step', + finishReason: 'tool-calls', + toolCalls: [{ toolCallId: 'call-1', toolName: 'test', args: {} }], + }) as unknown as ReturnType + ); + + const limitedExecutor = new TurnExecutor( + createMockModel(), + toolManager, + contextManager, + sessionEventBus, + resourceManager, + sessionId, + { maxSteps: 3, maxOutputTokens: 4096, temperature: 0.7 }, + llmContext, + logger, + messageQueue + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'Keep going' }]); + const result = await limitedExecutor.execute({ mcpManager }, true); + + expect(result.finishReason).toBe('max-steps'); + expect(result.stepCount).toBe(3); + }); + }); + + describe('Message Queue Injection', () => { + it('should inject queued messages into context', async () => { + messageQueue.enqueue({ + content: [{ type: 'text', text: 'User guidance: focus on performance' }], + }); + + await contextManager.addUserMessage([{ type: 'text', text: 'Initial request' }]); + await executor.execute({ mcpManager }, true); + + const history = await contextManager.getHistory(); + const userMessages = history.filter((m) => m.role === 'user'); + expect(userMessages.length).toBe(2); + + const injectedMsg = userMessages.find((m) => { + const content = Array.isArray(m.content) ? m.content : []; + return content.some((p) => p.type === 'text' && p.text.includes('User guidance')); + }); + expect(injectedMsg).toBeDefined(); + }); + + it('should continue processing when queue has messages on termination', async () => { + let callCount = 0; + vi.mocked(streamText).mockImplementation(() => { + callCount++; + if (callCount === 1) { + messageQueue.enqueue({ + content: [{ type: 'text', text: 'Follow-up question' }], + }); + return createMockStream({ + text: 'First response', + finishReason: 'stop', + }) as unknown as ReturnType; + } + return createMockStream({ + text: 'Second response', + finishReason: 'stop', + }) as unknown as ReturnType; + }); + + await contextManager.addUserMessage([{ type: 'text', text: 'Initial' }]); + const result = await executor.execute({ mcpManager }, true); + + expect(callCount).toBe(2); + expect(result.text).toBe('Second response'); + }); + }); + + describe('Tool Support Validation', () => { + it('should skip validation for providers without baseURL', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + expect(generateText).not.toHaveBeenCalled(); + }); + + it('should validate and cache tool support for custom baseURL', async () => { + vi.mocked(generateText).mockResolvedValue( + {} as Awaited> + ); + + const executor1 = new TurnExecutor( + createMockModel(), + toolManager, + contextManager, + sessionEventBus, + resourceManager, + sessionId, + { maxSteps: 10, baseURL: 'https://custom.api.com' }, + llmContext, + logger, + messageQueue + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor1.execute({ mcpManager }, true); + + expect(generateText).toHaveBeenCalledTimes(1); + + // Second executor with same baseURL should use cache + const newMessageQueue = new MessageQueueService(sessionEventBus, logger); + const executor2 = new TurnExecutor( + createMockModel(), + toolManager, + contextManager, + sessionEventBus, + resourceManager, + 'session-2', + { maxSteps: 10, baseURL: 'https://custom.api.com' }, + llmContext, + logger, + newMessageQueue + ); + + await executor2.execute({ mcpManager }, true); + expect(generateText).toHaveBeenCalledTimes(1); + }); + + it('should use empty tools when model does not support them', async () => { + vi.mocked(generateText).mockRejectedValue(new Error('Model does not support tools')); + + const executorWithBaseURL = new TurnExecutor( + createMockModel(), + toolManager, + contextManager, + sessionEventBus, + resourceManager, + sessionId, + { maxSteps: 10, baseURL: 'https://no-tools.api.com' }, + llmContext, + logger, + messageQueue + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executorWithBaseURL.execute({ mcpManager }, true); + + expect(streamText).toHaveBeenCalledWith( + expect.objectContaining({ + tools: {}, + }) + ); + }); + + it('should validate tool support for local providers even without custom baseURL', async () => { + vi.mocked(generateText).mockRejectedValue(new Error('Model does not support tools')); + + const ollamaLlmContext = { + provider: 'ollama' as const, + model: 'gemma3n:e2b', + }; + + const ollamaExecutor = new TurnExecutor( + createMockModel(), + toolManager, + contextManager, + sessionEventBus, + resourceManager, + sessionId, + { maxSteps: 10 }, // No baseURL + ollamaLlmContext, + logger, + messageQueue + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await ollamaExecutor.execute({ mcpManager }, true); + + // Should call generateText for validation even without baseURL + expect(generateText).toHaveBeenCalledTimes(1); + + // Should use empty tools in actual execution + expect(streamText).toHaveBeenCalledWith( + expect.objectContaining({ + tools: {}, + }) + ); + }); + + it('should emit llm:unsupported-input warning when model does not support tools', async () => { + vi.mocked(generateText).mockRejectedValue(new Error('Model does not support tools')); + + const warningHandler = vi.fn(); + sessionEventBus.on('llm:unsupported-input', warningHandler); + + const executorWithBaseURL = new TurnExecutor( + createMockModel(), + toolManager, + contextManager, + sessionEventBus, + resourceManager, + sessionId, + { maxSteps: 10, baseURL: 'https://no-tools.api.com' }, + llmContext, + logger, + messageQueue + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executorWithBaseURL.execute({ mcpManager }, true); + + expect(warningHandler).toHaveBeenCalledWith( + expect.objectContaining({ + errors: expect.arrayContaining([ + expect.stringContaining('does not support tool calling'), + expect.stringContaining('You can still chat'), + ]), + provider: llmContext.provider, + model: llmContext.model, + details: expect.objectContaining({ + feature: 'tool-calling', + supported: false, + }), + }) + ); + }); + }); + + describe('Error Handling', () => { + it('should emit llm:error and run:complete on failure', async () => { + vi.mocked(streamText).mockImplementation(() => { + throw new Error('Stream failed'); + }); + + const errorHandler = vi.fn(); + const completeHandler = vi.fn(); + sessionEventBus.on('llm:error', errorHandler); + sessionEventBus.on('run:complete', completeHandler); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await expect(executor.execute({ mcpManager }, true)).rejects.toThrow(); + + expect(errorHandler).toHaveBeenCalledWith( + expect.objectContaining({ + context: 'TurnExecutor', + recoverable: false, + }) + ); + expect(completeHandler).toHaveBeenCalledWith( + expect.objectContaining({ + finishReason: 'error', + }) + ); + }); + + it('should map rate limit errors correctly', async () => { + const { APICallError } = await import('ai'); + + // Create a real APICallError instance + const rateLimitError = new APICallError({ + message: 'Rate limit exceeded', + statusCode: 429, + responseHeaders: { 'retry-after': '60' }, + responseBody: 'Rate limit exceeded', + url: 'https://api.openai.com/v1/chat/completions', + requestBodyValues: {}, + isRetryable: true, + }); + + vi.mocked(streamText).mockImplementation(() => { + throw rateLimitError; + }); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + + await expect(executor.execute({ mcpManager }, true)).rejects.toMatchObject({ + code: 'llm_rate_limit_exceeded', + type: 'rate_limit', + }); + }); + }); + + describe('Cleanup and Resource Management', () => { + it('should clear message queue on normal completion', async () => { + messageQueue.enqueue({ content: [{ type: 'text', text: 'Pending' }] }); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + expect(messageQueue.dequeueAll()).toBeNull(); + }); + + it('should clear message queue on error', async () => { + messageQueue.enqueue({ content: [{ type: 'text', text: 'Pending' }] }); + + vi.mocked(streamText).mockImplementation(() => { + throw new Error('Failed'); + }); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await expect(executor.execute({ mcpManager }, true)).rejects.toThrow(); + + expect(messageQueue.dequeueAll()).toBeNull(); + }); + }); + + describe('External Abort Signal', () => { + it('should handle external abort signal', async () => { + const abortController = new AbortController(); + let callCount = 0; + + vi.mocked(streamText).mockImplementation(() => { + callCount++; + if (callCount === 1) { + abortController.abort(); + return createMockStream({ + finishReason: 'tool-calls', + toolCalls: [{ toolCallId: 'call-1', toolName: 'test', args: {} }], + }) as unknown as ReturnType; + } + return createMockStream({ finishReason: 'stop' }) as unknown as ReturnType< + typeof streamText + >; + }); + + const executorWithSignal = new TurnExecutor( + createMockModel(), + toolManager, + contextManager, + sessionEventBus, + resourceManager, + sessionId, + { maxSteps: 10 }, + llmContext, + logger, + messageQueue, + undefined, + abortController.signal + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + const result = await executorWithSignal.execute({ mcpManager }, true); + + expect(result.finishReason).toBe('cancelled'); + }); + }); + + describe('Reasoning Token Support', () => { + it('should handle reasoning tokens in usage', async () => { + vi.mocked(streamText).mockImplementation( + () => + createMockStream({ + text: 'Response', + reasoning: 'Let me think...', + finishReason: 'stop', + usage: { + inputTokens: 100, + outputTokens: 50, + totalTokens: 170, + }, + }) as unknown as ReturnType + ); + + const responseHandler = vi.fn(); + sessionEventBus.on('llm:response', responseHandler); + + await contextManager.addUserMessage([{ type: 'text', text: 'Think about this' }]); + const result = await executor.execute({ mcpManager }, true); + + expect(result.usage).toMatchObject({ + inputTokens: 100, + outputTokens: 50, + totalTokens: 170, + }); + expect(responseHandler).toHaveBeenCalled(); + }); + }); + + describe('Context Formatting', () => { + it('should format messages correctly for LLM', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + expect(streamText).toHaveBeenCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: 'user', + }), + ]), + }) + ); + }); + + it('should include system prompt in formatted messages', async () => { + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + const call = vi.mocked(streamText).mock.calls[0]?.[0]; + expect(call).toBeDefined(); + expect(call?.messages).toBeDefined(); + + const messages = call?.messages as ModelMessage[]; + const hasSystemContent = messages.some( + (m) => + m.role === 'system' || + (m.role === 'user' && + Array.isArray(m.content) && + m.content.some( + (c) => + typeof c === 'object' && + 'text' in c && + c.text.includes('helpful assistant') + )) + ); + expect(hasSystemContent).toBe(true); + }); + }); + + describe('Context Token Tracking', () => { + it('should store actual input tokens from LLM response in ContextManager', async () => { + const expectedInputTokens = 1234; + + vi.mocked(streamText).mockImplementation( + () => + createMockStream({ + text: 'Response', + finishReason: 'stop', + usage: { + inputTokens: expectedInputTokens, + outputTokens: 50, + totalTokens: expectedInputTokens + 50, + }, + }) as unknown as ReturnType + ); + + // Before LLM call, should be null + expect(contextManager.getLastActualInputTokens()).toBeNull(); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + // After LLM call, should have the actual token count + expect(contextManager.getLastActualInputTokens()).toBe(expectedInputTokens); + }); + + it('should update actual tokens on each LLM call', async () => { + // First call + vi.mocked(streamText).mockImplementationOnce( + () => + createMockStream({ + text: 'First response', + finishReason: 'stop', + usage: { inputTokens: 100, outputTokens: 20, totalTokens: 120 }, + }) as unknown as ReturnType + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'First message' }]); + await executor.execute({ mcpManager }, true); + expect(contextManager.getLastActualInputTokens()).toBe(100); + + // Second call with different token count + vi.mocked(streamText).mockImplementationOnce( + () => + createMockStream({ + text: 'Second response', + finishReason: 'stop', + usage: { inputTokens: 250, outputTokens: 30, totalTokens: 280 }, + }) as unknown as ReturnType + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'Second message' }]); + await executor.execute({ mcpManager }, true); + expect(contextManager.getLastActualInputTokens()).toBe(250); + }); + + it('should make actual tokens available via getContextTokenEstimate', async () => { + const expectedInputTokens = 5000; + + vi.mocked(streamText).mockImplementation( + () => + createMockStream({ + text: 'Response', + finishReason: 'stop', + usage: { + inputTokens: expectedInputTokens, + outputTokens: 100, + totalTokens: expectedInputTokens + 100, + }, + }) as unknown as ReturnType + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + // getContextTokenEstimate should return the actual value + const estimate = await contextManager.getContextTokenEstimate({ mcpManager }, {}); + expect(estimate.actual).toBe(expectedInputTokens); + }); + + it('should include cached tokens in actual context input tracking', async () => { + const noCacheTokens = 200; + const cacheReadTokens = 800; + + vi.mocked(streamText).mockImplementation( + () => + createMockStream({ + text: 'Response', + finishReason: 'stop', + usage: { + inputTokens: noCacheTokens, + outputTokens: 10, + totalTokens: noCacheTokens + 10, + }, + providerMetadata: { + anthropic: { + cacheReadInputTokens: cacheReadTokens, + cacheCreationInputTokens: 0, + }, + }, + }) as unknown as ReturnType + ); + + await contextManager.addUserMessage([{ type: 'text', text: 'Hello' }]); + await executor.execute({ mcpManager }, true); + + expect(contextManager.getLastActualInputTokens()).toBe(noCacheTokens + cacheReadTokens); + }); + }); +}); diff --git a/dexto/packages/core/src/llm/executor/turn-executor.ts b/dexto/packages/core/src/llm/executor/turn-executor.ts new file mode 100644 index 00000000..a31675ad --- /dev/null +++ b/dexto/packages/core/src/llm/executor/turn-executor.ts @@ -0,0 +1,1193 @@ +import { + LanguageModel, + streamText, + generateText, + stepCountIs, + ToolSet as VercelToolSet, + jsonSchema, + type ModelMessage, + APICallError, +} from 'ai'; +import { trace } from '@opentelemetry/api'; +import { ContextManager } from '../../context/manager.js'; +import type { TextPart, ImagePart, FilePart, UIResourcePart } from '../../context/types.js'; +import { ToolManager } from '../../tools/tool-manager.js'; +import { ToolSet } from '../../tools/types.js'; +import { StreamProcessor } from './stream-processor.js'; +import { ExecutorResult } from './types.js'; +import { buildProviderOptions } from './provider-options.js'; +import { TokenUsage } from '../types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import type { SessionEventBus, LLMFinishReason } from '../../events/index.js'; +import type { ResourceManager } from '../../resources/index.js'; +import { DynamicContributorContext } from '../../systemPrompt/types.js'; +import { LLMContext, type LLMProvider } from '../types.js'; +import type { MessageQueueService } from '../../session/message-queue.js'; +import type { StreamProcessorConfig } from './stream-processor.js'; +import type { CoalescedMessage } from '../../session/types.js'; +import { defer } from '../../utils/defer.js'; +import { DextoRuntimeError } from '../../errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '../../errors/types.js'; +import { LLMErrorCode } from '../error-codes.js'; +import { toError } from '../../utils/error-conversion.js'; +import { isOverflow, type ModelLimits } from '../../context/compaction/overflow.js'; +import { ReactiveOverflowStrategy } from '../../context/compaction/strategies/reactive-overflow.js'; +import type { ICompactionStrategy } from '../../context/compaction/types.js'; + +/** + * Static cache for tool support validation. + * Persists across TurnExecutor instances to avoid repeated validation calls. + * Key format: "provider:model:baseURL" + */ +const toolSupportCache = new Map(); + +/** + * Local providers that need tool support validation regardless of baseURL + */ +const LOCAL_PROVIDERS: readonly LLMProvider[] = ['ollama', 'local'] as const; + +/** + * TurnExecutor orchestrates the agent loop using `stopWhen: stepCountIs(1)`. + * + * This is the main entry point that replaces Vercel's internal loop with our + * controlled execution, giving us control between steps for: + * - Message queue injection (Phase 6) + * - Compression decisions (Phase 4) + * - Pruning old tool outputs (Phase 5) + * + * Key design: Uses stopWhen: stepCountIs(1) to regain control after each step. + * A "step" = ONE LLM call + ALL tool executions from that call. + */ +export class TurnExecutor { + private logger: IDextoLogger; + /** + * Per-step abort controller. Created fresh for each iteration of the loop. + * This allows soft cancel (abort current step) while still continuing with queued messages. + */ + private stepAbortController: AbortController; + private compactionStrategy: ICompactionStrategy | null = null; + /** + * Map to track approval metadata by toolCallId. + * Used to pass approval info from tool execution to result persistence. + */ + private approvalMetadata = new Map< + string, + { requireApproval: boolean; approvalStatus?: 'approved' | 'rejected' } + >(); + + constructor( + private model: LanguageModel, + private toolManager: ToolManager, + private contextManager: ContextManager, + private eventBus: SessionEventBus, + private resourceManager: ResourceManager, + private sessionId: string, + private config: { + maxSteps?: number | undefined; + maxOutputTokens?: number | undefined; + temperature?: number | undefined; + baseURL?: string | undefined; + // Provider-specific options + reasoningEffort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | undefined; + }, + private llmContext: LLMContext, + logger: IDextoLogger, + private messageQueue: MessageQueueService, + private modelLimits?: ModelLimits, + private externalSignal?: AbortSignal, + compactionStrategy?: ICompactionStrategy | null, + private compactionThresholdPercent: number = 1.0 + ) { + this.logger = logger.createChild(DextoLogComponent.EXECUTOR); + // Initial controller - will be replaced per-step in execute() + this.stepAbortController = new AbortController(); + + // NOTE: We intentionally do NOT link external signal here permanently. + // Instead, we link it per-step in execute() so that: + // - Soft cancel: aborts current step, but queue can continue with fresh controller + // - Hard cancel (external aborted + clearQueue): checked explicitly in loop + + // Use provided compaction strategy, or fallback to default behavior + if (compactionStrategy !== undefined) { + // Explicitly provided (could be null to disable, or a strategy instance) + this.compactionStrategy = compactionStrategy; + } else if (modelLimits) { + // Backward compatibility: create default strategy if model limits are provided + this.compactionStrategy = new ReactiveOverflowStrategy(model, {}, this.logger); + } + } + + /** + * Get StreamProcessor config from TurnExecutor state. + * @param estimatedInputTokens Optional estimated input tokens for analytics + */ + private getStreamProcessorConfig(estimatedInputTokens?: number): StreamProcessorConfig { + return { + provider: this.llmContext.provider, + model: this.llmContext.model, + ...(estimatedInputTokens !== undefined && { estimatedInputTokens }), + }; + } + + /** + * Main agent execution loop. + * Uses stopWhen: stepCountIs(1) to regain control after each step. + * + * @param contributorContext Context for system prompt contributors + * @param streaming If true, emits llm:chunk events during streaming. Default true. + */ + async execute( + contributorContext: DynamicContributorContext, + streaming: boolean = true + ): Promise { + // Automatic cleanup when scope exits (normal, throw, or return) + using _ = defer(() => this.cleanup()); + + // Track run duration + const startTime = Date.now(); + + let stepCount = 0; + let lastStepTokens: TokenUsage | null = null; + let lastFinishReason: LLMFinishReason = 'unknown'; + let lastText = ''; + + this.eventBus.emit('llm:thinking'); + + // Check tool support once before the loop + const supportsTools = await this.validateToolSupport(); + + // Emit warning if tools are not supported + if (!supportsTools) { + const modelKey = `${this.llmContext.provider}:${this.llmContext.model}`; + this.eventBus.emit('llm:unsupported-input', { + errors: [ + `Model '${modelKey}' does not support tool calling.`, + 'You can still chat, but the model will not be able to use tools or execute commands.', + ], + provider: this.llmContext.provider, + model: this.llmContext.model, + details: { + feature: 'tool-calling', + supported: false, + }, + }); + this.logger.warn( + `Model ${modelKey} does not support tools - continuing without tool calling` + ); + } + + // Track current abort handler to remove between iterations (prevents listener accumulation) + let currentAbortHandler: (() => void) | null = null; + + try { + while (true) { + // 0. Clean up previous iteration's abort handler (if any) + if (currentAbortHandler && this.externalSignal) { + this.externalSignal.removeEventListener('abort', currentAbortHandler); + } + + // Create fresh abort controller for this step + // This allows soft cancel (abort current step) while continuing with queued messages + this.stepAbortController = new AbortController(); + + // Link external signal to this step only (if not already aborted for hard cancel) + currentAbortHandler = () => this.stepAbortController.abort(); + if (this.externalSignal && !this.externalSignal.aborted) { + this.externalSignal.addEventListener('abort', currentAbortHandler, { + once: true, + }); + } + + // 1. Check for queued messages (mid-loop injection) + const coalesced = this.messageQueue.dequeueAll(); + if (coalesced) { + await this.injectQueuedMessages(coalesced); + } + + // 2. Prune old tool outputs BEFORE checking compaction + // This gives pruning a chance to free up space and potentially avoid compaction + await this.pruneOldToolOutputs(); + + // 3. Get formatted messages for this step + let prepared = await this.contextManager.getFormattedMessagesForLLM( + contributorContext, + this.llmContext + ); + + // 4. PRE-CHECK: Estimate tokens and compact if over threshold BEFORE LLM call + // This ensures we never make an oversized call and avoids unnecessary compaction on stop steps + // Uses the same formula as /context overlay: lastInput + lastOutput + newMessagesEstimate + const toolDefinitions = supportsTools ? await this.toolManager.getAllTools() : {}; + let estimatedTokens = await this.contextManager.getEstimatedNextInputTokens( + prepared.systemPrompt, + prepared.preparedHistory, + toolDefinitions + ); + if (this.shouldCompact(estimatedTokens)) { + this.logger.debug( + `Pre-check: estimated ${estimatedTokens} tokens exceeds threshold, compacting` + ); + const didCompact = await this.compactContext( + estimatedTokens, + contributorContext, + toolDefinitions + ); + + // If compaction occurred, rebuild messages (filterCompacted will handle it) + if (didCompact) { + prepared = await this.contextManager.getFormattedMessagesForLLM( + contributorContext, + this.llmContext + ); + // Recompute token estimate after compaction for accurate analytics/metrics + estimatedTokens = await this.contextManager.getEstimatedNextInputTokens( + prepared.systemPrompt, + prepared.preparedHistory, + toolDefinitions + ); + this.logger.debug( + `Post-compaction: recomputed estimate is ${estimatedTokens} tokens` + ); + } + } + + this.logger.debug(`Step ${stepCount}: Starting`); + + // 5. Create tools with execute callbacks and toModelOutput + // Use empty object if model doesn't support tools + const tools = supportsTools ? await this.createTools() : {}; + + // 6. Execute single step with stream processing + const streamProcessor = new StreamProcessor( + this.contextManager, + this.eventBus, + this.resourceManager, + this.stepAbortController.signal, + this.getStreamProcessorConfig(estimatedTokens), + this.logger, + streaming, + this.approvalMetadata + ); + + // Build provider-specific options (caching, reasoning, etc.) + const providerOptions = buildProviderOptions({ + provider: this.llmContext.provider, + model: this.llmContext.model, + reasoningEffort: this.config.reasoningEffort, + }); + + const result = await streamProcessor.process(() => + streamText({ + model: this.model, + stopWhen: stepCountIs(1), + tools, + abortSignal: this.stepAbortController.signal, + messages: prepared.formattedMessages, + ...(this.config.maxOutputTokens !== undefined && { + maxOutputTokens: this.config.maxOutputTokens, + }), + ...(this.config.temperature !== undefined && { + temperature: this.config.temperature, + }), + // Provider-specific options (caching, reasoning, etc.) + ...(providerOptions !== undefined && { + providerOptions: + providerOptions as import('@ai-sdk/provider').SharedV2ProviderOptions, + }), + // Log stream-level errors (tool errors, API errors during streaming) + onError: (error) => { + this.logger.error('Stream error', { error }); + }, + }) + ); + + // 7. Capture results for tracking and overflow check + lastStepTokens = result.usage; + lastFinishReason = result.finishReason; + lastText = result.text; + + this.logger.debug( + `Step ${stepCount}: Finished with reason="${result.finishReason}", ` + + `tokens=${JSON.stringify(result.usage)}` + ); + + // 7a. Store actual token counts for context estimation formula + // Formula: estimatedNextInput = lastInput + lastOutput + newMessagesEstimate + // + // Strategy: Only update tracking variables on successful calls with actual token data. + // - lastInput/lastOutput: ground truth from API, used as base for next estimate + // - lastCallMessageCount: boundary for identifying "new" messages to estimate + // + // On cancellation: Don't update anything. The partial response is saved to history, + // and newMessagesEstimate will include it when calculating from the last successful + // call's boundary. This minimizes estimation surface - we only estimate the delta + // since the last call that gave us actual token counts. Multiple consecutive + // cancellations accumulate in newMessagesEstimate until a successful call self-corrects. + // + // Tracking issue for AI SDK to support partial usage on cancel: + // https://github.com/vercel/ai/issues/7628 + if (result.finishReason === 'cancelled') { + // On cancellation, don't update any tracking variables. + // The partial response is saved to history, and newMessagesEstimate will + // include it when calculating from the last successful call's boundary. + // This keeps estimation surface minimal - we only estimate the delta since + // the last call that gave us actual token counts. + this.logger.info( + `Context estimation (cancelled): keeping last known actuals, partial response (${result.text.length} chars) will be estimated` + ); + } else if (result.usage?.inputTokens !== undefined) { + const contextInputTokens = this.getContextInputTokens(result.usage); + const actualInputTokens = contextInputTokens ?? result.usage.inputTokens; + + // Log verification metric: compare our estimate vs actual from API + const diff = estimatedTokens - actualInputTokens; + const diffPercent = + actualInputTokens > 0 + ? ((diff / actualInputTokens) * 100).toFixed(1) + : '0.0'; + this.logger.info( + `Context estimation accuracy: estimated=${estimatedTokens}, actual=${actualInputTokens}, ` + + `error=${diff} (${diffPercent}%)` + ); + this.contextManager.setLastActualInputTokens(actualInputTokens); + + if (result.usage?.outputTokens !== undefined) { + this.contextManager.setLastActualOutputTokens(result.usage.outputTokens); + } + + // Record message count boundary for identifying "new" messages + // Only update on successful calls - cancelled calls leave boundary unchanged + // so their content is included in newMessagesEstimate + await this.contextManager.recordLastCallMessageCount(); + } + + // 7b. POST-RESPONSE CHECK: Use actual inputTokens from API to detect overflow + // This catches cases where the LLM's response pushed us over the threshold + const contextInputTokens = result.usage + ? this.getContextInputTokens(result.usage) + : null; + if (contextInputTokens && this.shouldCompactFromActual(contextInputTokens)) { + this.logger.debug( + `Post-response: actual ${contextInputTokens} tokens exceeds threshold, compacting` + ); + await this.compactContext( + contextInputTokens, + contributorContext, + toolDefinitions + ); + } + + // 8. Check termination conditions + if (result.finishReason !== 'tool-calls') { + // Check queue before terminating - process queued messages if any + // Note: Hard cancel clears the queue BEFORE aborting, so if messages exist + // here it means soft cancel - we should continue processing them + const queuedOnTerminate = this.messageQueue.dequeueAll(); + if (queuedOnTerminate) { + this.logger.debug( + `Continuing: ${queuedOnTerminate.messages.length} queued message(s) to process` + ); + await this.injectQueuedMessages(queuedOnTerminate); + continue; // Keep looping with fresh controller - process queued messages + } + + this.logger.debug(`Terminating: finishReason is "${result.finishReason}"`); + break; + } + // Hard cancel check during tool-calls - if queue is empty and signal aborted, exit + if (this.externalSignal?.aborted && !this.messageQueue.hasPending()) { + this.logger.debug('Terminating: hard cancel - external abort signal received'); + lastFinishReason = 'cancelled'; + break; + } + stepCount++; + if (this.config.maxSteps !== undefined && stepCount >= this.config.maxSteps) { + this.logger.debug(`Terminating: reached maxSteps (${this.config.maxSteps})`); + lastFinishReason = 'max-steps'; + break; + } + } + } catch (error) { + // Map provider errors to DextoRuntimeError + const mappedError = this.mapProviderError(error); + this.logger.error('TurnExecutor failed', { error: mappedError }); + + this.eventBus.emit('llm:error', { + error: mappedError, + context: 'TurnExecutor', + recoverable: false, + }); + + // Flush any pending history updates before completing + await this.contextManager.flush(); + + // Emit run:complete with error before throwing + this.eventBus.emit('run:complete', { + finishReason: 'error', + stepCount, + durationMs: Date.now() - startTime, + error: mappedError, + }); + + throw mappedError; + } + + // Flush any pending history updates to ensure durability + await this.contextManager.flush(); + + // Set telemetry attributes for token usage + this.setTelemetryAttributes(lastStepTokens); + + // Emit run:complete event - the entire run is now finished + this.eventBus.emit('run:complete', { + finishReason: lastFinishReason, + stepCount, + durationMs: Date.now() - startTime, + }); + + return { + text: lastText, + stepCount, + usage: lastStepTokens, + finishReason: lastFinishReason, + }; + } + + /** + * Abort the current step execution. + * Note: For full run cancellation, use the external abort signal. + */ + abort(): void { + this.stepAbortController.abort(); + } + + /** + * Inject coalesced queued messages into the context as a single user message. + * This enables mid-task user guidance. + */ + private async injectQueuedMessages(coalesced: CoalescedMessage): Promise { + // Add as single user message with all guidance + await this.contextManager.addMessage({ + role: 'user', + content: coalesced.combinedContent, + metadata: { + coalesced: coalesced.messages.length > 1, + messageCount: coalesced.messages.length, + originalMessageIds: coalesced.messages.map((m) => m.id), + }, + }); + + this.logger.info(`Injected ${coalesced.messages.length} queued message(s) into context`, { + count: coalesced.messages.length, + firstQueued: coalesced.firstQueuedAt, + lastQueued: coalesced.lastQueuedAt, + }); + } + + /** + * Validates if the current model supports tools. + * Uses a static cache to avoid repeated validation calls. + * + * For local providers (Ollama, local) and custom baseURL endpoints, makes a test call to verify tool support. + * Known cloud providers without baseURL are assumed to support tools. + */ + private async validateToolSupport(): Promise { + const modelKey = `${this.llmContext.provider}:${this.llmContext.model}:${this.config.baseURL ?? ''}`; + + // Check cache first + if (toolSupportCache.has(modelKey)) { + return toolSupportCache.get(modelKey)!; + } + + // Local providers need validation regardless of baseURL (models have varying support) + const isLocalProvider = LOCAL_PROVIDERS.includes(this.llmContext.provider); + + // Skip validation only for known cloud providers without custom baseURL + if (!this.config.baseURL && !isLocalProvider) { + this.logger.debug( + `Skipping tool validation for ${modelKey} - known cloud provider without custom baseURL` + ); + toolSupportCache.set(modelKey, true); + return true; + } + + this.logger.debug( + `Testing tool support for ${isLocalProvider ? 'local provider' : 'custom endpoint'} model: ${modelKey}` + ); + + // Create a minimal test tool + const testTool = { + test_tool: { + inputSchema: jsonSchema({ + type: 'object', + properties: {}, + additionalProperties: false, + }), + execute: async () => ({ result: 'test' }), + }, + }; + + // Add timeout protection to fail fast if endpoint is unresponsive + const testAbort = new AbortController(); + const testTimeout = setTimeout(() => testAbort.abort(), 5000); // 5s timeout + + try { + // Make a minimal generateText call with tools to test support + await generateText({ + model: this.model, + messages: [{ role: 'user', content: 'Hello' }], + tools: testTool, + stopWhen: stepCountIs(1), + abortSignal: testAbort.signal, + }); + clearTimeout(testTimeout); + + // If we get here, tools are supported + toolSupportCache.set(modelKey, true); + this.logger.debug(`Model ${modelKey} supports tools`); + return true; + } catch (error: unknown) { + clearTimeout(testTimeout); + const errorMessage = error instanceof Error ? error.message : String(error); + if (errorMessage.includes('does not support tools')) { + toolSupportCache.set(modelKey, false); + this.logger.debug( + `Detected that model ${modelKey} does not support tool calling - tool functionality will be disabled` + ); + return false; + } + // Other errors (including timeout) - assume tools are supported and let the actual call handle it + this.logger.debug( + `Tool validation error for ${modelKey}, assuming supported: ${errorMessage}` + ); + toolSupportCache.set(modelKey, true); + return true; + } + } + + /** + * Creates tools with execute callbacks and toModelOutput. + * + * Key design decisions: + * - execute() returns raw result with inline images (async) + * - toModelOutput() formats for LLM consumption (sync) + * - StreamProcessor handles persistence via tool-result events + */ + private async createTools(): Promise { + const tools: ToolSet = await this.toolManager.getAllTools(); + + return Object.fromEntries( + Object.entries(tools).map(([name, tool]) => [ + name, + { + inputSchema: jsonSchema(tool.parameters), + ...(tool.description && { description: tool.description }), + + /** + * Execute callback - runs the tool and returns raw result. + * Does NOT persist - StreamProcessor handles that on tool-result event. + * + * Uses Promise.race to ensure we return quickly on abort, even if the + * underlying tool (especially MCP tools we don't control) keeps running. + */ + execute: async ( + args: unknown, + options: { toolCallId: string } + ): Promise => { + this.logger.debug( + `Executing tool: ${name} (toolCallId: ${options.toolCallId})` + ); + + const abortSignal = this.stepAbortController.signal; + + // Check if already aborted before starting + if (abortSignal.aborted) { + this.logger.debug(`Tool ${name} cancelled before execution`); + return { error: 'Cancelled by user', cancelled: true }; + } + + // Create abort handler for cleanup + let abortHandler: (() => void) | null = null; + + // Create abort promise for Promise.race + // This ensures we return quickly even if tool doesn't respect abort signal + const abortPromise = new Promise<{ error: string; cancelled: true }>( + (resolve) => { + abortHandler = () => { + this.logger.debug(`Tool ${name} cancelled during execution`); + resolve({ error: 'Cancelled by user', cancelled: true }); + }; + abortSignal.addEventListener('abort', abortHandler, { once: true }); + } + ); + + // Race: tool execution vs abort signal + try { + const result = await Promise.race([ + (async () => { + // Run tool via toolManager - returns result with approval metadata + // toolCallId is passed for tracking parallel tool calls in the UI + // Pass abortSignal so tools can do proper cleanup (e.g., kill processes) + const executionResult = await this.toolManager.executeTool( + name, + args as Record, + options.toolCallId, + this.sessionId, + abortSignal + ); + + // Store approval metadata for later retrieval by StreamProcessor + if (executionResult.requireApproval !== undefined) { + const metadata: { + requireApproval: boolean; + approvalStatus?: 'approved' | 'rejected'; + } = { + requireApproval: executionResult.requireApproval, + }; + if (executionResult.approvalStatus !== undefined) { + metadata.approvalStatus = + executionResult.approvalStatus; + } + this.approvalMetadata.set(options.toolCallId, metadata); + } + + // Return just the raw result for Vercel SDK + return executionResult.result; + })(), + abortPromise, + ]); + + return result; + } finally { + // Clean up abort listener to prevent memory leak + if (abortHandler) { + abortSignal.removeEventListener('abort', abortHandler); + } + } + }, + + /** + * toModelOutput - formats raw result for LLM consumption. + * Called by Vercel SDK when preparing messages for next LLM call. + * SYNC - images are already inline in the raw result. + */ + toModelOutput: (result: unknown) => { + return this.formatToolResultForLLM(result, name); + }, + }, + ]) + ); + } + + /** + * Format tool result for LLM consumption. + * Handles multimodal content (text + images). + * + * This handles RAW tool results - the structure may vary. + */ + private formatToolResultForLLM( + result: unknown, + toolName: string + ): + | { type: 'text'; value: string } + | { + type: 'content'; + value: Array< + | { type: 'text'; text: string } + | { type: 'media'; data: string; mediaType: string } + >; + } { + // Handle error results + if (result && typeof result === 'object' && 'error' in result) { + const errorResult = result as { error: string; denied?: boolean; timeout?: boolean }; + let errorFlags = ''; + if (errorResult.denied) errorFlags += ' (denied)'; + if (errorResult.timeout) errorFlags += ' (timeout)'; + return { + type: 'text', + value: `Tool ${toolName} failed${errorFlags}: ${errorResult.error}`, + }; + } + + // Handle multimodal results with content array + if (this.hasMultimodalContent(result)) { + const contentArray = ( + result as { content: Array<{ type: string; [key: string]: unknown }> } + ).content; + const contentValue: Array< + { type: 'text'; text: string } | { type: 'media'; data: string; mediaType: string } + > = []; + + for (const part of contentArray) { + if (part.type === 'text' && typeof part.text === 'string') { + contentValue.push({ type: 'text', text: part.text }); + } else if (part.type === 'image') { + // Handle various image formats - check both 'image' and 'data' fields + const imageData = this.extractImageData(part); + if (imageData) { + contentValue.push({ + type: 'media', + data: imageData, + mediaType: (part.mimeType as string) || 'image/jpeg', + }); + } + } else if (part.type === 'file') { + const fileData = this.extractFileData(part); + if (fileData) { + contentValue.push({ + type: 'media', + data: fileData, + mediaType: (part.mimeType as string) || 'application/octet-stream', + }); + } + } + } + + // If we have multimodal content (media), return it + if (contentValue.length > 0 && contentValue.some((v) => v.type === 'media')) { + return { type: 'content', value: contentValue }; + } + + // Text-only content array - concatenate text parts + const textParts = contentArray + .filter((p) => p.type === 'text' && typeof p.text === 'string') + .map((p) => p.text as string); + return { + type: 'text', + value: textParts.join('\n') || '[empty result]', + }; + } + + // Fallback: convert to string + if (typeof result === 'string') { + return { type: 'text', value: result }; + } + + return { + type: 'text', + value: + typeof result === 'object' && result !== null + ? JSON.stringify(result) + : String(result), + }; + } + + /** + * Extract image data from a part, handling various formats. + */ + private extractImageData(part: { [key: string]: unknown }): string | null { + // Try 'image' field first (our standard ImagePart format) + if (typeof part.image === 'string') { + return part.image; + } + // Try 'data' field (alternative format) + if (typeof part.data === 'string') { + return part.data; + } + // Handle Buffer/ArrayBuffer + if (part.image instanceof Buffer) { + return part.image.toString('base64'); + } + if (part.data instanceof Buffer) { + return (part.data as Buffer).toString('base64'); + } + if (part.image instanceof ArrayBuffer) { + return Buffer.from(new Uint8Array(part.image)).toString('base64'); + } + if (part.data instanceof ArrayBuffer) { + return Buffer.from(new Uint8Array(part.data as ArrayBuffer)).toString('base64'); + } + return null; + } + + /** + * Extract file data from a part. + */ + private extractFileData(part: { [key: string]: unknown }): string | null { + if (typeof part.data === 'string') { + return part.data; + } + if (part.data instanceof Buffer) { + return (part.data as Buffer).toString('base64'); + } + if (part.data instanceof ArrayBuffer) { + return Buffer.from(new Uint8Array(part.data as ArrayBuffer)).toString('base64'); + } + return null; + } + + /** + * Check if result has multimodal content array + */ + private hasMultimodalContent(result: unknown): boolean { + return ( + result !== null && + typeof result === 'object' && + 'content' in result && + Array.isArray((result as { content: unknown }).content) + ); + } + + /** + * Constants for pruning thresholds + */ + private static readonly PRUNE_PROTECT = 40_000; // Keep last 40K tokens of tool outputs + private static readonly PRUNE_MINIMUM = 20_000; // Only prune if we can save 20K+ + + /** + * Prunes old tool outputs by marking them with compactedAt timestamp. + * Does NOT modify content - transformation happens at format time in + * ContextManager.prepareHistory(). + * + * Algorithm: + * 1. Go backwards through history (most recent first) + * 2. Stop at summary message (only process post-summary messages) + * 3. Count tool message tokens + * 4. If total exceeds PRUNE_PROTECT, mark older ones for pruning + * 5. Only prune if savings exceed PRUNE_MINIMUM + */ + private async pruneOldToolOutputs(): Promise<{ prunedCount: number; savedTokens: number }> { + const history = await this.contextManager.getHistory(); + let totalToolTokens = 0; + let prunedTokens = 0; + const toPrune: string[] = []; // Message IDs to mark + + // Go backwards through history (most recent first) + for (let i = history.length - 1; i >= 0; i--) { + const msg = history[i]; + if (!msg) continue; + + // Stop at summary message - only prune AFTER the summary + if (msg.metadata?.isSummary === true) break; + + // Only process tool messages + if (msg.role !== 'tool') continue; + + // Skip already pruned messages + if (msg.compactedAt) continue; + + // Tool message content is always an array after sanitization + if (!Array.isArray(msg.content)) continue; + + const tokens = this.estimateToolTokens(msg.content); + totalToolTokens += tokens; + + // If we've exceeded protection threshold, mark for pruning + if (totalToolTokens > TurnExecutor.PRUNE_PROTECT && msg.id) { + prunedTokens += tokens; + toPrune.push(msg.id); + } + } + + // Only prune if significant savings + if (prunedTokens > TurnExecutor.PRUNE_MINIMUM && toPrune.length > 0) { + const markedCount = await this.contextManager.markMessagesAsCompacted(toPrune); + + this.eventBus.emit('context:pruned', { + prunedCount: markedCount, + savedTokens: prunedTokens, + }); + + this.logger.debug(`Pruned ${markedCount} tool outputs, saving ~${prunedTokens} tokens`); + + return { prunedCount: markedCount, savedTokens: prunedTokens }; + } + + return { prunedCount: 0, savedTokens: 0 }; + } + + /** + * Estimates tokens for tool message content using simple heuristic (length/4). + * Used for pruning decisions only - actual token counts come from API. + * + * Tool message content is always Array + * after sanitization via SanitizedToolResult. + */ + private estimateToolTokens( + content: Array + ): number { + return content.reduce((sum, part) => { + if (part.type === 'text') { + return sum + Math.ceil(part.text.length / 4); + } + // Images/files contribute ~1000 tokens estimate + if (part.type === 'image' || part.type === 'file') { + return sum + 1000; + } + // UIResourcePart - minimal token contribution + return sum; + }, 0); + } + + /** + * Cleanup resources when execution scope exits. + * Called automatically via defer() on normal exit, throw, or abort. + */ + private cleanup(): void { + this.logger.debug('TurnExecutor cleanup triggered'); + + // Abort any pending operations for current step + if (!this.stepAbortController.signal.aborted) { + this.stepAbortController.abort(); + } + + // Clear any pending queued messages + this.messageQueue.clear(); + } + + /** + * Check if context should be compacted based on estimated token count. + * Uses the threshold percentage from compaction config to trigger earlier (e.g., at 90%). + * + * @param estimatedTokens Estimated token count from the current context + * @returns true if compaction is needed before making the LLM call + */ + private shouldCompact(estimatedTokens: number): boolean { + if (!this.modelLimits || !this.compactionStrategy) { + return false; + } + // Use the overflow logic with threshold to trigger compaction earlier + return isOverflow( + { inputTokens: estimatedTokens }, + this.modelLimits, + this.compactionThresholdPercent + ); + } + + /** + * Check if context should be compacted based on actual token count from API response. + * This is a post-response check using real token counts rather than estimates. + * + * @param actualTokens Actual input token count from the API response + * @returns true if compaction is needed + */ + private shouldCompactFromActual(actualTokens: number): boolean { + if (!this.modelLimits || !this.compactionStrategy) { + return false; + } + // Use the same overflow logic but with actual tokens from API + return isOverflow( + { inputTokens: actualTokens }, + this.modelLimits, + this.compactionThresholdPercent + ); + } + + /** + * Compact context by generating a summary and adding it to the same session. + * + * The summary message is added to the conversation history with `isSummary: true` metadata. + * When the context is loaded via getFormattedMessagesForLLM(), filterCompacted() will + * exclude all messages before the summary, effectively compacting the context. + * + * @param originalTokens The estimated input token count that triggered overflow + * @param contributorContext Context for system prompt contributors (needed for accurate token estimation) + * @param tools Tool definitions (needed for accurate token estimation) + * @returns true if compaction occurred, false if skipped + */ + private async compactContext( + originalTokens: number, + contributorContext: DynamicContributorContext, + tools: Record + ): Promise { + if (!this.compactionStrategy) { + return false; + } + + this.logger.info( + `Context overflow detected (${originalTokens} tokens), checking if compression is possible` + ); + + const history = await this.contextManager.getHistory(); + const { filterCompacted } = await import('../../context/utils.js'); + const originalFiltered = filterCompacted(history); + const originalMessages = originalFiltered.length; + + // Pre-check if history is long enough for compaction (need at least 4 messages for meaningful summary) + if (history.length < 4) { + this.logger.debug('Compaction skipped: history too short to summarize'); + return false; + } + + // Emit event BEFORE the LLM summarization call so UI shows indicator during compaction + this.eventBus.emit('context:compacting', { + estimatedTokens: originalTokens, + }); + + // Generate summary message(s) - this makes an LLM call + const summaryMessages = await this.compactionStrategy.compact(history); + + if (summaryMessages.length === 0) { + // Compaction returned empty - nothing to summarize (e.g., already compacted) + // Still emit context:compacted to clear the UI's compacting state + this.logger.debug( + 'Compaction skipped: strategy returned no summary (likely already compacted or nothing to summarize)' + ); + this.eventBus.emit('context:compacted', { + originalTokens, + compactedTokens: originalTokens, // No change + originalMessages, + compactedMessages: originalMessages, // No change + strategy: this.compactionStrategy.name, + reason: 'overflow', + }); + return false; + } + + // Add summary to history - filterCompacted() will exclude pre-summary messages at read-time + for (const summary of summaryMessages) { + await this.contextManager.addMessage(summary); + } + + // Reset actual token tracking since context has fundamentally changed + // The formula (lastInput + lastOutput + newEstimate) is no longer valid after compaction + this.contextManager.resetActualTokenTracking(); + + // Get accurate token estimate after compaction using the same method as /context command + // This ensures consistency between what we report and what /context shows + const afterEstimate = await this.contextManager.getContextTokenEstimate( + contributorContext, + tools + ); + const compactedTokens = afterEstimate.estimated; + const compactedMessages = afterEstimate.stats.filteredMessageCount; + + this.eventBus.emit('context:compacted', { + originalTokens, + compactedTokens, + originalMessages, + compactedMessages, + strategy: this.compactionStrategy.name, + reason: 'overflow', + }); + + this.logger.info( + `Compaction complete: ${originalTokens} → ~${compactedTokens} tokens ` + + `(${originalMessages} → ${compactedMessages} messages after filtering)` + ); + + return true; + } + + /** + * Set telemetry span attributes for token usage. + */ + private setTelemetryAttributes(usage: TokenUsage | null): void { + const activeSpan = trace.getActiveSpan(); + if (!activeSpan || !usage) { + return; + } + + if (usage.inputTokens !== undefined) { + activeSpan.setAttribute('gen_ai.usage.input_tokens', usage.inputTokens); + } + if (usage.outputTokens !== undefined) { + activeSpan.setAttribute('gen_ai.usage.output_tokens', usage.outputTokens); + } + if (usage.totalTokens !== undefined) { + activeSpan.setAttribute('gen_ai.usage.total_tokens', usage.totalTokens); + } + if (usage.reasoningTokens !== undefined) { + activeSpan.setAttribute('gen_ai.usage.reasoning_tokens', usage.reasoningTokens); + } + } + + private getContextInputTokens(usage: TokenUsage): number | null { + if (usage.inputTokens === undefined) return null; + return usage.inputTokens + (usage.cacheReadTokens ?? 0) + (usage.cacheWriteTokens ?? 0); + } + + /** + * Map provider errors to DextoRuntimeError. + */ + private mapProviderError(err: unknown): Error { + if (APICallError.isInstance?.(err)) { + const status = err.statusCode; + const headers = (err.responseHeaders || {}) as Record; + const retryAfter = headers['retry-after'] ? Number(headers['retry-after']) : undefined; + const body = + typeof err.responseBody === 'string' + ? err.responseBody + : JSON.stringify(err.responseBody ?? ''); + + if (status === 402) { + // Dexto gateway returns 402 with INSUFFICIENT_CREDITS when balance is low + // Try to extract balance from response body + let balance: number | undefined; + try { + const parsed = JSON.parse(body); + // Format: { error: { code: 'INSUFFICIENT_CREDITS', message: '...Balance: $X.XX...' } } + const msg = parsed?.error?.message || ''; + const match = msg.match(/Balance:\s*\$?([\d.]+)/i); + if (match) { + balance = parseFloat(match[1]); + } + } catch { + // Ignore parse errors + } + return new DextoRuntimeError( + LLMErrorCode.INSUFFICIENT_CREDITS, + ErrorScope.LLM, + ErrorType.PAYMENT_REQUIRED, + `Insufficient Dexto credits${balance !== undefined ? `. Balance: $${balance.toFixed(2)}` : ''}`, + { + sessionId: this.sessionId, + provider: this.llmContext.provider, + model: this.llmContext.model, + status, + balance, + body, + }, + 'Run `dexto billing` to check your balance' + ); + } + if (status === 429) { + return new DextoRuntimeError( + LLMErrorCode.RATE_LIMIT_EXCEEDED, + ErrorScope.LLM, + ErrorType.RATE_LIMIT, + `Rate limit exceeded${body ? ` - ${body}` : ''}`, + { + sessionId: this.sessionId, + provider: this.llmContext.provider, + model: this.llmContext.model, + status, + retryAfter, + body, + } + ); + } + if (status === 408) { + return new DextoRuntimeError( + LLMErrorCode.GENERATION_FAILED, + ErrorScope.LLM, + ErrorType.TIMEOUT, + `Provider timed out${body ? ` - ${body}` : ''}`, + { + sessionId: this.sessionId, + provider: this.llmContext.provider, + model: this.llmContext.model, + status, + body, + } + ); + } + return new DextoRuntimeError( + LLMErrorCode.GENERATION_FAILED, + ErrorScope.LLM, + ErrorType.THIRD_PARTY, + `Provider error ${status}${body ? ` - ${body}` : ''}`, + { + sessionId: this.sessionId, + provider: this.llmContext.provider, + model: this.llmContext.model, + status, + body, + } + ); + } + + return toError(err, this.logger); + } +} diff --git a/dexto/packages/core/src/llm/executor/types.ts b/dexto/packages/core/src/llm/executor/types.ts new file mode 100644 index 00000000..5eda07ac --- /dev/null +++ b/dexto/packages/core/src/llm/executor/types.ts @@ -0,0 +1,28 @@ +import { TokenUsage } from '../types.js'; +import { LLMFinishReason } from '../../events/index.js'; + +export interface ExecutorResult { + /** + * The accumulated text from assistant responses. + * TODO: Some LLMs are multimodal and can generate non-text content (images, audio, etc.). + * Consider extending this to support multimodal output in the future. + */ + text: string; + /** Number of steps executed */ + stepCount: number; + /** Token usage from the last step */ + usage: TokenUsage | null; + /** Reason the execution finished */ + finishReason: LLMFinishReason; +} + +export interface StreamProcessorResult { + /** + * The accumulated text from text-delta events. + * TODO: Some LLMs are multimodal and can generate non-text content (images, audio, etc.). + * Consider extending this to support multimodal output in the future. + */ + text: string; + finishReason: LLMFinishReason; + usage: TokenUsage; +} diff --git a/dexto/packages/core/src/llm/formatters/vercel.test.ts b/dexto/packages/core/src/llm/formatters/vercel.test.ts new file mode 100644 index 00000000..b3b818d4 --- /dev/null +++ b/dexto/packages/core/src/llm/formatters/vercel.test.ts @@ -0,0 +1,290 @@ +import { describe, test, expect, vi } from 'vitest'; +import { VercelMessageFormatter } from './vercel.js'; +import { createMockLogger } from '../../logger/v2/test-utils.js'; +import type { InternalMessage } from '../../context/types.js'; +import * as registry from '../registry.js'; + +// Mock the registry to allow all file types +vi.mock('../registry.js'); +const mockValidateModelFileSupport = vi.mocked(registry.validateModelFileSupport); +mockValidateModelFileSupport.mockReturnValue({ isSupported: true, fileType: 'pdf' }); + +const mockLogger = createMockLogger(); + +describe('VercelMessageFormatter', () => { + describe('URL string auto-detection', () => { + test('should convert image URL string to URL object', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Describe this image' }, + { + type: 'image', + image: 'https://example.com/image.png', + mimeType: 'image/png', + }, + ], + }, + ]; + + const result = formatter.format( + messages, + { provider: 'openai', model: 'gpt-4o' }, + 'You are helpful' + ); + + // Find the user message + const userMessage = result.find((m) => m.role === 'user'); + expect(userMessage).toBeDefined(); + + const content = userMessage!.content as Array<{ type: string; image?: URL | string }>; + const imagePart = content.find((p) => p.type === 'image'); + expect(imagePart).toBeDefined(); + expect(imagePart!.image).toBeInstanceOf(URL); + expect((imagePart!.image as URL).href).toBe('https://example.com/image.png'); + }); + + test('should convert file URL string to URL object', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Summarize this document' }, + { + type: 'file', + data: 'https://example.com/document.pdf', + mimeType: 'application/pdf', + }, + ], + }, + ]; + + const result = formatter.format( + messages, + { provider: 'openai', model: 'gpt-4o' }, + 'You are helpful' + ); + + const userMessage = result.find((m) => m.role === 'user'); + expect(userMessage).toBeDefined(); + + const content = userMessage!.content as Array<{ type: string; data?: URL | string }>; + const filePart = content.find((p) => p.type === 'file'); + expect(filePart).toBeDefined(); + expect(filePart!.data).toBeInstanceOf(URL); + expect((filePart!.data as URL).href).toBe('https://example.com/document.pdf'); + }); + + test('should preserve base64 strings as-is', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const base64Image = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + const messages: InternalMessage[] = [ + { + role: 'user', + content: [{ type: 'image', image: base64Image, mimeType: 'image/png' }], + }, + ]; + + const result = formatter.format( + messages, + { provider: 'openai', model: 'gpt-4o' }, + 'You are helpful' + ); + + const userMessage = result.find((m) => m.role === 'user'); + const content = userMessage!.content as Array<{ type: string; image?: string }>; + const imagePart = content.find((p) => p.type === 'image'); + expect(imagePart!.image).toBe(base64Image); + expect(typeof imagePart!.image).toBe('string'); + }); + + test('should preserve data URI strings as-is', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const dataUri = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + const messages: InternalMessage[] = [ + { + role: 'user', + content: [{ type: 'image', image: dataUri, mimeType: 'image/png' }], + }, + ]; + + const result = formatter.format( + messages, + { provider: 'openai', model: 'gpt-4o' }, + 'You are helpful' + ); + + const userMessage = result.find((m) => m.role === 'user'); + const content = userMessage!.content as Array<{ type: string; image?: string }>; + const imagePart = content.find((p) => p.type === 'image'); + expect(imagePart!.image).toBe(dataUri); + expect(typeof imagePart!.image).toBe('string'); + }); + + test('should handle http:// URLs (not just https://)', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { + type: 'image', + image: 'http://example.com/image.png', + mimeType: 'image/png', + }, + ], + }, + ]; + + const result = formatter.format( + messages, + { provider: 'openai', model: 'gpt-4o' }, + 'You are helpful' + ); + + const userMessage = result.find((m) => m.role === 'user'); + const content = userMessage!.content as Array<{ type: string; image?: URL }>; + const imagePart = content.find((p) => p.type === 'image'); + expect(imagePart!.image).toBeInstanceOf(URL); + }); + + test('should preserve URL objects as-is', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const urlObj = new URL('https://example.com/image.png'); + const messages: InternalMessage[] = [ + { + role: 'user', + content: [ + { + type: 'image', + image: urlObj as unknown as string, + mimeType: 'image/png', + }, + ], + }, + ]; + + const result = formatter.format( + messages, + { provider: 'openai', model: 'gpt-4o' }, + 'You are helpful' + ); + + const userMessage = result.find((m) => m.role === 'user'); + const content = userMessage!.content as Array<{ type: string; image?: URL }>; + const imagePart = content.find((p) => p.type === 'image'); + // URL object should be preserved (or converted back to URL) + expect(imagePart!.image).toBeInstanceOf(URL); + }); + }); + + describe('Reasoning round-trip', () => { + test('should include reasoning part in assistant message when reasoning is present', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const messages: InternalMessage[] = [ + { + role: 'assistant', + content: [{ type: 'text', text: 'Here is my answer' }], + reasoning: 'Let me think about this carefully...', + }, + ]; + + const result = formatter.format( + messages, + { provider: 'anthropic', model: 'claude-3-5-sonnet-20241022' }, + 'You are helpful' + ); + + const assistantMessage = result.find((m) => m.role === 'assistant'); + expect(assistantMessage).toBeDefined(); + + const content = assistantMessage!.content as Array<{ type: string; text?: string }>; + const reasoningPart = content.find((p) => p.type === 'reasoning'); + expect(reasoningPart).toBeDefined(); + expect(reasoningPart!.text).toBe('Let me think about this carefully...'); + }); + + test('should include providerOptions in reasoning part when reasoningMetadata is present', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const reasoningMetadata = { anthropic: { cacheId: 'cache-123' } }; + const messages: InternalMessage[] = [ + { + role: 'assistant', + content: [{ type: 'text', text: 'Answer' }], + reasoning: 'Thinking...', + reasoningMetadata, + }, + ]; + + const result = formatter.format( + messages, + { provider: 'anthropic', model: 'claude-3-5-sonnet-20241022' }, + 'You are helpful' + ); + + const assistantMessage = result.find((m) => m.role === 'assistant'); + const content = assistantMessage!.content as Array<{ + type: string; + providerOptions?: Record; + }>; + const reasoningPart = content.find((p) => p.type === 'reasoning'); + + expect(reasoningPart).toBeDefined(); + expect(reasoningPart!.providerOptions).toEqual(reasoningMetadata); + }); + + test('should place reasoning part before text content', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const messages: InternalMessage[] = [ + { + role: 'assistant', + content: [{ type: 'text', text: 'Final answer' }], + reasoning: 'Step by step reasoning...', + }, + ]; + + const result = formatter.format( + messages, + { provider: 'anthropic', model: 'claude-3-5-sonnet-20241022' }, + 'You are helpful' + ); + + const assistantMessage = result.find((m) => m.role === 'assistant'); + const content = assistantMessage!.content as Array<{ type: string }>; + + // Reasoning should come before text + const reasoningIndex = content.findIndex((p) => p.type === 'reasoning'); + const textIndex = content.findIndex((p) => p.type === 'text'); + + expect(reasoningIndex).toBeLessThan(textIndex); + }); + + test('should not include reasoning part when reasoning is not present', () => { + const formatter = new VercelMessageFormatter(mockLogger); + const messages: InternalMessage[] = [ + { + role: 'assistant', + content: [{ type: 'text', text: 'Simple answer' }], + // No reasoning field + }, + ]; + + const result = formatter.format( + messages, + { provider: 'openai', model: 'gpt-4o' }, + 'You are helpful' + ); + + const assistantMessage = result.find((m) => m.role === 'assistant'); + const content = assistantMessage!.content as Array<{ type: string }>; + const reasoningPart = content.find((p) => p.type === 'reasoning'); + + expect(reasoningPart).toBeUndefined(); + }); + }); +}); diff --git a/dexto/packages/core/src/llm/formatters/vercel.ts b/dexto/packages/core/src/llm/formatters/vercel.ts new file mode 100644 index 00000000..45749b65 --- /dev/null +++ b/dexto/packages/core/src/llm/formatters/vercel.ts @@ -0,0 +1,372 @@ +import type { ModelMessage, AssistantContent, ToolContent, ToolResultPart } from 'ai'; +import { LLMContext } from '../types.js'; +import type { InternalMessage, AssistantMessage, ToolMessage } from '@core/context/types.js'; +import { getImageData, getFileData, filterMessagesByLLMCapabilities } from '@core/context/utils.js'; +import type { IDextoLogger } from '@core/logger/v2/types.js'; +import { DextoLogComponent } from '@core/logger/v2/types.js'; + +/** + * Checks if a string is a URL (http:// or https://). + * Returns a URL object if it's a valid URL string, otherwise returns the original value. + */ +function toUrlIfString(value: T): T | URL { + if (typeof value === 'string' && /^https?:\/\//i.test(value)) { + try { + return new URL(value); + } catch { + // Invalid URL, return original string + return value; + } + } + return value; +} + +/** + * Message formatter for Vercel AI SDK. + * + * Converts the internal message format to Vercel's specific structure: + * - System prompt is included in the messages array + * - Tool calls use function_call property instead of tool_calls + * - Tool results use the 'tool' role (SDK v5) + * + * Note: Vercel's implementation is different from OpenAI's standard, + * particularly in its handling of function calls and responses. + */ +export class VercelMessageFormatter { + private logger: IDextoLogger; + + constructor(logger: IDextoLogger) { + this.logger = logger.createChild(DextoLogComponent.LLM); + } + /** + * Formats internal messages into Vercel AI SDK format + * + * @param history Array of internal messages to format + * @param systemPrompt System prompt to include at the beginning of messages + * @returns Array of messages formatted for Vercel's API + */ + format( + history: Readonly, + context: LLMContext, + systemPrompt: string | null + ): ModelMessage[] { + // Returns Vercel-specific type + const formatted: ModelMessage[] = []; + + // Apply model-aware capability filtering for Vercel + let filteredHistory: InternalMessage[]; + try { + filteredHistory = filterMessagesByLLMCapabilities([...history], context, this.logger); + + const modelInfo = `${context.provider}/${context.model}`; + this.logger.debug(`Applied Vercel filtering for ${modelInfo}`); + } catch (error) { + this.logger.warn( + `Failed to apply capability filtering, using original history: ${error}` + ); + filteredHistory = [...history]; + } + + // Add system message if present + if (systemPrompt) { + // For Anthropic/Bedrock/Vertex Claude, add cacheControl to enable prompt caching + // This marks the system prompt as cacheable (ephemeral = cached for session duration) + const modelLower = context.model.toLowerCase(); + const isClaudeModel = modelLower.includes('claude'); + const isAnthropicProvider = + context.provider === 'anthropic' || + (context.provider === 'bedrock' && isClaudeModel) || + (context.provider === 'vertex' && isClaudeModel); + + formatted.push({ + role: 'system', + content: systemPrompt, + ...(isAnthropicProvider && { + providerOptions: { + anthropic: { cacheControl: { type: 'ephemeral' } }, + }, + }), + }); + } + + // Track pending tool calls to detect orphans (tool calls without results) + // Map stores toolCallId -> toolName for proper synthetic result generation + const pendingToolCalls = new Map(); + + for (const msg of filteredHistory) { + switch (msg.role) { + case 'user': + // Images (and text) in user content arrays are handled natively + // by the Vercel SDK. We can forward the array of TextPart/ImagePart directly. + if (msg.content !== null) { + // Convert internal content types to AI SDK types + // Filter out UIResourcePart - these are for UI rendering, not LLM processing + const content = + typeof msg.content === 'string' + ? msg.content + : msg.content + .filter((part) => part.type !== 'ui-resource') + .map((part) => { + if (part.type === 'file') { + return { + type: 'file' as const, + data: toUrlIfString(part.data), + mediaType: part.mimeType, // Convert mimeType -> mediaType + ...(part.filename && { filename: part.filename }), + }; + } else if (part.type === 'image') { + return { + type: 'image' as const, + image: toUrlIfString(part.image), + ...(part.mimeType && { + mediaType: part.mimeType, + }), // Convert mimeType -> mediaType + }; + } + return part; // TextPart doesn't need conversion + }); + + formatted.push({ + role: 'user', + content, + }); + } + break; + + case 'system': + // System messages + if (msg.content !== null) { + formatted.push({ + role: 'system', + content: String(msg.content), + }); + } + break; + + case 'assistant': + formatted.push({ role: 'assistant', ...this.formatAssistantMessage(msg) }); + // Track tool call IDs and names as pending + if (msg.toolCalls && msg.toolCalls.length > 0) { + for (const toolCall of msg.toolCalls) { + pendingToolCalls.set(toolCall.id, toolCall.function.name); + } + } + break; + + case 'tool': + // Only add if we've seen the corresponding tool call + if (msg.toolCallId && pendingToolCalls.has(msg.toolCallId)) { + formatted.push({ role: 'tool', ...this.formatToolMessage(msg) }); + // Remove from pending since we found its result + pendingToolCalls.delete(msg.toolCallId); + } else { + // Orphaned tool result (result without matching call) + // Skip it to prevent API errors - can't send result without corresponding call + this.logger.warn( + `Skipping orphaned tool result ${msg.toolCallId} (no matching tool call found) - cannot send to Vercel AI SDK without corresponding tool-call` + ); + } + break; + } + } + + // Add synthetic error results for any orphaned tool calls + // This can happen when CLI crashes/interrupts before tool execution completes + if (pendingToolCalls.size > 0) { + for (const [toolCallId, toolName] of pendingToolCalls.entries()) { + // Vercel AI SDK uses tool-result content parts with output property + formatted.push({ + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: toolCallId, + toolName: toolName, + output: { + type: 'text', + value: 'Error: Tool execution was interrupted (session crashed or cancelled before completion)', + }, + isError: true, + } as ToolResultPart, + ], + }); + this.logger.warn( + `Tool call ${toolCallId} (${toolName}) had no matching tool result - added synthetic error result to prevent API errors` + ); + } + } + + return formatted; + } + + /** + * Vercel handles system prompts in the messages array + * This method returns null since the system prompt is already + * included directly in the formatted messages. + * + * @returns null as Vercel doesn't need a separate system prompt + */ + formatSystemPrompt(): null { + return null; + } + + // Helper to format Assistant messages (with optional tool calls and reasoning) + private formatAssistantMessage(msg: AssistantMessage): { + content: AssistantContent; + function_call?: { name: string; arguments: string }; + } { + const contentParts: AssistantContent = []; + + // Add reasoning part if present (for round-tripping extended thinking) + if (msg.reasoning) { + // Cast to AssistantContent element type - providerOptions is Record + // which is compatible with our Record storage + const reasoningPart = { + type: 'reasoning' as const, + text: msg.reasoning, + ...(msg.reasoningMetadata && { providerOptions: msg.reasoningMetadata }), + }; + contentParts.push(reasoningPart as (typeof contentParts)[number]); + } + + // Add text content + if (Array.isArray(msg.content)) { + const combined = msg.content + .map((part) => (part.type === 'text' ? part.text : '')) + .filter(Boolean) + .join('\n'); + if (combined) { + contentParts.push({ type: 'text', text: combined }); + } + } else if (typeof msg.content === 'string') { + contentParts.push({ type: 'text', text: msg.content }); + } + + // Add tool calls if present + if (msg.toolCalls && msg.toolCalls.length > 0) { + for (const toolCall of msg.toolCalls) { + const rawArgs = toolCall.function.arguments; + let parsed: unknown = {}; + if (typeof rawArgs === 'string') { + try { + parsed = JSON.parse(rawArgs); + } catch { + parsed = {}; + this.logger.warn( + `Vercel formatter: invalid tool args JSON for ${toolCall.function.name}` + ); + } + } else { + parsed = rawArgs ?? {}; + } + // AI SDK v5 expects 'input' for tool-call arguments (not 'args'). + // Include providerOptions if present (e.g., Gemini 3 thought signatures) + const toolCallPart: (typeof contentParts)[number] = { + type: 'tool-call', + toolCallId: toolCall.id, + toolName: toolCall.function.name, + input: parsed, + }; + // Pass through providerOptions for round-tripping (thought signatures, etc.) + if (toolCall.providerOptions) { + (toolCallPart as { providerOptions?: unknown }).providerOptions = + toolCall.providerOptions; + } + contentParts.push(toolCallPart); + } + + const firstToolCall = msg.toolCalls[0]!; + // Ensure function_call.arguments is always a valid JSON string + const argString = (() => { + const raw = firstToolCall.function.arguments; + if (typeof raw === 'string') return raw; + try { + return JSON.stringify(raw ?? {}); + } catch { + return '{}'; + } + })(); + return { + content: contentParts, + function_call: { + name: firstToolCall.function.name, + arguments: argString, + }, + }; + } + + return { + content: contentParts.length > 0 ? contentParts : [], + }; + } + + // Helper to format Tool result messages + private formatToolMessage(msg: ToolMessage): { content: ToolContent } { + let toolResultPart: ToolResultPart; + if (Array.isArray(msg.content)) { + if (msg.content[0]?.type === 'image') { + const imagePart = msg.content[0]; + const imageDataBase64 = getImageData(imagePart, this.logger); + toolResultPart = { + type: 'tool-result', + toolCallId: msg.toolCallId, + toolName: msg.name, + output: { + type: 'content', + value: [ + { + type: 'media', + data: imageDataBase64, + mediaType: imagePart.mimeType || 'image/jpeg', + }, + ], + }, + }; + } else if (msg.content[0]?.type === 'file') { + const filePart = msg.content[0]; + const fileDataBase64 = getFileData(filePart, this.logger); + toolResultPart = { + type: 'tool-result', + toolCallId: msg.toolCallId, + toolName: msg.name, + output: { + type: 'content', + value: [ + { + type: 'media', + data: fileDataBase64, + mediaType: filePart.mimeType, + }, + ], + }, + }; + } else { + const textContent = Array.isArray(msg.content) + ? msg.content + .map((part) => (part.type === 'text' ? part.text : JSON.stringify(part))) + .join('\n') + : String(msg.content); + toolResultPart = { + type: 'tool-result', + toolCallId: msg.toolCallId, + toolName: msg.name, + output: { + type: 'text', + value: textContent, + }, + }; + } + } else { + toolResultPart = { + type: 'tool-result', + toolCallId: msg.toolCallId, + toolName: msg.name, + output: { + type: 'text', + value: String(msg.content || ''), + }, + }; + } + return { content: [toolResultPart] }; + } +} diff --git a/dexto/packages/core/src/llm/index.ts b/dexto/packages/core/src/llm/index.ts new file mode 100644 index 00000000..c897de0c --- /dev/null +++ b/dexto/packages/core/src/llm/index.ts @@ -0,0 +1,18 @@ +export * from './errors.js'; +export * from './error-codes.js'; +export * from './registry.js'; +export * from './validation.js'; +export * from './types.js'; +export * from './services/index.js'; +export * from './schemas.js'; +export { + lookupOpenRouterModel, + refreshOpenRouterModelCache, + getOpenRouterModelContextLength, + getOpenRouterModelInfo, + type LookupStatus, + type OpenRouterModelInfo, +} from './providers/openrouter-model-registry.js'; + +// Local model providers +export * from './providers/local/index.js'; diff --git a/dexto/packages/core/src/llm/providers/local/ai-sdk-adapter.ts b/dexto/packages/core/src/llm/providers/local/ai-sdk-adapter.ts new file mode 100644 index 00000000..0b7af62e --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/ai-sdk-adapter.ts @@ -0,0 +1,466 @@ +/** + * Vercel AI SDK adapter for node-llama-cpp. + * + * This module creates a LanguageModelV2 implementation that wraps node-llama-cpp, + * allowing local GGUF models to be used with the Vercel AI SDK. + */ + +/* global ReadableStream, ReadableStreamDefaultController */ + +import type { + LanguageModelV2, + LanguageModelV2CallOptions, + LanguageModelV2StreamPart, + LanguageModelV2Content, + LanguageModelV2FinishReason, + LanguageModelV2Usage, + LanguageModelV2CallWarning, +} from '@ai-sdk/provider'; +import { + loadModel, + isNodeLlamaCppInstalled, + type ModelSession, + type LoadedModel, +} from './node-llama-provider.js'; +import { LocalModelError } from './errors.js'; +import { getLocalModelById } from './registry.js'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +/** + * Configuration for the local model AI SDK adapter. + */ +export interface LocalModelAdapterConfig { + /** Model ID from the local registry */ + modelId: string; + /** Direct path to model file (optional, overrides modelId lookup) */ + modelPath?: string; + /** Context window size (default: 4096) */ + contextSize?: number; + /** Number of GPU layers to offload (-1 = all, 0 = CPU only) */ + gpuLayers?: number; + /** Number of CPU threads */ + threads?: number; +} + +/** + * Installed model info structure (matches agent-management schema) + */ +interface InstalledModelInfo { + id: string; + filePath: string; + sizeBytes: number; + downloadedAt: string; +} + +/** + * Model state structure (matches agent-management schema) + */ +interface ModelState { + version: string; + installed: Record; + activeModelId?: string; +} + +/** + * Get the models directory path. + */ +function getModelsDirectory(): string { + return path.join(os.homedir(), '.dexto', 'models'); +} + +/** + * Read installed models from state file. + * This is a standalone implementation that doesn't depend on agent-management. + */ +function getInstalledModelInfo(modelId: string): InstalledModelInfo | null { + const stateFile = path.join(getModelsDirectory(), 'state.json'); + + try { + if (!fs.existsSync(stateFile)) { + return null; + } + + const content = fs.readFileSync(stateFile, 'utf-8'); + const state: ModelState = JSON.parse(content); + + return state.installed[modelId] ?? null; + } catch { + return null; + } +} + +/** + * Custom model info structure (matches agent-management schema) + */ +interface CustomModelInfo { + name: string; + provider: string; + filePath?: string; + displayName?: string; + maxInputTokens?: number; +} + +/** + * Custom models storage structure + */ +interface CustomModelsStorage { + version: number; + models: CustomModelInfo[]; +} + +/** + * Read custom models from custom-models.json. + * This is a standalone implementation that doesn't depend on agent-management. + * Used to resolve custom GGUF file paths for local models. + */ +function getCustomModelFilePath(modelId: string): string | null { + const customModelsFile = path.join(getModelsDirectory(), 'custom-models.json'); + + try { + if (!fs.existsSync(customModelsFile)) { + return null; + } + + const content = fs.readFileSync(customModelsFile, 'utf-8'); + const storage: CustomModelsStorage = JSON.parse(content); + + // Find a custom model with matching name and local provider + const customModel = storage.models.find( + (m) => m.name === modelId && m.provider === 'local' && m.filePath + ); + + return customModel?.filePath ?? null; + } catch { + return null; + } +} + +/** + * Create a Vercel AI SDK compatible LanguageModelV2 from a local GGUF model. + * This is a synchronous function that returns a LanguageModel with lazy initialization. + * The actual model loading happens on first use. + */ +export function createLocalLanguageModel(config: LocalModelAdapterConfig): LanguageModelV2 { + return new LocalLanguageModel(config); +} + +/** + * LanguageModelV2 implementation for local GGUF models. + * Uses lazy initialization - model is loaded on first use. + */ +class LocalLanguageModel implements LanguageModelV2 { + readonly specificationVersion = 'v2' as const; + readonly provider = 'local'; + readonly modelId: string; + + // Local models don't support URL-based content natively + readonly supportedUrls: Record = {}; + + private config: LocalModelAdapterConfig; + private session: ModelSession | null = null; + private loadedModel: LoadedModel | null = null; + private initPromise: Promise | null = null; + private deviceName: string = 'Local'; + + constructor(config: LocalModelAdapterConfig) { + this.modelId = config.modelId; + this.config = config; + } + + /** + * Initialize the model lazily on first use. + */ + private async ensureInitialized(): Promise { + if (this.session) { + return; + } + + if (this.initPromise) { + return this.initPromise; + } + + this.initPromise = this.initialize(); + return this.initPromise; + } + + private async initialize(): Promise { + const { + modelId, + modelPath: directPath, + contextSize, // Let node-llama-cpp default to "auto" if not specified + gpuLayers = -1, + threads, + } = this.config; + + // Check if node-llama-cpp is installed + const isInstalled = await isNodeLlamaCppInstalled(); + if (!isInstalled) { + throw LocalModelError.nodeLlamaNotInstalled(); + } + + // Resolve model path + let modelPath: string; + + if (directPath) { + // Use directly provided path + modelPath = directPath; + } else { + // Look up installed model by ID (from state.json - downloaded models) + const installedModel = getInstalledModelInfo(modelId); + if (installedModel) { + modelPath = installedModel.filePath; + } else { + // Check custom models (from custom-models.json - user-provided GGUF paths) + const customPath = getCustomModelFilePath(modelId); + if (customPath) { + modelPath = customPath; + } else { + // Try to get from registry for a better error message + const registryModel = getLocalModelById(modelId); + if (!registryModel) { + throw LocalModelError.modelNotFound(modelId); + } + throw LocalModelError.modelNotDownloaded(modelId); + } + } + } + + // Build config object, only including optional fields if defined + const loadConfig: { + modelPath: string; + contextSize?: number; + gpuLayers: number; + threads?: number; + } = { + modelPath, + gpuLayers, + }; + + if (contextSize !== undefined) { + loadConfig.contextSize = contextSize; + } + if (threads !== undefined) { + loadConfig.threads = threads; + } + + // Load the model + this.loadedModel = await loadModel(loadConfig); + + this.deviceName = this.loadedModel.gpuInfo.deviceName || 'Local'; + + // Create a session for this model + this.session = await this.loadedModel.createSession(); + } + + /** + * Non-streaming text generation (V2 interface). + */ + async doGenerate(options: LanguageModelV2CallOptions) { + await this.ensureInitialized(); + + const prompt = this.formatPrompt(options); + const maxTokens = options.maxOutputTokens ?? 1024; + const temperature = options.temperature ?? 0.7; + + // Build prompt options, only including signal if defined + const promptOptions: { + maxTokens: number; + temperature: number; + signal?: AbortSignal; + } = { + maxTokens, + temperature, + }; + + if (options.abortSignal) { + promptOptions.signal = options.abortSignal; + } + + const response = await this.session!.prompt(prompt, promptOptions); + + // Estimate token counts (rough approximation) + const inputTokens = Math.ceil(prompt.length / 4); + const outputTokens = Math.ceil(response.length / 4); + + const content: LanguageModelV2Content[] = [{ type: 'text', text: response }]; + const finishReason: LanguageModelV2FinishReason = 'stop'; + const usage: LanguageModelV2Usage = { + inputTokens, + outputTokens, + totalTokens: inputTokens + outputTokens, + }; + const warnings: LanguageModelV2CallWarning[] = []; + + return { + content, + finishReason, + usage, + providerMetadata: { + local: { + device: this.deviceName, + }, + }, + warnings, + }; + } + + /** + * Streaming text generation (V2 interface). + */ + async doStream(options: LanguageModelV2CallOptions) { + await this.ensureInitialized(); + + const prompt = this.formatPrompt(options); + const maxTokens = options.maxOutputTokens ?? 1024; + const temperature = options.temperature ?? 0.7; + + const inputTokens = Math.ceil(prompt.length / 4); + let outputTokens = 0; + + const session = this.session!; + const textId = 'text-0'; + + // Build prompt options for streaming + const streamPromptOptions: { + maxTokens: number; + temperature: number; + signal?: AbortSignal; + onToken: (token: string) => void; + } = { + maxTokens, + temperature, + onToken: (_token: string) => { + // Will be set up in the stream + }, + }; + + if (options.abortSignal) { + streamPromptOptions.signal = options.abortSignal; + } + + // Need to capture controller reference for the onToken callback + let controller: ReadableStreamDefaultController; + + const stream = new ReadableStream({ + async start(ctrl) { + controller = ctrl; + + // Emit stream-start + controller.enqueue({ + type: 'stream-start', + warnings: [], + }); + + // Emit text-start + controller.enqueue({ + type: 'text-start', + id: textId, + }); + + try { + // Set up the onToken callback to emit text-delta + streamPromptOptions.onToken = (token: string) => { + outputTokens += 1; + controller.enqueue({ + type: 'text-delta', + id: textId, + delta: token, + }); + }; + + await session.prompt(prompt, streamPromptOptions); + + // Emit text-end + controller.enqueue({ + type: 'text-end', + id: textId, + }); + + // Send finish event + controller.enqueue({ + type: 'finish', + finishReason: 'stop', + usage: { + inputTokens, + outputTokens, + totalTokens: inputTokens + outputTokens, + }, + }); + + controller.close(); + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + // Emit text-end on abort + controller.enqueue({ + type: 'text-end', + id: textId, + }); + + controller.enqueue({ + type: 'finish', + finishReason: 'stop', + usage: { + inputTokens, + outputTokens, + totalTokens: inputTokens + outputTokens, + }, + }); + controller.close(); + } else { + controller.enqueue({ + type: 'error', + error, + }); + controller.close(); + } + } + }, + }); + + return { + stream, + }; + } + + /** + * Format the prompt from AI SDK message format. + */ + private formatPrompt(options: LanguageModelV2CallOptions): string { + const parts: string[] = []; + + // Handle prompt messages + if (options.prompt && Array.isArray(options.prompt)) { + for (const message of options.prompt) { + if (message.role === 'system') { + // System message content is a string + parts.push(`System: ${message.content}`); + } else if (message.role === 'user') { + // User message content is an array of parts + if (Array.isArray(message.content)) { + const textParts = message.content + .filter((p): p is { type: 'text'; text: string } => p.type === 'text') + .map((p) => p.text); + if (textParts.length > 0) { + parts.push(`User: ${textParts.join('\n')}`); + } + } + } else if (message.role === 'assistant') { + // Assistant message content is an array of parts + if (Array.isArray(message.content)) { + const textParts = message.content + .filter((p): p is { type: 'text'; text: string } => p.type === 'text') + .map((p) => p.text); + if (textParts.length > 0) { + parts.push(`Assistant: ${textParts.join('\n')}`); + } + } + } + } + } + + parts.push('Assistant:'); + return parts.join('\n\n'); + } +} diff --git a/dexto/packages/core/src/llm/providers/local/downloader.ts b/dexto/packages/core/src/llm/providers/local/downloader.ts new file mode 100644 index 00000000..5d030a0b --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/downloader.ts @@ -0,0 +1,436 @@ +/** + * Model downloader for local GGUF models. + * + * Downloads models from HuggingFace with: + * - Progress tracking via events + * - Resume support for interrupted downloads + * - Hash verification after download + */ + +import { createWriteStream, promises as fs, existsSync, createReadStream } from 'fs'; +import { createHash } from 'crypto'; +import * as path from 'path'; +import type { ModelDownloadProgress, ModelDownloadStatus } from './types.js'; +import { LocalModelError } from './errors.js'; +import { getLocalModelById } from './registry.js'; + +/** + * Event emitter interface for download progress. + */ +export interface DownloadEvents { + onProgress?: (progress: ModelDownloadProgress) => void; + onComplete?: (modelId: string, filePath: string) => void; + onError?: (modelId: string, error: Error) => void; +} + +/** + * Download options. + */ +export interface DownloadOptions { + /** Directory to save the model */ + targetDir: string; + /** Events for progress tracking */ + events?: DownloadEvents; + /** HuggingFace token for gated models */ + hfToken?: string; + /** Whether to verify hash after download */ + verifyHash?: boolean; + /** Abort signal for cancellation */ + signal?: AbortSignal; + /** Expected SHA-256 hash for verification */ + expectedHash?: string; +} + +/** + * Download result. + */ +export interface DownloadResult { + /** Whether download succeeded */ + success: boolean; + /** Full path to downloaded file */ + filePath: string; + /** File size in bytes */ + sizeBytes: number; + /** SHA-256 hash of the file */ + sha256?: string; + /** Whether download was resumed from partial */ + resumed: boolean; +} + +/** + * Build the HuggingFace download URL for a model file. + */ +function buildHuggingFaceUrl(huggingfaceId: string, filename: string): string { + // HuggingFace URL format: https://huggingface.co/{repo}/resolve/main/{filename} + return `https://huggingface.co/${huggingfaceId}/resolve/main/${filename}`; +} + +/** + * Get the size of a partial download file. + */ +async function getPartialSize(filePath: string): Promise { + try { + const stats = await fs.stat(filePath); + return stats.size; + } catch { + return 0; + } +} + +/** + * Calculate SHA-256 hash of a file. + */ +export async function calculateFileHash(filePath: string): Promise { + return new Promise((resolve, reject) => { + const hash = createHash('sha256'); + const stream = createReadStream(filePath); + + stream.on('data', (chunk) => hash.update(chunk)); + stream.on('end', () => resolve(hash.digest('hex'))); + stream.on('error', reject); + }); +} + +/** + * Create a progress event object. + */ +function createProgressEvent( + modelId: string, + status: ModelDownloadStatus, + bytesDownloaded: number, + totalBytes: number, + speed?: number, + eta?: number, + error?: string +): ModelDownloadProgress { + const progress: ModelDownloadProgress = { + modelId, + status, + bytesDownloaded, + totalBytes, + percentage: totalBytes > 0 ? (bytesDownloaded / totalBytes) * 100 : 0, + }; + + if (speed !== undefined) { + progress.speed = speed; + } + if (eta !== undefined) { + progress.eta = eta; + } + if (error !== undefined) { + progress.error = error; + } + + return progress; +} + +/** + * Download a model from HuggingFace. + */ +async function downloadFromHuggingFace( + url: string, + targetPath: string, + options: DownloadOptions, + modelId: string, + expectedSize: number +): Promise { + const { events, hfToken, signal } = options; + + // Check for partial download to support resume + const tempPath = `${targetPath}.download`; + const partialSize = await getPartialSize(tempPath); + const resumed = partialSize > 0; + + const headers: Record = { + 'User-Agent': 'Dexto/1.0', + }; + + // Add auth token for gated models + if (hfToken) { + headers['Authorization'] = `Bearer ${hfToken}`; + } + + // Add range header for resume + if (partialSize > 0) { + headers['Range'] = `bytes=${partialSize}-`; + } + + try { + // Build fetch options - only include signal if provided + const fetchOptions: RequestInit = { headers }; + if (signal) { + fetchOptions.signal = signal; + } + + const response = await fetch(url, fetchOptions); + + // Check for auth errors (gated models) + if (response.status === 401 || response.status === 403) { + throw LocalModelError.hfAuthRequired(modelId); + } + + if (!response.ok && response.status !== 206) { + throw LocalModelError.downloadFailed( + modelId, + `HTTP ${response.status}: ${response.statusText}` + ); + } + + // Get content length for progress tracking + const contentLengthHeader = response.headers.get('content-length'); + const contentLength = contentLengthHeader ? parseInt(contentLengthHeader, 10) : 0; + const totalSize = partialSize + contentLength; + + // Ensure target directory exists + await fs.mkdir(path.dirname(tempPath), { recursive: true }); + + // Open file for writing (append if resuming) + const writeStream = createWriteStream(tempPath, { + flags: resumed ? 'a' : 'w', + }); + + // Track download progress + let bytesDownloaded = partialSize; + const startTime = Date.now(); + let lastProgressUpdate = startTime; + + const reader = response.body?.getReader(); + if (!reader) { + writeStream.destroy(); + throw LocalModelError.downloadFailed(modelId, 'No response body'); + } + + try { + // Read and write chunks + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + writeStream.write(value); + bytesDownloaded += value.length; + + // Emit progress every 100ms + const now = Date.now(); + if (now - lastProgressUpdate > 100 || done) { + lastProgressUpdate = now; + const elapsedSeconds = (now - startTime) / 1000; + const speed = + elapsedSeconds > 0 ? (bytesDownloaded - partialSize) / elapsedSeconds : 0; + const remainingBytes = totalSize - bytesDownloaded; + const eta = speed > 0 ? remainingBytes / speed : 0; + + const progress = createProgressEvent( + modelId, + 'downloading', + bytesDownloaded, + totalSize || expectedSize, + speed, + eta + ); + + events?.onProgress?.(progress); + } + } + + // Close write stream + await new Promise((resolve, reject) => { + writeStream.end((err: Error | null | undefined) => { + if (err) reject(err); + else resolve(); + }); + }); + } catch (error) { + writeStream.destroy(); + throw error; + } + + // Emit verifying status + events?.onProgress?.(createProgressEvent(modelId, 'verifying', bytesDownloaded, totalSize)); + + // Rename temp file to final path + await fs.rename(tempPath, targetPath); + + // Get final file size + const stats = await fs.stat(targetPath); + + // Emit complete status + events?.onProgress?.(createProgressEvent(modelId, 'complete', stats.size, stats.size)); + + return { + success: true, + filePath: targetPath, + sizeBytes: stats.size, + resumed, + }; + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + throw LocalModelError.downloadInterrupted(modelId); + } + throw error; + } +} + +/** + * Download a local model by ID. + */ +export async function downloadModel( + modelId: string, + options: DownloadOptions +): Promise { + const modelInfo = getLocalModelById(modelId); + if (!modelInfo) { + throw LocalModelError.modelNotFound(modelId); + } + + const targetPath = path.join(options.targetDir, modelInfo.filename); + const url = buildHuggingFaceUrl(modelInfo.huggingfaceId, modelInfo.filename); + + // Check if file already exists + if (existsSync(targetPath)) { + const stats = await fs.stat(targetPath); + // Verify size matches expected + if (stats.size === modelInfo.sizeBytes) { + return { + success: true, + filePath: targetPath, + sizeBytes: stats.size, + resumed: false, + }; + } + // Delete partial/corrupt file + await fs.unlink(targetPath); + } + + try { + // Emit pending status + options.events?.onProgress?.( + createProgressEvent(modelId, 'pending', 0, modelInfo.sizeBytes) + ); + + const result = await downloadFromHuggingFace( + url, + targetPath, + options, + modelId, + modelInfo.sizeBytes + ); + + // Verify hash if requested and expected hash is provided + if (options.verifyHash && options.expectedHash) { + const actualHash = await calculateFileHash(targetPath); + if (actualHash !== options.expectedHash) { + // Delete corrupted file + await fs.unlink(targetPath); + throw LocalModelError.hashMismatch(modelId, options.expectedHash, actualHash); + } + result.sha256 = actualHash; + } + + options.events?.onComplete?.(modelId, targetPath); + return result; + } catch (error) { + options.events?.onError?.(modelId, error as Error); + throw error; + } +} + +/** + * Download a model directly from a URL (for custom models). + */ +export async function downloadModelFromUrl( + modelId: string, + url: string, + filename: string, + options: DownloadOptions +): Promise { + const targetPath = path.join(options.targetDir, filename); + + try { + // Emit pending status + options.events?.onProgress?.(createProgressEvent(modelId, 'pending', 0, 0)); + + const result = await downloadFromHuggingFace(url, targetPath, options, modelId, 0); + options.events?.onComplete?.(modelId, targetPath); + return result; + } catch (error) { + options.events?.onError?.(modelId, error as Error); + throw error; + } +} + +/** + * Check available disk space at a path. + */ +export async function checkDiskSpace(targetDir: string): Promise { + // This is a simplified check - in production, use a library like check-disk-space + // For now, we'll return a large value and let the OS handle space errors + try { + await fs.access(targetDir); + return Number.MAX_SAFE_INTEGER; + } catch { + // Directory doesn't exist, try to create it to check permissions + try { + await fs.mkdir(targetDir, { recursive: true }); + return Number.MAX_SAFE_INTEGER; + } catch { + return 0; + } + } +} + +/** + * Validate that there's enough disk space for a model. + */ +export async function validateDiskSpace( + modelId: string, + requiredBytes: number, + targetDir: string +): Promise { + const available = await checkDiskSpace(targetDir); + if (available < requiredBytes) { + throw LocalModelError.insufficientDiskSpace(modelId, requiredBytes, available); + } +} + +/** + * Clean up partial download files. + */ +export async function cleanupPartialDownload(targetDir: string, filename: string): Promise { + const tempPath = path.join(targetDir, `${filename}.download`); + try { + await fs.unlink(tempPath); + } catch { + // Ignore if file doesn't exist + } +} + +/** + * Check if a download is in progress (partial file exists). + */ +export async function isDownloadInProgress(targetDir: string, filename: string): Promise { + const tempPath = path.join(targetDir, `${filename}.download`); + try { + await fs.access(tempPath); + return true; + } catch { + return false; + } +} + +/** + * Get the progress of a partial download. + */ +export async function getPartialDownloadProgress( + modelId: string, + targetDir: string, + filename: string, + totalBytes: number +): Promise { + const tempPath = path.join(targetDir, `${filename}.download`); + try { + const stats = await fs.stat(tempPath); + return createProgressEvent(modelId, 'downloading', stats.size, totalBytes); + } catch { + return null; + } +} diff --git a/dexto/packages/core/src/llm/providers/local/error-codes.ts b/dexto/packages/core/src/llm/providers/local/error-codes.ts new file mode 100644 index 00000000..0e9478e9 --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/error-codes.ts @@ -0,0 +1,74 @@ +/** + * Error codes for local model operations. + * Format: LOCAL_XXX where XXX groups by category: + * - 001-009: Installation errors + * - 010-019: Download errors + * - 020-029: Model errors + * - 030-039: GPU errors + * - 040-049: Ollama errors + */ +export enum LocalModelErrorCode { + // Installation errors (001-009) + /** node-llama-cpp package is not installed */ + NODE_LLAMA_NOT_INSTALLED = 'LOCAL_001', + /** Failed to install node-llama-cpp */ + NODE_LLAMA_INSTALL_FAILED = 'LOCAL_002', + /** CMake not found (required for building from source) */ + CMAKE_NOT_FOUND = 'LOCAL_003', + /** Build from source failed */ + BUILD_FAILED = 'LOCAL_004', + + // Download errors (010-019) + /** Model download failed */ + DOWNLOAD_FAILED = 'LOCAL_010', + /** Download was interrupted */ + DOWNLOAD_INTERRUPTED = 'LOCAL_011', + /** Downloaded file hash doesn't match expected */ + DOWNLOAD_HASH_MISMATCH = 'LOCAL_012', + /** Insufficient disk space for download */ + INSUFFICIENT_DISK_SPACE = 'LOCAL_013', + /** HuggingFace authentication required for gated model */ + HF_AUTH_REQUIRED = 'LOCAL_014', + /** Network error during download */ + NETWORK_ERROR = 'LOCAL_015', + + // Model errors (020-029) + /** Model not found in registry */ + MODEL_NOT_FOUND = 'LOCAL_020', + /** Model not downloaded locally */ + MODEL_NOT_DOWNLOADED = 'LOCAL_021', + /** Failed to load model */ + MODEL_LOAD_FAILED = 'LOCAL_022', + /** Model file is corrupted */ + MODEL_CORRUPT = 'LOCAL_023', + /** Invalid GGUF format */ + INVALID_GGUF = 'LOCAL_024', + /** Model context too large for available memory */ + CONTEXT_TOO_LARGE = 'LOCAL_025', + + // GPU errors (030-039) + /** No GPU acceleration available */ + GPU_NOT_AVAILABLE = 'LOCAL_030', + /** Insufficient VRAM for model */ + INSUFFICIENT_VRAM = 'LOCAL_031', + /** GPU driver error */ + GPU_DRIVER_ERROR = 'LOCAL_032', + /** Metal not available (macOS only) */ + METAL_NOT_AVAILABLE = 'LOCAL_033', + /** CUDA not available */ + CUDA_NOT_AVAILABLE = 'LOCAL_034', + /** Vulkan not available */ + VULKAN_NOT_AVAILABLE = 'LOCAL_035', + + // Ollama errors (040-049) + /** Ollama server is not running */ + OLLAMA_NOT_RUNNING = 'LOCAL_040', + /** Model not found on Ollama server */ + OLLAMA_MODEL_NOT_FOUND = 'LOCAL_041', + /** Failed to pull model from Ollama */ + OLLAMA_PULL_FAILED = 'LOCAL_042', + /** Ollama API error */ + OLLAMA_API_ERROR = 'LOCAL_043', + /** Ollama version incompatible */ + OLLAMA_VERSION_INCOMPATIBLE = 'LOCAL_044', +} diff --git a/dexto/packages/core/src/llm/providers/local/errors.ts b/dexto/packages/core/src/llm/providers/local/errors.ts new file mode 100644 index 00000000..723c5575 --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/errors.ts @@ -0,0 +1,242 @@ +/** + * Error factory for local model errors. + * Follows the project's error factory pattern. + */ + +import { DextoRuntimeError } from '../../../errors/DextoRuntimeError.js'; +import { ErrorType } from '../../../errors/types.js'; +import { LocalModelErrorCode } from './error-codes.js'; + +const SCOPE = 'local-models'; + +/** + * Error factory for local model operations. + */ +export const LocalModelError = { + // Installation errors + nodeLlamaNotInstalled(): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.NODE_LLAMA_NOT_INSTALLED, + SCOPE, + ErrorType.NOT_FOUND, + 'node-llama-cpp is not installed. Run `dexto setup` and select "local" provider to install it.', + {}, + 'Run `dexto setup` and select "local" provider to install local model support' + ); + }, + + nodeLlamaInstallFailed(error: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.NODE_LLAMA_INSTALL_FAILED, + SCOPE, + ErrorType.THIRD_PARTY, + `Failed to install node-llama-cpp: ${error}`, + { error }, + 'Check your Node.js version and try again. CMake may be required for your platform.' + ); + }, + + cmakeNotFound(): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.CMAKE_NOT_FOUND, + SCOPE, + ErrorType.NOT_FOUND, + 'CMake is required to build node-llama-cpp from source but was not found.', + {}, + 'Install CMake: brew install cmake (macOS), apt install cmake (Linux), or download from cmake.org (Windows)' + ); + }, + + // Download errors + downloadFailed(modelId: string, error: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.DOWNLOAD_FAILED, + SCOPE, + ErrorType.THIRD_PARTY, + `Failed to download model '${modelId}': ${error}`, + { modelId, error }, + 'Check your internet connection and try again' + ); + }, + + downloadInterrupted(modelId: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.DOWNLOAD_INTERRUPTED, + SCOPE, + ErrorType.THIRD_PARTY, + `Download of model '${modelId}' was interrupted`, + { modelId }, + 'Run the download command again to resume' + ); + }, + + hashMismatch(modelId: string, expected: string, actual: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.DOWNLOAD_HASH_MISMATCH, + SCOPE, + ErrorType.USER, + `Downloaded model '${modelId}' has invalid hash. Expected: ${expected}, Got: ${actual}`, + { modelId, expected, actual }, + 'Delete the file and download again' + ); + }, + + insufficientDiskSpace(modelId: string, required: number, available: number): DextoRuntimeError { + const requiredGB = (required / (1024 * 1024 * 1024)).toFixed(1); + const availableGB = (available / (1024 * 1024 * 1024)).toFixed(1); + return new DextoRuntimeError( + LocalModelErrorCode.INSUFFICIENT_DISK_SPACE, + SCOPE, + ErrorType.USER, + `Insufficient disk space to download '${modelId}'. Required: ${requiredGB}GB, Available: ${availableGB}GB`, + { modelId, required, available }, + 'Free up disk space or choose a smaller model' + ); + }, + + hfAuthRequired(modelId: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.HF_AUTH_REQUIRED, + SCOPE, + ErrorType.FORBIDDEN, + `Model '${modelId}' is a gated model and requires HuggingFace authentication`, + { modelId }, + 'Set HF_TOKEN environment variable or run `huggingface-cli login`' + ); + }, + + // Model errors + modelNotFound(modelId: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.MODEL_NOT_FOUND, + SCOPE, + ErrorType.NOT_FOUND, + `Model '${modelId}' not found in local model registry`, + { modelId }, + 'Run `dexto setup` and select "local" to see available models' + ); + }, + + modelNotDownloaded(modelId: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.MODEL_NOT_DOWNLOADED, + SCOPE, + ErrorType.NOT_FOUND, + `Model '${modelId}' is not downloaded. Download it first.`, + { modelId }, + 'Run `dexto setup` and select "local" to download models' + ); + }, + + modelLoadFailed(modelId: string, error: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.MODEL_LOAD_FAILED, + SCOPE, + ErrorType.THIRD_PARTY, + `Failed to load model '${modelId}': ${error}`, + { modelId, error }, + 'The model file may be corrupted. Try re-downloading it.' + ); + }, + + modelCorrupt(modelId: string, filePath: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.MODEL_CORRUPT, + SCOPE, + ErrorType.USER, + `Model file for '${modelId}' appears to be corrupted`, + { modelId, filePath }, + `Delete ${filePath} and download the model again` + ); + }, + + contextTooLarge(modelId: string, requested: number, maxSupported: number): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.CONTEXT_TOO_LARGE, + SCOPE, + ErrorType.USER, + `Requested context size ${requested} exceeds model's maximum of ${maxSupported}`, + { modelId, requested, maxSupported }, + `Use a context size of ${maxSupported} or less` + ); + }, + + // GPU errors + gpuNotAvailable(): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.GPU_NOT_AVAILABLE, + SCOPE, + ErrorType.NOT_FOUND, + 'No GPU acceleration available. Running on CPU.', + {}, + 'For better performance, ensure GPU drivers are installed' + ); + }, + + insufficientVRAM(modelId: string, required: number, available: number): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.INSUFFICIENT_VRAM, + SCOPE, + ErrorType.USER, + `Model '${modelId}' requires ${required}GB VRAM but only ${available}GB available`, + { modelId, required, available }, + 'Use a smaller quantization or reduce GPU layers' + ); + }, + + gpuDriverError(error: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.GPU_DRIVER_ERROR, + SCOPE, + ErrorType.THIRD_PARTY, + `GPU driver error: ${error}`, + { error }, + 'Update your GPU drivers' + ); + }, + + // Ollama errors + ollamaNotRunning(url: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.OLLAMA_NOT_RUNNING, + SCOPE, + ErrorType.THIRD_PARTY, + `Ollama server is not running at ${url}`, + { url }, + 'Start Ollama with `ollama serve` or ensure it is running' + ); + }, + + ollamaModelNotFound(modelName: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.OLLAMA_MODEL_NOT_FOUND, + SCOPE, + ErrorType.NOT_FOUND, + `Model '${modelName}' not found on Ollama server`, + { modelName }, + `Pull the model with \`ollama pull ${modelName}\`` + ); + }, + + ollamaPullFailed(modelName: string, error: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.OLLAMA_PULL_FAILED, + SCOPE, + ErrorType.THIRD_PARTY, + `Failed to pull model '${modelName}' from Ollama: ${error}`, + { modelName, error }, + 'Check your internet connection and Ollama server status' + ); + }, + + ollamaApiError(error: string): DextoRuntimeError { + return new DextoRuntimeError( + LocalModelErrorCode.OLLAMA_API_ERROR, + SCOPE, + ErrorType.THIRD_PARTY, + `Ollama API error: ${error}`, + { error }, + 'Check Ollama server logs for details' + ); + }, +}; diff --git a/dexto/packages/core/src/llm/providers/local/gpu-detector.ts b/dexto/packages/core/src/llm/providers/local/gpu-detector.ts new file mode 100644 index 00000000..8dd423f4 --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/gpu-detector.ts @@ -0,0 +1,266 @@ +/** + * GPU detection for local model acceleration. + * + * Detects available GPU backends: + * - Metal: Apple Silicon (M1/M2/M3/M4 series) + * - CUDA: NVIDIA GPUs on Linux/Windows + * - Vulkan: Cross-platform fallback for AMD/Intel GPUs + * - CPU: Fallback when no GPU is available + */ + +import { exec } from 'child_process'; +import { promisify } from 'util'; +import * as os from 'os'; +import type { GPUBackend, GPUInfo } from './types.js'; + +const execAsync = promisify(exec); + +/** + * Detect available GPU backend for the current system. + * Returns the best available option. + */ +export async function detectGPU(): Promise { + const platform = os.platform(); + + // macOS: Check for Metal (Apple Silicon or discrete GPU) + if (platform === 'darwin') { + const metalInfo = await detectMetal(); + if (metalInfo.available) { + return metalInfo; + } + } + + // Linux/Windows: Check for CUDA (NVIDIA) + if (platform === 'linux' || platform === 'win32') { + const cudaInfo = await detectCUDA(); + if (cudaInfo.available) { + return cudaInfo; + } + + // Fallback to Vulkan + const vulkanInfo = await detectVulkan(); + if (vulkanInfo.available) { + return vulkanInfo; + } + } + + // Default to CPU + return { + backend: 'cpu', + available: true, + deviceName: `${os.cpus()[0]?.model ?? 'Unknown CPU'}`, + }; +} + +/** + * Detect Metal GPU on macOS. + */ +async function detectMetal(): Promise { + try { + // Use system_profiler to get GPU info on macOS + const { stdout } = await execAsync('system_profiler SPDisplaysDataType -json 2>/dev/null'); + const data = JSON.parse(stdout); + const gpuData = data?.SPDisplaysDataType?.[0]; + + if (gpuData) { + const chipName = gpuData.sppci_model ?? gpuData._name ?? 'Apple GPU'; + const isAppleSilicon = + chipName.toLowerCase().includes('apple') || + chipName.toLowerCase().includes('m1') || + chipName.toLowerCase().includes('m2') || + chipName.toLowerCase().includes('m3') || + chipName.toLowerCase().includes('m4'); + + // Apple Silicon has unified memory, so VRAM = system RAM + // For discrete GPUs, try to parse VRAM + const result: GPUInfo = { + backend: 'metal', + available: true, + deviceName: chipName, + }; + + if (isAppleSilicon) { + // Unified memory - use total system memory + result.vramMB = Math.round(os.totalmem() / (1024 * 1024)); + } else if (gpuData.sppci_vram) { + // Parse VRAM string like "8 GB" + const vramMatch = gpuData.sppci_vram.match(/(\d+)\s*(GB|MB)/i); + if (vramMatch) { + result.vramMB = + parseInt(vramMatch[1]!) * (vramMatch[2]!.toUpperCase() === 'GB' ? 1024 : 1); + } + } + + return result; + } + } catch { + // Ignore errors - Metal not available + } + + return { + backend: 'metal', + available: false, + }; +} + +/** + * Detect NVIDIA CUDA GPU. + */ +async function detectCUDA(): Promise { + try { + // Use nvidia-smi to detect NVIDIA GPU + const { stdout } = await execAsync( + 'nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader,nounits 2>/dev/null' + ); + + const lines = stdout.trim().split('\n'); + if (lines.length > 0 && lines[0]) { + const [name, memoryMB, driverVersion] = lines[0].split(', ').map((s) => s.trim()); + + const result: GPUInfo = { + backend: 'cuda', + available: true, + }; + + if (name) { + result.deviceName = name; + } + if (memoryMB) { + result.vramMB = parseInt(memoryMB); + } + if (driverVersion) { + result.driverVersion = driverVersion; + } + + return result; + } + } catch { + // nvidia-smi not available or no NVIDIA GPU + } + + return { + backend: 'cuda', + available: false, + }; +} + +/** + * Detect Vulkan GPU support. + */ +async function detectVulkan(): Promise { + try { + // Try vulkaninfo command (available when Vulkan SDK is installed) + const { stdout } = await execAsync('vulkaninfo --summary 2>/dev/null'); + + // Parse device name from vulkaninfo output + const deviceMatch = stdout.match(/deviceName\s*=\s*(.+)/); + const deviceName = deviceMatch?.[1]?.trim() ?? 'Vulkan GPU'; + + const result: GPUInfo = { + backend: 'vulkan', + available: true, + deviceName, + }; + + // Parse VRAM if available + const heapMatch = stdout.match(/heapSize\s*=\s*(\d+)/); + if (heapMatch) { + result.vramMB = Math.round(parseInt(heapMatch[1]!) / (1024 * 1024)); + } + + return result; + } catch { + // vulkaninfo not available + } + + // Fallback: Check for common AMD/Intel GPU indicators on Linux + if (os.platform() === 'linux') { + try { + const { stdout } = await execAsync('lspci | grep -i "vga\\|3d\\|display" 2>/dev/null'); + if (stdout.includes('AMD') || stdout.includes('Intel') || stdout.includes('Radeon')) { + // GPU detected but Vulkan tools not installed + const deviceMatch = stdout.match(/: (.+)/); + return { + backend: 'vulkan', + available: true, + deviceName: deviceMatch?.[1]?.trim() ?? 'GPU (Vulkan)', + }; + } + } catch { + // lspci not available + } + } + + return { + backend: 'vulkan', + available: false, + }; +} + +/** + * Get a human-readable summary of GPU detection results. + */ +export function formatGPUInfo(info: GPUInfo): string { + if (!info.available) { + return `${info.backend.toUpperCase()} not available`; + } + + const parts = [info.deviceName ?? info.backend.toUpperCase()]; + + if (info.vramMB) { + const vramGB = (info.vramMB / 1024).toFixed(1); + parts.push(`${vramGB}GB`); + } + + if (info.driverVersion) { + parts.push(`Driver: ${info.driverVersion}`); + } + + return parts.join(' • '); +} + +/** + * Check if a specific backend is available. + */ +export async function isBackendAvailable(backend: GPUBackend): Promise { + switch (backend) { + case 'metal': + return (await detectMetal()).available; + case 'cuda': + return (await detectCUDA()).available; + case 'vulkan': + return (await detectVulkan()).available; + case 'cpu': + return true; + default: + return false; + } +} + +/** + * Get all available backends on the current system. + */ +export async function getAvailableBackends(): Promise { + const backends: GPUBackend[] = []; + const platform = os.platform(); + + if (platform === 'darwin') { + if ((await detectMetal()).available) { + backends.push('metal'); + } + } + + if (platform === 'linux' || platform === 'win32') { + if ((await detectCUDA()).available) { + backends.push('cuda'); + } + if ((await detectVulkan()).available) { + backends.push('vulkan'); + } + } + + // CPU is always available + backends.push('cpu'); + + return backends; +} diff --git a/dexto/packages/core/src/llm/providers/local/index.ts b/dexto/packages/core/src/llm/providers/local/index.ts new file mode 100644 index 00000000..56d4e24d --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/index.ts @@ -0,0 +1,103 @@ +/** + * Native local model support via node-llama-cpp and Ollama. + * + * This module provides: + * - Local model registry with curated GGUF models + * - GPU detection (Metal/CUDA/Vulkan) + * - Model downloading with progress + * - node-llama-cpp provider for native GGUF execution + * - Ollama provider for Ollama server integration + */ + +// Types +export * from './types.js'; + +// Error codes and factory +export { LocalModelErrorCode } from './error-codes.js'; +export { LocalModelError } from './errors.js'; + +// Schemas +export { + GPUBackendSchema, + QuantizationTypeSchema, + LocalModelCategorySchema, + ModelSourceSchema, + ModelDownloadStatusSchema, + LocalModelInfoSchema, + ModelDownloadProgressSchema, + GPUInfoSchema, + LocalLLMConfigSchema, + InstalledModelSchema, + ModelStateSchema, + ModelDownloadOptionsSchema, + OllamaModelInfoSchema, + OllamaStatusSchema, +} from './schemas.js'; + +// Registry +export { + LOCAL_MODEL_REGISTRY, + getAllLocalModels, + getLocalModelById, + getLocalModelsByCategory, + getRecommendedLocalModels, + getModelsForVRAM, + getModelsForRAM, + searchLocalModels, + getDefaultLocalModelId, +} from './registry.js'; + +// GPU Detection +export { + detectGPU, + formatGPUInfo, + isBackendAvailable, + getAvailableBackends, +} from './gpu-detector.js'; + +// Downloader +export { + downloadModel, + downloadModelFromUrl, + calculateFileHash, + checkDiskSpace, + validateDiskSpace, + cleanupPartialDownload, + isDownloadInProgress, + getPartialDownloadProgress, + type DownloadEvents, + type DownloadOptions, + type DownloadResult, +} from './downloader.js'; + +// Ollama Provider +export { + DEFAULT_OLLAMA_URL, + checkOllamaStatus, + listOllamaModels, + isOllamaModelAvailable, + pullOllamaModel, + createOllamaModel, + createValidatedOllamaModel, + getOllamaModelInfo, + deleteOllamaModel, + generateOllamaEmbeddings, + type OllamaConfig, +} from './ollama-provider.js'; + +// node-llama-cpp Provider +export { + isNodeLlamaCppInstalled, + requireNodeLlamaCpp, + loadModel, + unloadModel, + unloadAllModels, + isModelLoaded, + getLoadedModelCount, + type NodeLlamaConfig, + type ModelSession, + type LoadedModel, +} from './node-llama-provider.js'; + +// AI SDK Adapter +export { createLocalLanguageModel, type LocalModelAdapterConfig } from './ai-sdk-adapter.js'; diff --git a/dexto/packages/core/src/llm/providers/local/node-llama-provider.ts b/dexto/packages/core/src/llm/providers/local/node-llama-provider.ts new file mode 100644 index 00000000..9ae568bd --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/node-llama-provider.ts @@ -0,0 +1,353 @@ +/** + * node-llama-cpp provider for native local model execution. + * + * This module provides utilities for loading and using GGUF models via node-llama-cpp. + * Since node-llama-cpp is an optional dependency, all functions handle the case + * where it's not installed gracefully. + * + * For Vercel AI SDK integration, we recommend using Ollama which provides + * an OpenAI-compatible API that works seamlessly with the SDK. + */ + +import type { GPUInfo } from './types.js'; +import { LocalModelError } from './errors.js'; +import { detectGPU } from './gpu-detector.js'; +import { getDextoGlobalPath } from '../../../utils/path.js'; +import { createRequire } from 'module'; +import * as path from 'path'; + +/** + * Get the global deps path where node-llama-cpp may be installed. + */ +function getGlobalNodeLlamaCppPath(): string { + return path.join(getDextoGlobalPath('deps'), 'node_modules', 'node-llama-cpp'); +} + +/** + * Check if node-llama-cpp is installed. + * Checks both standard node resolution (for dev/projects) and global deps (~/.dexto/deps). + */ +export async function isNodeLlamaCppInstalled(): Promise { + // Try 1: Standard node resolution (works in dev mode, dexto-project with local install) + try { + // @ts-ignore - Optional dependency may not be installed (TS2307 in CI) + await import('node-llama-cpp'); + return true; + } catch { + // Continue to fallback + } + + // Try 2: Global deps location (~/.dexto/deps/node_modules/node-llama-cpp) + try { + const globalPath = getGlobalNodeLlamaCppPath(); + const require = createRequire(import.meta.url); + require.resolve(globalPath); + return true; + } catch { + return false; + } +} + +/** + * Dynamically import node-llama-cpp. + * Returns null if not installed. + * Checks both standard node resolution and global deps (~/.dexto/deps). + */ +// Using Record type for dynamic import result since we can't type node-llama-cpp at compile time +async function importNodeLlamaCpp(): Promise | null> { + // Try 1: Standard node resolution (works in dev mode, dexto-project with local install) + try { + // @ts-ignore - Optional dependency may not be installed (TS2307 in CI) + return await import('node-llama-cpp'); + } catch { + // Continue to fallback + } + + // Try 2: Global deps location (~/.dexto/deps/node_modules/node-llama-cpp) + try { + const globalPath = getGlobalNodeLlamaCppPath(); + // Use dynamic import with full path to entry point (ES modules don't support directory imports) + const entryPoint = path.join(globalPath, 'dist', 'index.js'); + // @ts-ignore - Dynamic path import + return await import(entryPoint); + } catch { + return null; + } +} + +/** + * Throws an error indicating node-llama-cpp needs to be installed. + */ +export function requireNodeLlamaCpp(): never { + throw LocalModelError.nodeLlamaNotInstalled(); +} + +/** + * Configuration for the node-llama-cpp model. + */ +export interface NodeLlamaConfig { + /** Path to the .gguf model file */ + modelPath: string; + /** Number of GPU layers to offload (-1 = all, 0 = CPU only) */ + gpuLayers?: number; + /** Context window size */ + contextSize?: number; + /** Number of CPU threads */ + threads?: number; + /** Batch size for inference */ + batchSize?: number; + /** Whether to use Flash Attention (if available) */ + flashAttention?: boolean; +} + +/** + * Model session interface for node-llama-cpp. + * This provides a simplified interface for text generation. + */ +export interface ModelSession { + /** Generate a response from a prompt */ + prompt( + text: string, + options?: { + maxTokens?: number; + temperature?: number; + topP?: number; + signal?: AbortSignal; + onToken?: (token: string) => void; + } + ): Promise; + + /** Dispose the session and free resources */ + dispose(): Promise; +} + +/** + * Loaded model interface. + */ +export interface LoadedModel { + /** Model file path */ + modelPath: string; + /** GPU info used for loading */ + gpuInfo: GPUInfo; + /** Create a new chat session */ + createSession(): Promise; + /** Dispose the model and free resources */ + dispose(): Promise; +} + +// Cache for loaded models +const modelCache = new Map>(); + +/** + * Load a GGUF model using node-llama-cpp. + * + * @throws {DextoRuntimeError} If node-llama-cpp is not installed + */ +export async function loadModel(config: NodeLlamaConfig): Promise { + const { modelPath, gpuLayers = -1, contextSize, threads, batchSize = 512 } = config; + + // Check cache first + const cacheKey = `${modelPath}:${gpuLayers}:${contextSize}`; + const cached = modelCache.get(cacheKey); + if (cached) { + return cached; + } + + // Create loading promise + const loadPromise = (async (): Promise => { + // Try to import node-llama-cpp + const nodeLlama = await importNodeLlamaCpp(); + if (!nodeLlama) { + throw LocalModelError.nodeLlamaNotInstalled(); + } + + try { + // Detect GPU for optimal configuration + const gpuInfo = await detectGPU(); + + // Access getLlama from dynamic import (cast to function type) + const getLlama = nodeLlama['getLlama'] as (config: { + logLevel: unknown; + gpu: boolean | string; + }) => Promise<{ + loadModel: (config: { modelPath: string; gpuLayers: number | string }) => Promise<{ + createContext: (options: Record) => Promise<{ + getSequence: () => unknown; + dispose: () => Promise; + }>; + dispose: () => Promise; + }>; + }>; + const LlamaLogLevel = nodeLlama['LlamaLogLevel'] as { warn: unknown }; + const LlamaChatSession = nodeLlama['LlamaChatSession'] as new (options: { + contextSequence: unknown; + }) => { + prompt: ( + text: string, + options: { + maxTokens: number; + temperature: number; + topP: number; + signal?: AbortSignal; + stopOnAbortSignal: boolean; + trimWhitespaceSuffix: boolean; + onTextChunk?: (text: string) => void; + } + ) => Promise; + }; + + // Initialize llama.cpp runtime + const llama = await getLlama({ + logLevel: LlamaLogLevel.warn, + gpu: gpuInfo.backend === 'cpu' ? false : 'auto', + }); + + // Load the model + const model = await llama.loadModel({ + modelPath, + gpuLayers: gpuLayers === -1 ? 'auto' : gpuLayers, + }); + + // Create context with specified options + // contextSize defaults to "auto" in node-llama-cpp, which uses the model's + // training context and auto-retries with smaller sizes on failure + const contextOptions: Record = { + batchSize, + }; + if (contextSize !== undefined) { + contextOptions.contextSize = contextSize; + } + if (threads !== undefined) { + contextOptions.threads = threads; + } + + const context = await model.createContext(contextOptions); + + return { + modelPath, + gpuInfo, + async createSession(): Promise { + const session = new LlamaChatSession({ + contextSequence: context.getSequence(), + }); + + return { + async prompt(text, options = {}): Promise { + const { + maxTokens = 1024, + temperature = 0.7, + topP = 0.9, + signal, + onToken, + } = options; + + // Build options object, only including optional properties if defined + const promptOptions: { + maxTokens: number; + temperature: number; + topP: number; + stopOnAbortSignal: boolean; + trimWhitespaceSuffix: boolean; + signal?: AbortSignal; + onTextChunk?: (text: string) => void; + } = { + maxTokens, + temperature, + topP, + stopOnAbortSignal: true, + trimWhitespaceSuffix: true, + }; + + if (signal) { + promptOptions.signal = signal; + } + if (onToken) { + promptOptions.onTextChunk = onToken; + } + + const response = await session.prompt(text, promptOptions); + + return response; + }, + async dispose(): Promise { + // Session cleanup is handled by context disposal + }, + }; + }, + async dispose(): Promise { + await context.dispose(); + await model.dispose(); + modelCache.delete(cacheKey); + }, + }; + } catch (error) { + modelCache.delete(cacheKey); + if (error instanceof Error && 'code' in error) { + throw error; // Re-throw DextoRuntimeError + } + throw LocalModelError.modelLoadFailed( + modelPath, + error instanceof Error ? error.message : String(error) + ); + } + })(); + + modelCache.set(cacheKey, loadPromise); + return loadPromise; +} + +/** + * Unload a model and free resources. + * Removes all cache entries for the given model path (across different configs). + */ +export async function unloadModel(modelPath: string): Promise { + for (const [key, loadPromise] of modelCache.entries()) { + // Cache key format is "modelPath:gpuLayers:contextSize" + const keyModelPath = key.split(':')[0]; + if (keyModelPath === modelPath) { + try { + const loaded = await loadPromise; + await loaded.dispose(); + } catch { + // Ignore errors during unload + } + modelCache.delete(key); + } + } +} + +/** + * Unload all models and free resources. + */ +export async function unloadAllModels(): Promise { + for (const [key, loadPromise] of modelCache.entries()) { + try { + const loaded = await loadPromise; + await loaded.dispose(); + } catch { + // Ignore errors during unload + } + modelCache.delete(key); + } +} + +/** + * Check if a model is currently loaded. + */ +export function isModelLoaded(modelPath: string): boolean { + for (const key of modelCache.keys()) { + // Cache key format is "modelPath:gpuLayers:contextSize" + const keyModelPath = key.split(':')[0]; + if (keyModelPath === modelPath) { + return true; + } + } + return false; +} + +/** + * Get the number of currently loaded models. + */ +export function getLoadedModelCount(): number { + return modelCache.size; +} diff --git a/dexto/packages/core/src/llm/providers/local/ollama-provider.ts b/dexto/packages/core/src/llm/providers/local/ollama-provider.ts new file mode 100644 index 00000000..44a53687 --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/ollama-provider.ts @@ -0,0 +1,346 @@ +/* global TextDecoder */ +/** + * Ollama provider for local model inference. + * + * Uses Ollama's OpenAI-compatible API for seamless integration + * with the Vercel AI SDK. + */ + +import { createOpenAI } from '@ai-sdk/openai'; +import type { LanguageModel } from 'ai'; +import type { OllamaModelInfo, OllamaStatus } from './types.js'; +import { LocalModelError } from './errors.js'; + +/** + * Default Ollama server URL. + */ +export const DEFAULT_OLLAMA_URL = 'http://localhost:11434'; + +/** + * Ollama configuration options. + */ +export interface OllamaConfig { + /** Ollama server base URL (default: http://localhost:11434) */ + baseURL?: string; +} + +/** + * Check if the Ollama server is running. + */ +export async function checkOllamaStatus( + baseURL: string = DEFAULT_OLLAMA_URL +): Promise { + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 5000); + + const response = await fetch(`${baseURL}/api/version`, { + signal: controller.signal, + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + return { + running: false, + url: baseURL, + error: `HTTP ${response.status}: ${response.statusText}`, + }; + } + + const data = (await response.json()) as { version?: string }; + + // Fetch available models + const models = await listOllamaModels(baseURL); + + const status: OllamaStatus = { + running: true, + url: baseURL, + models, + }; + + if (data.version) { + status.version = data.version; + } + + return status; + } catch (error) { + const errorMessage = + error instanceof Error + ? error.name === 'AbortError' + ? 'Connection timed out' + : error.message + : 'Unknown error'; + + return { + running: false, + url: baseURL, + error: errorMessage, + }; + } +} + +/** + * List available models on the Ollama server. + */ +export async function listOllamaModels( + baseURL: string = DEFAULT_OLLAMA_URL +): Promise { + try { + const response = await fetch(`${baseURL}/api/tags`); + + if (!response.ok) { + return []; + } + + const data = (await response.json()) as { + models?: Array<{ + name: string; + size: number; + digest: string; + modified_at: string; + details?: { + family?: string; + parameter_size?: string; + quantization_level?: string; + }; + }>; + }; + + return (data.models ?? []).map((model) => { + const modelInfo: OllamaModelInfo = { + name: model.name, + size: model.size, + digest: model.digest, + modifiedAt: model.modified_at, + }; + + if (model.details) { + const details: NonNullable = {}; + if (model.details.family) { + details.family = model.details.family; + } + if (model.details.parameter_size) { + details.parameterSize = model.details.parameter_size; + } + if (model.details.quantization_level) { + details.quantizationLevel = model.details.quantization_level; + } + if (Object.keys(details).length > 0) { + modelInfo.details = details; + } + } + + return modelInfo; + }); + } catch { + return []; + } +} + +/** + * Check if a specific model is available on Ollama. + */ +export async function isOllamaModelAvailable( + modelName: string, + baseURL: string = DEFAULT_OLLAMA_URL +): Promise { + const models = await listOllamaModels(baseURL); + return models.some( + (m) => + m.name === modelName || + m.name.startsWith(`${modelName}:`) || + modelName.startsWith(`${m.name}:`) + ); +} + +/** + * Pull a model from the Ollama registry. + * Returns a stream of progress events. + * + * @param modelName - Name of the model to pull + * @param baseURL - Ollama server URL (default: http://localhost:11434) + * @param onProgress - Optional callback for progress updates + * @param signal - Optional AbortSignal for cancellation + */ +export async function pullOllamaModel( + modelName: string, + baseURL: string = DEFAULT_OLLAMA_URL, + onProgress?: (progress: { status: string; completed?: number; total?: number }) => void, + signal?: AbortSignal +): Promise { + try { + const fetchOptions: RequestInit = { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: modelName }), + }; + if (signal) { + fetchOptions.signal = signal; + } + const response = await fetch(`${baseURL}/api/pull`, fetchOptions); + + if (!response.ok) { + throw LocalModelError.ollamaPullFailed( + modelName, + `HTTP ${response.status}: ${response.statusText}` + ); + } + + const reader = response.body?.getReader(); + if (!reader) { + throw LocalModelError.ollamaPullFailed(modelName, 'No response body'); + } + + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + // Ollama sends newline-delimited JSON + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + if (!line.trim()) continue; + try { + const progress = JSON.parse(line) as { + status: string; + completed?: number; + total?: number; + error?: string; + }; + + if (progress.error) { + throw LocalModelError.ollamaPullFailed(modelName, progress.error); + } + + onProgress?.(progress); + } catch (e) { + if (e instanceof Error && e.message.includes('ollamaPullFailed')) { + throw e; + } + // Ignore JSON parse errors + } + } + } + } catch (error) { + if (error instanceof Error && error.message.includes('ECONNREFUSED')) { + throw LocalModelError.ollamaNotRunning(baseURL); + } + throw error; + } +} + +/** + * Create an Ollama language model using the OpenAI-compatible API. + */ +export function createOllamaModel(modelName: string, config: OllamaConfig = {}): LanguageModel { + const { baseURL = DEFAULT_OLLAMA_URL } = config; + + // Ollama's OpenAI-compatible endpoint is at /v1 + const openai = createOpenAI({ + baseURL: `${baseURL}/v1`, + apiKey: 'ollama', // Ollama doesn't require an API key, but the SDK requires a non-empty string + }); + + return openai(modelName); +} + +/** + * Create an Ollama model with status validation. + * Throws if Ollama is not running or model is not available. + */ +export async function createValidatedOllamaModel( + modelName: string, + config: OllamaConfig = {} +): Promise { + const { baseURL = DEFAULT_OLLAMA_URL } = config; + + // Check if Ollama is running + const status = await checkOllamaStatus(baseURL); + if (!status.running) { + throw LocalModelError.ollamaNotRunning(baseURL); + } + + // Check if model is available + const isAvailable = await isOllamaModelAvailable(modelName, baseURL); + if (!isAvailable) { + throw LocalModelError.ollamaModelNotFound(modelName); + } + + return createOllamaModel(modelName, config); +} + +/** + * Get information about a specific Ollama model. + */ +export async function getOllamaModelInfo( + modelName: string, + baseURL: string = DEFAULT_OLLAMA_URL +): Promise { + const models = await listOllamaModels(baseURL); + return ( + models.find( + (m) => + m.name === modelName || + m.name.startsWith(`${modelName}:`) || + modelName.startsWith(`${m.name}:`) + ) ?? null + ); +} + +/** + * Delete a model from Ollama. + */ +export async function deleteOllamaModel( + modelName: string, + baseURL: string = DEFAULT_OLLAMA_URL +): Promise { + try { + const response = await fetch(`${baseURL}/api/delete`, { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: modelName }), + }); + + return response.ok; + } catch { + return false; + } +} + +/** + * Generate embeddings using Ollama. + * Uses the /api/embed endpoint which supports batch processing. + * + * Note: Reserved for future RAG/vector search functionality. + */ +export async function generateOllamaEmbeddings( + modelName: string, + input: string | string[], + baseURL: string = DEFAULT_OLLAMA_URL +): Promise { + const inputs = Array.isArray(input) ? input : [input]; + + // Use /api/embed endpoint which accepts arrays for batch processing + const response = await fetch(`${baseURL}/api/embed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: modelName, + input: inputs, + }), + }); + + if (!response.ok) { + throw LocalModelError.ollamaApiError(`HTTP ${response.status}: ${response.statusText}`); + } + + const data = (await response.json()) as { embeddings: number[][] }; + return data.embeddings; +} diff --git a/dexto/packages/core/src/llm/providers/local/registry.ts b/dexto/packages/core/src/llm/providers/local/registry.ts new file mode 100644 index 00000000..9b002571 --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/registry.ts @@ -0,0 +1,346 @@ +/** + * Curated registry of local GGUF models. + * + * This registry contains vetted models from HuggingFace that are known to work + * well with node-llama-cpp. Models are organized by size and use case. + * + * Model selection criteria: + * - Well-maintained quantizations (bartowski, TheBloke, official repos) + * - Good performance/size trade-off (Q4_K_M as default) + * - Clear licensing for commercial use where possible + * - Tested with node-llama-cpp + */ + +import type { LocalModelInfo } from './types.js'; + +/** + * Curated list of recommended local models. + * Sorted by category and size for easy selection. + */ +export const LOCAL_MODEL_REGISTRY: LocalModelInfo[] = [ + // ============================================ + // RECOMMENDED: Best balance of quality and size + // ============================================ + { + id: 'llama-3.3-8b-q4', + name: 'Llama 3.3 8B Instruct', + description: + "Meta's latest 8B model. Excellent general-purpose performance with 128K context.", + huggingfaceId: 'bartowski/Llama-3.3-8B-Instruct-GGUF', + filename: 'Llama-3.3-8B-Instruct-Q4_K_M.gguf', + quantization: 'Q4_K_M', + sizeBytes: 5_020_000_000, // ~5GB + contextLength: 131072, + categories: ['general', 'coding'], + minVRAM: 6, + minRAM: 8, + recommended: true, + author: 'Meta', + license: 'llama3.3', + supportsTools: true, + }, + { + id: 'qwen-2.5-coder-7b-q4', + name: 'Qwen 2.5 Coder 7B Instruct', + description: "Alibaba's coding-focused model. Excellent for code generation and review.", + huggingfaceId: 'Qwen/Qwen2.5-Coder-7B-Instruct-GGUF', + filename: 'qwen2.5-coder-7b-instruct-q4_k_m.gguf', + quantization: 'Q4_K_M', + sizeBytes: 4_680_000_000, // ~4.7GB + contextLength: 131072, + categories: ['coding'], + minVRAM: 6, + minRAM: 8, + recommended: true, + author: 'Alibaba', + license: 'apache-2.0', + supportsTools: true, + }, + + // ============================================ + // SMALL: Fast models for quick tasks (< 4GB) + // ============================================ + { + id: 'phi-3.5-mini-q4', + name: 'Phi 3.5 Mini Instruct', + description: "Microsoft's compact model. Great for simple tasks with minimal resources.", + huggingfaceId: 'bartowski/Phi-3.5-mini-instruct-GGUF', + filename: 'Phi-3.5-mini-instruct-Q4_K_M.gguf', + quantization: 'Q4_K_M', + sizeBytes: 2_390_000_000, // ~2.4GB + contextLength: 131072, + categories: ['small', 'general'], + minVRAM: 4, + minRAM: 4, + recommended: true, + author: 'Microsoft', + license: 'mit', + }, + { + id: 'qwen-2.5-3b-q4', + name: 'Qwen 2.5 3B Instruct', + description: 'Compact but capable. Good for basic chat and simple tasks.', + huggingfaceId: 'Qwen/Qwen2.5-3B-Instruct-GGUF', + filename: 'qwen2.5-3b-instruct-q4_k_m.gguf', + quantization: 'Q4_K_M', + sizeBytes: 2_050_000_000, // ~2GB + contextLength: 32768, + categories: ['small', 'general'], + minVRAM: 3, + minRAM: 4, + author: 'Alibaba', + license: 'apache-2.0', + }, + { + id: 'gemma-2-2b-q4', + name: 'Gemma 2 2B Instruct', + description: "Google's efficient small model. Good balance of speed and capability.", + huggingfaceId: 'bartowski/gemma-2-2b-it-GGUF', + filename: 'gemma-2-2b-it-Q4_K_M.gguf', + quantization: 'Q4_K_M', + sizeBytes: 1_790_000_000, // ~1.8GB + contextLength: 8192, + categories: ['small', 'general'], + minVRAM: 3, + minRAM: 4, + author: 'Google', + license: 'gemma', + }, + + // ============================================ + // CODING: Optimized for code generation + // ============================================ + { + id: 'qwen-2.5-coder-14b-q4', + name: 'Qwen 2.5 Coder 14B Instruct', + description: 'Larger coding model for complex tasks. Better code understanding.', + huggingfaceId: 'Qwen/Qwen2.5-Coder-14B-Instruct-GGUF', + filename: 'qwen2.5-coder-14b-instruct-q4_k_m.gguf', + quantization: 'Q4_K_M', + sizeBytes: 8_900_000_000, // ~8.9GB + contextLength: 131072, + categories: ['coding'], + minVRAM: 10, + minRAM: 12, + author: 'Alibaba', + license: 'apache-2.0', + supportsTools: true, + }, + { + id: 'deepseek-coder-v2-lite-q4', + name: 'DeepSeek Coder V2 Lite', + description: "DeepSeek's efficient coding model. Great for code completion.", + huggingfaceId: 'bartowski/DeepSeek-Coder-V2-Lite-Instruct-GGUF', + filename: 'DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf', + quantization: 'Q4_K_M', + sizeBytes: 9_200_000_000, // ~9.2GB + contextLength: 131072, + categories: ['coding'], + minVRAM: 12, + minRAM: 16, + author: 'DeepSeek', + license: 'deepseek', + }, + { + id: 'codestral-22b-q4', + name: 'Codestral 22B', + description: "Mistral's dedicated coding model. Supports 80+ languages.", + huggingfaceId: 'bartowski/Codestral-22B-v0.1-GGUF', + filename: 'Codestral-22B-v0.1-Q4_K_M.gguf', + quantization: 'Q4_K_M', + sizeBytes: 13_500_000_000, // ~13.5GB + contextLength: 32768, + categories: ['coding'], + minVRAM: 16, + minRAM: 20, + author: 'Mistral AI', + license: 'mnpl', + }, + + // ============================================ + // GENERAL: Versatile all-purpose models + // ============================================ + { + id: 'mistral-7b-q4', + name: 'Mistral 7B Instruct v0.3', + description: "Mistral's efficient 7B model. Good balance of speed and quality.", + huggingfaceId: 'bartowski/Mistral-7B-Instruct-v0.3-GGUF', + filename: 'Mistral-7B-Instruct-v0.3-Q4_K_M.gguf', + quantization: 'Q4_K_M', + sizeBytes: 4_370_000_000, // ~4.4GB + contextLength: 32768, + categories: ['general'], + minVRAM: 6, + minRAM: 8, + author: 'Mistral AI', + license: 'apache-2.0', + supportsTools: true, + }, + { + id: 'gemma-2-9b-q4', + name: 'Gemma 2 9B Instruct', + description: "Google's capable 9B model. Strong reasoning and instruction following.", + huggingfaceId: 'bartowski/gemma-2-9b-it-GGUF', + filename: 'gemma-2-9b-it-Q4_K_M.gguf', + quantization: 'Q4_K_M', + sizeBytes: 5_760_000_000, // ~5.8GB + contextLength: 8192, + categories: ['general'], + minVRAM: 8, + minRAM: 10, + author: 'Google', + license: 'gemma', + }, + { + id: 'llama-3.1-8b-q4', + name: 'Llama 3.1 8B Instruct', + description: "Meta's Llama 3.1. Solid general-purpose performance.", + huggingfaceId: 'bartowski/Meta-Llama-3.1-8B-Instruct-GGUF', + filename: 'Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf', + quantization: 'Q4_K_M', + sizeBytes: 4_920_000_000, // ~4.9GB + contextLength: 131072, + categories: ['general'], + minVRAM: 6, + minRAM: 8, + author: 'Meta', + license: 'llama3.1', + supportsTools: true, + }, + + // ============================================ + // REASONING: Strong reasoning capabilities + // ============================================ + { + id: 'qwen-2.5-14b-q4', + name: 'Qwen 2.5 14B Instruct', + description: "Alibaba's mid-size model. Strong reasoning and long context.", + huggingfaceId: 'Qwen/Qwen2.5-14B-Instruct-GGUF', + filename: 'qwen2.5-14b-instruct-q4_k_m.gguf', + quantization: 'Q4_K_M', + sizeBytes: 8_700_000_000, // ~8.7GB + contextLength: 131072, + categories: ['reasoning', 'general'], + minVRAM: 10, + minRAM: 12, + author: 'Alibaba', + license: 'apache-2.0', + supportsTools: true, + }, + { + id: 'qwen-2.5-32b-q4', + name: 'Qwen 2.5 32B Instruct', + description: "Alibaba's large model. Excellent reasoning and complex tasks.", + huggingfaceId: 'Qwen/Qwen2.5-32B-Instruct-GGUF', + filename: 'qwen2.5-32b-instruct-q4_k_m.gguf', + quantization: 'Q4_K_M', + sizeBytes: 19_300_000_000, // ~19.3GB + contextLength: 131072, + categories: ['reasoning', 'general'], + minVRAM: 24, + minRAM: 32, + author: 'Alibaba', + license: 'apache-2.0', + supportsTools: true, + }, + + // ============================================ + // VISION: Multimodal models with image support + // ============================================ + { + id: 'llava-v1.6-mistral-7b-q4', + name: 'LLaVA v1.6 Mistral 7B', + description: 'Vision-language model. Can understand and discuss images.', + huggingfaceId: 'cjpais/llava-v1.6-mistral-7b-gguf', + filename: 'llava-v1.6-mistral-7b.Q4_K_M.gguf', + quantization: 'Q4_K_M', + sizeBytes: 4_500_000_000, // ~4.5GB + contextLength: 4096, + categories: ['vision', 'general'], + minVRAM: 8, + minRAM: 10, + author: 'Microsoft/LLaVA', + license: 'llama2', + supportsVision: true, + }, + { + id: 'qwen-2-vl-7b-q4', + name: 'Qwen2 VL 7B Instruct', + description: "Alibaba's vision-language model. High-quality image understanding.", + huggingfaceId: 'Qwen/Qwen2-VL-7B-Instruct-GGUF', + filename: 'qwen2-vl-7b-instruct-q4_k_m.gguf', + quantization: 'Q4_K_M', + sizeBytes: 5_100_000_000, // ~5.1GB + contextLength: 32768, + categories: ['vision', 'general'], + minVRAM: 8, + minRAM: 10, + author: 'Alibaba', + license: 'apache-2.0', + supportsVision: true, + }, +]; + +/** + * Get all models from the registry. + */ +export function getAllLocalModels(): LocalModelInfo[] { + return [...LOCAL_MODEL_REGISTRY]; +} + +/** + * Get a model by ID. + */ +export function getLocalModelById(id: string): LocalModelInfo | undefined { + return LOCAL_MODEL_REGISTRY.find((m) => m.id === id); +} + +/** + * Get models by category. + */ +export function getLocalModelsByCategory(category: string): LocalModelInfo[] { + return LOCAL_MODEL_REGISTRY.filter((m) => m.categories.includes(category as any)); +} + +/** + * Get recommended models (featured in UI). + */ +export function getRecommendedLocalModels(): LocalModelInfo[] { + return LOCAL_MODEL_REGISTRY.filter((m) => m.recommended); +} + +/** + * Get models that fit within VRAM constraints. + */ +export function getModelsForVRAM(vramGB: number): LocalModelInfo[] { + return LOCAL_MODEL_REGISTRY.filter((m) => !m.minVRAM || m.minVRAM <= vramGB); +} + +/** + * Get models that fit within RAM constraints (CPU inference). + */ +export function getModelsForRAM(ramGB: number): LocalModelInfo[] { + return LOCAL_MODEL_REGISTRY.filter((m) => !m.minRAM || m.minRAM <= ramGB); +} + +/** + * Search models by name or description. + */ +export function searchLocalModels(query: string): LocalModelInfo[] { + const q = query.toLowerCase(); + return LOCAL_MODEL_REGISTRY.filter( + (m) => + m.id.toLowerCase().includes(q) || + m.name.toLowerCase().includes(q) || + m.description.toLowerCase().includes(q) + ); +} + +/** + * Get the default model ID for first-time setup. + */ +export function getDefaultLocalModelId(): string { + // Return the first recommended model as default + const recommended = getRecommendedLocalModels(); + return recommended[0]?.id ?? 'llama-3.3-8b-q4'; +} diff --git a/dexto/packages/core/src/llm/providers/local/schemas.ts b/dexto/packages/core/src/llm/providers/local/schemas.ts new file mode 100644 index 00000000..c0347332 --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/schemas.ts @@ -0,0 +1,204 @@ +/** + * Zod schemas for local model configuration validation. + */ + +import { z } from 'zod'; + +/** + * GPU backend options. + */ +export const GPUBackendSchema = z.enum(['metal', 'cuda', 'vulkan', 'cpu']); + +/** + * Quantization type options. + */ +export const QuantizationTypeSchema = z.enum([ + 'Q2_K', + 'Q3_K_S', + 'Q3_K_M', + 'Q3_K_L', + 'Q4_0', + 'Q4_K_S', + 'Q4_K_M', + 'Q5_0', + 'Q5_K_S', + 'Q5_K_M', + 'Q6_K', + 'Q8_0', + 'F16', + 'F32', +]); + +/** + * Local model category options. + */ +export const LocalModelCategorySchema = z.enum([ + 'general', + 'coding', + 'reasoning', + 'small', + 'vision', +]); + +/** + * Model source options. + */ +export const ModelSourceSchema = z.enum(['huggingface', 'ollama']); + +/** + * Model download status. + */ +export const ModelDownloadStatusSchema = z.enum([ + 'pending', + 'downloading', + 'verifying', + 'complete', + 'error', +]); + +/** + * Schema for local model info (registry entry). + */ +export const LocalModelInfoSchema = z + .object({ + id: z.string().min(1).describe('Unique model identifier'), + name: z.string().min(1).describe('Human-readable display name'), + description: z.string().describe('Short description of model capabilities'), + huggingfaceId: z.string().min(1).describe('HuggingFace repository ID'), + filename: z.string().min(1).describe('GGUF filename to download'), + quantization: QuantizationTypeSchema.describe('Quantization level'), + sizeBytes: z.number().int().positive().describe('File size in bytes'), + contextLength: z.number().int().positive().describe('Maximum context window'), + categories: z.array(LocalModelCategorySchema).describe('Model categories'), + minVRAM: z.number().positive().optional().describe('Minimum VRAM in GB'), + minRAM: z.number().positive().optional().describe('Minimum RAM in GB'), + recommended: z.boolean().optional().describe('Whether model is featured'), + author: z.string().optional().describe('Model author/organization'), + license: z.string().optional().describe('License type'), + supportsVision: z.boolean().optional().describe('Whether model supports images'), + supportsTools: z.boolean().optional().describe('Whether model supports function calling'), + }) + .strict(); + +/** + * Schema for model download progress. + */ +export const ModelDownloadProgressSchema = z + .object({ + modelId: z.string().min(1), + status: ModelDownloadStatusSchema, + bytesDownloaded: z.number().int().nonnegative(), + totalBytes: z.number().int().nonnegative(), + percentage: z.number().min(0).max(100), + speed: z.number().nonnegative().optional(), + eta: z.number().nonnegative().optional(), + error: z.string().optional(), + }) + .strict(); + +/** + * Schema for GPU info. + */ +export const GPUInfoSchema = z + .object({ + backend: GPUBackendSchema, + available: z.boolean(), + deviceName: z.string().optional(), + vramMB: z.number().int().nonnegative().optional(), + driverVersion: z.string().optional(), + }) + .strict(); + +/** + * Schema for local LLM configuration. + */ +export const LocalLLMConfigSchema = z + .object({ + provider: z.enum(['local', 'ollama']), + model: z.string().min(1).describe('Model ID or GGUF path'), + gpuLayers: z.number().int().optional().describe('GPU layers (-1=auto, 0=CPU)'), + contextSize: z.number().int().positive().optional().describe('Override context size'), + threads: z.number().int().positive().optional().describe('CPU threads'), + batchSize: z.number().int().positive().optional().describe('Inference batch size'), + modelPath: z.string().optional().describe('Resolved path to model file'), + }) + .strict(); + +/** + * Schema for installed model metadata. + */ +export const InstalledModelSchema = z + .object({ + id: z.string().min(1), + filePath: z.string().min(1), + sizeBytes: z.number().int().positive(), + downloadedAt: z.string().datetime(), + lastUsedAt: z.string().datetime().optional(), + sha256: z.string().optional(), + source: ModelSourceSchema, + }) + .strict(); + +/** + * Schema for model state (persisted state file). + */ +export const ModelStateSchema = z + .object({ + version: z.string().default('1.0'), + installed: z.record(z.string(), InstalledModelSchema).default({}), + activeModelId: z.string().optional(), + downloadQueue: z.array(z.string()).default([]), + }) + .strict(); + +/** + * Schema for model download options. + */ +export const ModelDownloadOptionsSchema = z + .object({ + modelId: z.string().min(1), + outputDir: z.string().optional(), + showProgress: z.boolean().default(true), + hfToken: z.string().optional(), + }) + .strict(); + +/** + * Schema for Ollama model info (from API). + */ +export const OllamaModelInfoSchema = z + .object({ + name: z.string().min(1), + size: z.number().int().nonnegative(), + digest: z.string(), + modifiedAt: z.string(), + details: z + .object({ + family: z.string().optional(), + parameterSize: z.string().optional(), + quantizationLevel: z.string().optional(), + }) + .optional(), + }) + .strict(); + +/** + * Schema for Ollama server status. + */ +export const OllamaStatusSchema = z + .object({ + running: z.boolean(), + url: z.string().url(), + version: z.string().optional(), + models: z.array(OllamaModelInfoSchema).optional(), + error: z.string().optional(), + }) + .strict(); + +// Export inferred types for convenience +export type LocalModelInfoInput = z.input; +export type ModelDownloadProgressInput = z.input; +export type GPUInfoInput = z.input; +export type LocalLLMConfigInput = z.input; +export type InstalledModelInput = z.input; +export type ModelStateInput = z.input; diff --git a/dexto/packages/core/src/llm/providers/local/types.ts b/dexto/packages/core/src/llm/providers/local/types.ts new file mode 100644 index 00000000..0a828865 --- /dev/null +++ b/dexto/packages/core/src/llm/providers/local/types.ts @@ -0,0 +1,303 @@ +/** + * Types for native local model support via node-llama-cpp and Ollama. + */ + +/** + * GPU acceleration backends supported by node-llama-cpp. + * - metal: Apple Silicon (M1/M2/M3) via Metal API + * - cuda: NVIDIA GPUs via CUDA + * - vulkan: Cross-platform GPU acceleration + * - cpu: CPU-only execution (fallback) + */ +export type GPUBackend = 'metal' | 'cuda' | 'vulkan' | 'cpu'; + +/** + * Common GGUF quantization types. + * Lower quantization = smaller file size, slightly lower quality. + * Q4_K_M is a good balance for most use cases. + */ +export type QuantizationType = + | 'Q2_K' + | 'Q3_K_S' + | 'Q3_K_M' + | 'Q3_K_L' + | 'Q4_0' + | 'Q4_K_S' + | 'Q4_K_M' + | 'Q5_0' + | 'Q5_K_S' + | 'Q5_K_M' + | 'Q6_K' + | 'Q8_0' + | 'F16' + | 'F32'; + +/** + * Categories for organizing local models in the UI. + */ +export type LocalModelCategory = 'general' | 'coding' | 'reasoning' | 'small' | 'vision'; + +/** + * Model source - where the model can be downloaded from. + */ +export type ModelSource = 'huggingface' | 'ollama'; + +/** + * Curated local model entry from the registry. + * These are pre-vetted models with known configurations. + */ +export interface LocalModelInfo { + /** Unique identifier (e.g., 'llama-3.3-8b-q4') */ + id: string; + + /** Human-readable display name */ + name: string; + + /** Short description of the model's capabilities */ + description: string; + + /** HuggingFace repository ID (e.g., 'bartowski/Llama-3.3-8B-Instruct-GGUF') */ + huggingfaceId: string; + + /** Filename of the GGUF file to download */ + filename: string; + + /** Quantization level */ + quantization: QuantizationType; + + /** Expected file size in bytes (for progress estimation) */ + sizeBytes: number; + + /** Maximum context window size in tokens */ + contextLength: number; + + /** Model categories for filtering */ + categories: LocalModelCategory[]; + + /** Minimum VRAM required in GB (for GPU inference) */ + minVRAM?: number; + + /** Minimum RAM required in GB (for CPU inference) */ + minRAM?: number; + + /** Whether this model is recommended (featured in UI) */ + recommended?: boolean; + + /** Model author/organization */ + author?: string; + + /** License type (e.g., 'llama3.3', 'apache-2.0', 'mit') */ + license?: string; + + /** Whether model supports vision/images */ + supportsVision?: boolean; + + /** Whether model supports tool/function calling */ + supportsTools?: boolean; +} + +/** + * State of a model download. + */ +export type ModelDownloadStatus = 'pending' | 'downloading' | 'verifying' | 'complete' | 'error'; + +/** + * Progress information for a model download. + * Emitted via events during download. + */ +export interface ModelDownloadProgress { + /** Model ID being downloaded */ + modelId: string; + + /** Current download status */ + status: ModelDownloadStatus; + + /** Bytes downloaded so far */ + bytesDownloaded: number; + + /** Total file size in bytes */ + totalBytes: number; + + /** Download progress as percentage (0-100) */ + percentage: number; + + /** Download speed in bytes per second */ + speed?: number; + + /** Estimated time remaining in seconds */ + eta?: number; + + /** Error message if status is 'error' */ + error?: string; +} + +/** + * GPU detection result. + */ +export interface GPUInfo { + /** Detected GPU backend */ + backend: GPUBackend; + + /** Whether GPU acceleration is available */ + available: boolean; + + /** GPU device name (e.g., 'Apple M2 Pro', 'NVIDIA RTX 4090') */ + deviceName?: string; + + /** Available VRAM in megabytes */ + vramMB?: number; + + /** GPU driver version */ + driverVersion?: string; +} + +/** + * Extended LLM configuration for local models. + * Extends the base config with local-specific options. + */ +export interface LocalLLMConfig { + /** Provider type */ + provider: 'local' | 'ollama'; + + /** Model ID from local registry or custom GGUF path */ + model: string; + + /** Number of layers to offload to GPU (-1 = auto, 0 = CPU only) */ + gpuLayers?: number; + + /** Override context size (tokens) */ + contextSize?: number; + + /** Number of CPU threads to use */ + threads?: number; + + /** Inference batch size */ + batchSize?: number; + + /** Path to model file (resolved from model ID) */ + modelPath?: string; +} + +/** + * Installed model metadata (persisted to state file). + */ +export interface InstalledModel { + /** Model ID from registry */ + id: string; + + /** Absolute path to the .gguf file */ + filePath: string; + + /** File size in bytes */ + sizeBytes: number; + + /** When the model was downloaded (ISO timestamp) */ + downloadedAt: string; + + /** When the model was last used (ISO timestamp) */ + lastUsedAt?: string; + + /** SHA-256 hash of the file for integrity verification */ + sha256?: string; + + /** Source of the download */ + source: ModelSource; +} + +/** + * Model state manager state (persisted to ~/.dexto/models/state.json). + */ +export interface ModelState { + /** Schema version for migrations */ + version: string; + + /** Map of model ID to installed model info */ + installed: Record; + + /** Currently active/selected model ID */ + activeModelId?: string; + + /** Queue of model IDs pending download */ + downloadQueue: string[]; +} + +/** + * Options for downloading a model. + */ +export interface ModelDownloadOptions { + /** Model ID to download */ + modelId: string; + + /** Directory to save the model (default: ~/.dexto/models/) */ + outputDir?: string; + + /** Whether to show CLI progress (default: true) */ + showProgress?: boolean; + + /** Callback for progress updates */ + onProgress?: (progress: ModelDownloadProgress) => void; + + /** HuggingFace token for gated models */ + hfToken?: string; +} + +/** + * Result of a model download operation. + */ +export interface ModelDownloadResult { + /** Whether download was successful */ + success: boolean; + + /** Path to the downloaded model file */ + filePath?: string; + + /** SHA-256 hash of the downloaded file */ + sha256?: string; + + /** Error message if download failed */ + error?: string; +} + +/** + * Ollama model info (from Ollama API /api/tags). + */ +export interface OllamaModelInfo { + /** Model name (e.g., 'llama3.3:8b') */ + name: string; + + /** Model size in bytes */ + size: number; + + /** Model digest/hash */ + digest: string; + + /** When the model was last modified */ + modifiedAt: string; + + /** Model details (parameters, family, etc.) */ + details?: { + family?: string; + parameterSize?: string; + quantizationLevel?: string; + }; +} + +/** + * Ollama server status. + */ +export interface OllamaStatus { + /** Whether Ollama server is running */ + running: boolean; + + /** Ollama server URL */ + url: string; + + /** Ollama version */ + version?: string; + + /** Available models on the server */ + models?: OllamaModelInfo[]; + + /** Error message if not running */ + error?: string; +} diff --git a/dexto/packages/core/src/llm/providers/openrouter-model-registry.ts b/dexto/packages/core/src/llm/providers/openrouter-model-registry.ts new file mode 100644 index 00000000..7606721d --- /dev/null +++ b/dexto/packages/core/src/llm/providers/openrouter-model-registry.ts @@ -0,0 +1,441 @@ +/** + * OpenRouter Model Registry + * + * Provides dynamic model validation against OpenRouter's catalog of 100+ models. + * Fetches and caches the model list from OpenRouter's API with a 24-hour TTL. + * + * Features: + * - Lazy loading: Cache is populated on first lookup + * - Background refresh: Non-blocking cache updates + * - Graceful degradation: Returns 'unknown' when cache is stale, allowing config + * - Throttled requests: Max 1 refresh per 5 minutes to avoid rate limits + */ + +import { promises as fs } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { getDextoGlobalPath } from '../../utils/path.js'; +import { logger } from '../../logger/logger.js'; + +const OPENROUTER_MODELS_ENDPOINT = 'https://openrouter.ai/api/v1/models'; +const CACHE_FILENAME = 'openrouter-models.json'; +const CACHE_SUBDIR = 'cache'; +const CACHE_TTL_MS = 1000 * 60 * 60 * 24; // 24 hours +const MIN_REFRESH_INTERVAL_MS = 1000 * 60 * 5; // 5 minutes throttle between refresh attempts + +export type LookupStatus = 'valid' | 'invalid' | 'unknown'; + +/** Model info stored in cache */ +export interface OpenRouterModelInfo { + id: string; + contextLength: number; +} + +interface CacheFile { + fetchedAt: string; + models: OpenRouterModelInfo[]; +} + +interface RefreshOptions { + apiKey?: string; + force?: boolean; +} + +/** Default context length when not available from API */ +const DEFAULT_CONTEXT_LENGTH = 128000; + +class OpenRouterModelRegistry { + /** Map from normalized model ID to model info */ + private models: Map | null = null; + private lastFetchedAt: number | null = null; + private refreshPromise: Promise | null = null; + private lastRefreshAttemptAt: number | null = null; + private lastUsedApiKey?: string; + + constructor(private readonly cachePath: string) { + this.loadCacheFromDisk(); + } + + /** + * Look up a model ID against the OpenRouter catalog. + * @returns 'valid' if model exists, 'invalid' if not found, 'unknown' if cache is stale/empty + */ + lookup(modelId: string): LookupStatus { + const normalized = this.normalizeModelId(modelId); + if (!normalized) { + return 'unknown'; + } + + if (!this.models || this.models.size === 0) { + // No cache yet - kick off a background refresh and allow for now + this.scheduleRefresh({ force: true }); + return 'unknown'; + } + + if (!this.isCacheFresh()) { + // Don't rely on stale data - refresh in background and treat as unknown + this.scheduleRefresh(); + return 'unknown'; + } + + return this.models.has(normalized) ? 'valid' : 'invalid'; + } + + /** + * Get context length for a model ID. + * @returns context length if model is in cache, null if not found or cache is stale + */ + getContextLength(modelId: string): number | null { + const normalized = this.normalizeModelId(modelId); + if (!normalized) { + return null; + } + + if (!this.models || this.models.size === 0 || !this.isCacheFresh()) { + return null; + } + + const info = this.models.get(normalized); + return info?.contextLength ?? null; + } + + /** + * Get model info for a model ID. + * @returns model info if found in cache, null otherwise + */ + getModelInfo(modelId: string): OpenRouterModelInfo | null { + const normalized = this.normalizeModelId(modelId); + if (!normalized) { + return null; + } + + if (!this.models || this.models.size === 0 || !this.isCacheFresh()) { + return null; + } + + return this.models.get(normalized) ?? null; + } + + /** + * Schedule a non-blocking background refresh of the model cache. + */ + scheduleRefresh(options?: RefreshOptions): void { + const apiKey = options?.apiKey ?? this.lastUsedApiKey; + if (apiKey) { + this.lastUsedApiKey = apiKey; + } + + if (this.refreshPromise) { + return; // Refresh already in-flight + } + + const now = Date.now(); + if ( + !options?.force && + this.lastRefreshAttemptAt && + now - this.lastRefreshAttemptAt < MIN_REFRESH_INTERVAL_MS + ) { + return; // Throttle refresh attempts + } + + this.lastRefreshAttemptAt = now; + this.refreshPromise = this.refreshInternal(apiKey) + .catch((error) => { + logger.warn( + `Failed to refresh OpenRouter model registry: ${error instanceof Error ? error.message : String(error)}` + ); + }) + .finally(() => { + this.refreshPromise = null; + }); + } + + /** + * Blocking refresh of the model cache. + */ + async refresh(options?: RefreshOptions): Promise { + const apiKey = options?.apiKey ?? this.lastUsedApiKey; + if (apiKey) { + this.lastUsedApiKey = apiKey; + } + + if (!options?.force && this.refreshPromise) { + await this.refreshPromise; + return; + } + + if (!options?.force) { + const now = Date.now(); + if ( + this.lastRefreshAttemptAt && + now - this.lastRefreshAttemptAt < MIN_REFRESH_INTERVAL_MS + ) { + if (this.refreshPromise) { + await this.refreshPromise; + } + return; + } + this.lastRefreshAttemptAt = now; + } else { + this.lastRefreshAttemptAt = Date.now(); + } + + const promise = this.refreshInternal(apiKey).finally(() => { + this.refreshPromise = null; + }); + + this.refreshPromise = promise; + await promise; + } + + /** + * Get all cached model IDs (or null if cache is empty). + */ + getCachedModels(): string[] | null { + if (!this.models || this.models.size === 0) { + return null; + } + return Array.from(this.models.keys()); + } + + /** + * Get all cached model info (or null if cache is empty). + */ + getCachedModelsWithInfo(): OpenRouterModelInfo[] | null { + if (!this.models || this.models.size === 0) { + return null; + } + return Array.from(this.models.values()); + } + + /** + * Get cache metadata for debugging/monitoring. + */ + getCacheMetadata(): { lastFetchedAt: Date | null; modelCount: number; isFresh: boolean } { + return { + lastFetchedAt: this.lastFetchedAt ? new Date(this.lastFetchedAt) : null, + modelCount: this.models ? this.models.size : 0, + isFresh: this.isCacheFresh(), + }; + } + + private async refreshInternal(apiKey?: string): Promise { + try { + const headers: Record = { + Accept: 'application/json', + }; + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + + logger.debug('Refreshing OpenRouter model registry from remote source'); + const response = await fetch(OPENROUTER_MODELS_ENDPOINT, { headers }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`HTTP ${response.status}: ${body}`); + } + + const payload = await response.json(); + const models = this.extractModels(payload); + if (models.length === 0) { + throw new Error('No model identifiers returned by OpenRouter'); + } + + await this.writeCache(models); + logger.info(`OpenRouter model registry refreshed with ${models.length} models`); + } catch (error) { + throw error instanceof Error ? error : new Error(String(error)); + } + } + + private loadCacheFromDisk(): void { + if (!existsSync(this.cachePath)) { + return; + } + + try { + const raw = readFileSync(this.cachePath, 'utf-8'); + const parsed = JSON.parse(raw) as CacheFile; + if (!Array.isArray(parsed.models) || typeof parsed.fetchedAt !== 'string') { + logger.warn(`Invalid OpenRouter model cache structure at ${this.cachePath}`); + return; + } + + // Build map from model ID to info + this.models = new Map(); + for (const model of parsed.models) { + if ( + typeof model === 'object' && + model.id && + typeof model.contextLength === 'number' + ) { + this.models.set(model.id.toLowerCase(), model); + } + } + + const timestamp = Date.parse(parsed.fetchedAt); + this.lastFetchedAt = Number.isNaN(timestamp) ? null : timestamp; + + logger.debug( + `Loaded ${this.models.size} OpenRouter models from cache (fetched at ${parsed.fetchedAt})` + ); + } catch (error) { + logger.warn( + `Failed to load OpenRouter model cache: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + private normalizeModelId(modelId: string): string | null { + if (!modelId) { + return null; + } + return modelId.trim().toLowerCase(); + } + + private isCacheFresh(): boolean { + if (!this.lastFetchedAt) { + return false; + } + return Date.now() - this.lastFetchedAt < CACHE_TTL_MS; + } + + private async writeCache(models: OpenRouterModelInfo[]): Promise { + // Deduplicate by ID and sort + const modelMap = new Map(); + for (const model of models) { + if (model.id.trim()) { + modelMap.set(model.id.toLowerCase(), model); + } + } + const uniqueModels = Array.from(modelMap.values()).sort((a, b) => a.id.localeCompare(b.id)); + + await fs.mkdir(path.dirname(this.cachePath), { recursive: true }); + + const now = new Date(); + const cachePayload: CacheFile = { + fetchedAt: now.toISOString(), + models: uniqueModels, + }; + + await fs.writeFile(this.cachePath, JSON.stringify(cachePayload, null, 2), 'utf-8'); + + this.models = new Map(uniqueModels.map((m) => [m.id.toLowerCase(), m])); + this.lastFetchedAt = now.getTime(); + } + + private extractModels(payload: unknown): OpenRouterModelInfo[] { + if (!payload) { + return []; + } + + const raw = + (payload as { data?: unknown; models?: unknown }).data ?? + (payload as { data?: unknown; models?: unknown }).models ?? + payload; + + if (!Array.isArray(raw)) { + return []; + } + + const models: OpenRouterModelInfo[] = []; + for (const item of raw) { + if (item && typeof item === 'object') { + const record = item as Record; + const id = this.firstString([record.id, record.model, record.name]); + if (id) { + // Get context_length from item or top_provider + let contextLength = DEFAULT_CONTEXT_LENGTH; + if (typeof record.context_length === 'number') { + contextLength = record.context_length; + } else if ( + record.top_provider && + typeof record.top_provider === 'object' && + typeof (record.top_provider as Record).context_length === + 'number' + ) { + contextLength = (record.top_provider as Record) + .context_length as number; + } + models.push({ id, contextLength }); + } + } + } + return models; + } + + private firstString(values: Array): string | null { + for (const value of values) { + if (typeof value === 'string' && value.trim().length > 0) { + return value; + } + } + return null; + } +} + +// Singleton instance with global cache path +const cachePath = getDextoGlobalPath(CACHE_SUBDIR, CACHE_FILENAME); +export const openRouterModelRegistry = new OpenRouterModelRegistry(cachePath); + +/** + * Look up a model ID against the OpenRouter catalog. + * @returns 'valid' if model exists, 'invalid' if not found, 'unknown' if cache is stale/empty + */ +export function lookupOpenRouterModel(modelId: string): LookupStatus { + return openRouterModelRegistry.lookup(modelId); +} + +/** + * Schedule a non-blocking background refresh of the OpenRouter model cache. + */ +export function scheduleOpenRouterModelRefresh(options?: RefreshOptions): void { + openRouterModelRegistry.scheduleRefresh(options); +} + +/** + * Perform a blocking refresh of the OpenRouter model cache. + */ +export async function refreshOpenRouterModelCache(options?: RefreshOptions): Promise { + await openRouterModelRegistry.refresh(options); +} + +/** + * Get all cached OpenRouter model IDs (or null if cache is empty). + */ +export function getCachedOpenRouterModels(): string[] | null { + return openRouterModelRegistry.getCachedModels(); +} + +/** + * Get context length for an OpenRouter model. + * @returns context length if model is in cache, null if not found or cache is stale + */ +export function getOpenRouterModelContextLength(modelId: string): number | null { + return openRouterModelRegistry.getContextLength(modelId); +} + +/** + * Get model info for an OpenRouter model. + * @returns model info if found in cache, null otherwise + */ +export function getOpenRouterModelInfo(modelId: string): OpenRouterModelInfo | null { + return openRouterModelRegistry.getModelInfo(modelId); +} + +/** + * Get cache metadata for debugging/monitoring. + */ +export function getOpenRouterModelCacheInfo(): { + lastFetchedAt: Date | null; + modelCount: number; + isFresh: boolean; +} { + return openRouterModelRegistry.getCacheMetadata(); +} + +// Export internal constants for testing purposes +export const __TEST_ONLY__ = { + cachePath, + CACHE_TTL_MS, +}; diff --git a/dexto/packages/core/src/llm/registry.test.ts b/dexto/packages/core/src/llm/registry.test.ts new file mode 100644 index 00000000..4dd26a5c --- /dev/null +++ b/dexto/packages/core/src/llm/registry.test.ts @@ -0,0 +1,1034 @@ +import { describe, it, expect, vi } from 'vitest'; +import { LLM_PROVIDERS } from './types.js'; +import { + LLM_REGISTRY, + getSupportedProviders, + getSupportedModels, + getMaxInputTokensForModel, + isValidProviderModel, + getProviderFromModel, + getAllSupportedModels, + getEffectiveMaxInputTokens, + supportsBaseURL, + requiresBaseURL, + acceptsAnyModel, + getDefaultModelForProvider, + getSupportedFileTypesForModel, + modelSupportsFileType, + validateModelFileSupport, + stripBedrockRegionPrefix, + getModelPricing, + getModelDisplayName, + resolveModelOrigin, + transformModelNameForProvider, + getAllModelsForProvider, + isModelValidForProvider, + hasAllRegistryModelsSupport, +} from './registry.js'; +import { LLMErrorCode } from './error-codes.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; + +// Mock the OpenRouter model registry +vi.mock('./providers/openrouter-model-registry.js', () => ({ + getOpenRouterModelContextLength: vi.fn(), +})); + +import { getOpenRouterModelContextLength } from './providers/openrouter-model-registry.js'; + +const mockLogger: IDextoLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), +} as any; + +describe('LLM Registry Core Functions', () => { + describe('getSupportedProviders', () => { + it('returns all provider keys from registry', () => { + const providers = getSupportedProviders(); + expect(providers).toEqual(Object.keys(LLM_REGISTRY)); + }); + }); + + describe('getSupportedModels', () => { + it('returns models for known provider', () => { + const expected = LLM_REGISTRY.openai.models.map((m) => m.name); + expect(getSupportedModels('openai')).toEqual(expected); + }); + }); + + describe('getMaxInputTokensForModel', () => { + it('returns correct maxInputTokens for valid provider and model', () => { + expect(getMaxInputTokensForModel('openai', 'o4-mini', mockLogger)).toBe(200000); + }); + + it('throws DextoRuntimeError with model unknown code for unknown model', () => { + expect(() => getMaxInputTokensForModel('openai', 'unknown-model', mockLogger)).toThrow( + expect.objectContaining({ + code: LLMErrorCode.MODEL_UNKNOWN, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }) + ); + }); + }); + + describe('isValidProviderModel', () => { + it('returns true for valid provider-model combinations', () => { + expect(isValidProviderModel('openai', 'o4-mini')).toBe(true); + }); + + it('returns false for invalid model', () => { + expect(isValidProviderModel('openai', 'unknown-model')).toBe(false); + }); + }); + + describe('getProviderFromModel', () => { + it('returns correct provider for valid model', () => { + expect(getProviderFromModel('o4-mini')).toBe('openai'); + }); + + it('throws CantInferProviderError for unknown model', () => { + expect(() => getProviderFromModel('unknown-model')).toThrow( + expect.objectContaining({ + code: LLMErrorCode.MODEL_UNKNOWN, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }) + ); + }); + + it('returns correct provider for OpenRouter format models', () => { + // Anthropic models + expect(getProviderFromModel('anthropic/claude-opus-4.5')).toBe('anthropic'); + expect(getProviderFromModel('anthropic/claude-sonnet-4.5')).toBe('anthropic'); + expect(getProviderFromModel('anthropic/claude-haiku-4.5')).toBe('anthropic'); + + // OpenAI models + expect(getProviderFromModel('openai/gpt-5-mini')).toBe('openai'); + expect(getProviderFromModel('openai/o4-mini')).toBe('openai'); + + // Google models + expect(getProviderFromModel('google/gemini-3-flash-preview')).toBe('google'); + + // xAI models (note: x-ai prefix) + expect(getProviderFromModel('x-ai/grok-4')).toBe('xai'); + + // Cohere models + expect(getProviderFromModel('cohere/command-a-03-2025')).toBe('cohere'); + }); + + it('handles case insensitivity for OpenRouter format models', () => { + expect(getProviderFromModel('ANTHROPIC/claude-opus-4.5')).toBe('anthropic'); + expect(getProviderFromModel('Anthropic/Claude-Opus-4.5')).toBe('anthropic'); + }); + }); + + describe('getAllSupportedModels', () => { + it('returns all models from all providers', () => { + const allModels = getAllSupportedModels(); + const expected = Object.values(LLM_REGISTRY).flatMap((info) => + info.models.map((m) => m.name) + ); + expect(allModels).toEqual(expected); + }); + }); + + describe('getDefaultModelForProvider', () => { + it('returns default model for provider with default', () => { + expect(getDefaultModelForProvider('openai')).toBe('gpt-5-mini'); + expect(getDefaultModelForProvider('groq')).toBe('llama-3.3-70b-versatile'); + }); + + it('returns null for provider without default (openai-compatible)', () => { + expect(getDefaultModelForProvider('openai-compatible')).toBe(null); + }); + }); +}); + +describe('Provider Capabilities', () => { + describe('supportsBaseURL', () => { + it('returns true for providers supporting baseURL', () => { + expect(supportsBaseURL('openai-compatible')).toBe(true); + }); + + it('returns false for providers not supporting baseURL', () => { + expect(supportsBaseURL('openai')).toBe(false); + }); + }); + + describe('requiresBaseURL', () => { + it('returns true for providers requiring baseURL', () => { + expect(requiresBaseURL('openai-compatible')).toBe(true); + }); + + it('returns false for providers not requiring baseURL', () => { + expect(requiresBaseURL('openai')).toBe(false); + }); + }); + + describe('acceptsAnyModel', () => { + it('returns true for providers accepting any model', () => { + expect(acceptsAnyModel('openai-compatible')).toBe(true); + }); + + it('returns false for providers with fixed models', () => { + expect(acceptsAnyModel('openai')).toBe(false); + }); + }); +}); + +describe('Case Sensitivity', () => { + it('handles model names case-insensitively across all functions', () => { + // Test multiple functions with case variations + expect(getMaxInputTokensForModel('openai', 'O4-MINI', mockLogger)).toBe(200000); + expect(getMaxInputTokensForModel('openai', 'o4-mini', mockLogger)).toBe(200000); + expect(isValidProviderModel('openai', 'O4-MINI')).toBe(true); + expect(isValidProviderModel('openai', 'o4-mini')).toBe(true); + expect(getProviderFromModel('O4-MINI')).toBe('openai'); + expect(getProviderFromModel('o4-mini')).toBe('openai'); + }); +}); + +describe('Registry Consistency', () => { + it('maintains consistency between LLM_PROVIDERS and LLM_REGISTRY', () => { + const registryKeys = Object.keys(LLM_REGISTRY).sort(); + const providersArray = [...LLM_PROVIDERS].sort(); + expect(registryKeys).toEqual(providersArray); + }); + + it('handles all valid LLMProvider enum values correctly', () => { + LLM_PROVIDERS.forEach((provider) => { + expect(() => getSupportedModels(provider)).not.toThrow(); + expect(Array.isArray(getSupportedModels(provider))).toBe(true); + expect(typeof supportsBaseURL(provider)).toBe('boolean'); + expect(typeof requiresBaseURL(provider)).toBe('boolean'); + expect(typeof acceptsAnyModel(provider)).toBe('boolean'); + }); + }); +}); + +describe('getEffectiveMaxInputTokens', () => { + it('returns explicit override when provided and within registry limit', () => { + const config = { provider: 'openai', model: 'o4-mini', maxInputTokens: 1000 } as any; + expect(getEffectiveMaxInputTokens(config, mockLogger)).toBe(1000); + }); + + it('caps override exceeding registry limit to registry value', () => { + const registryLimit = getMaxInputTokensForModel('openai', 'o4-mini', mockLogger); + const config = { + provider: 'openai', + model: 'o4-mini', + maxInputTokens: registryLimit + 1, + } as any; + expect(getEffectiveMaxInputTokens(config, mockLogger)).toBe(registryLimit); + }); + + it('returns override for unknown model when provided', () => { + const config = { + provider: 'openai', + model: 'unknown-model', + maxInputTokens: 50000, + } as any; + expect(getEffectiveMaxInputTokens(config, mockLogger)).toBe(50000); + }); + + it('defaults to 128000 when baseURL is set and maxInputTokens is missing', () => { + const config = { + provider: 'openai-compatible', + model: 'custom-model', + baseURL: 'https://example.com', + } as any; + expect(getEffectiveMaxInputTokens(config, mockLogger)).toBe(128000); + }); + + it('returns provided maxInputTokens when baseURL is set', () => { + const config = { + provider: 'openai-compatible', + model: 'custom-model', + baseURL: 'https://example.com', + maxInputTokens: 12345, + } as any; + expect(getEffectiveMaxInputTokens(config, mockLogger)).toBe(12345); + }); + + it('defaults to 128000 for providers accepting any model without baseURL', () => { + const config = { provider: 'openai-compatible', model: 'any-model' } as any; + expect(getEffectiveMaxInputTokens(config, mockLogger)).toBe(128000); + }); + + it('uses registry when no override or baseURL is present', () => { + const registryLimit = getMaxInputTokensForModel('openai', 'o4-mini', mockLogger); + const config = { provider: 'openai', model: 'o4-mini' } as any; + expect(getEffectiveMaxInputTokens(config, mockLogger)).toBe(registryLimit); + }); + + it('throws EffectiveMaxInputTokensError when lookup fails without override or baseURL', () => { + const config = { provider: 'openai', model: 'non-existent-model' } as any; + expect(() => getEffectiveMaxInputTokens(config, mockLogger)).toThrow( + expect.objectContaining({ + code: LLMErrorCode.MODEL_UNKNOWN, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }) + ); + }); + + describe('OpenRouter provider', () => { + it('uses context length from OpenRouter registry when available', () => { + vi.mocked(getOpenRouterModelContextLength).mockReturnValue(200000); + const config = { + provider: 'openrouter', + model: 'anthropic/claude-3.5-sonnet', + } as any; + expect(getEffectiveMaxInputTokens(config, mockLogger)).toBe(200000); + }); + + it('falls back to 128000 when model not in OpenRouter cache', () => { + vi.mocked(getOpenRouterModelContextLength).mockReturnValue(null); + const config = { + provider: 'openrouter', + model: 'unknown/model', + } as any; + expect(getEffectiveMaxInputTokens(config, mockLogger)).toBe(128000); + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('not found in cache') + ); + }); + }); +}); + +describe('File Support Functions', () => { + describe('getSupportedFileTypesForModel', () => { + it('returns correct file types for models with specific support', () => { + expect(getSupportedFileTypesForModel('openai', 'gpt-4o-audio-preview')).toEqual([ + 'audio', + ]); + expect(getSupportedFileTypesForModel('openai', 'gpt-5')).toEqual(['pdf', 'image']); + }); + + it('returns empty array for models without file support', () => { + expect(getSupportedFileTypesForModel('groq', 'gemma-2-9b-it')).toEqual([]); + }); + + it('returns provider supportedFileTypes for openai-compatible with any model', () => { + expect(getSupportedFileTypesForModel('openai-compatible', 'custom-model')).toEqual([ + 'pdf', + 'image', + 'audio', + ]); + }); + + it('throws DextoRuntimeError for unknown model', () => { + expect(() => getSupportedFileTypesForModel('openai', 'unknown-model')).toThrow( + expect.objectContaining({ + code: LLMErrorCode.MODEL_UNKNOWN, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }) + ); + }); + + // Case sensitivity already tested in main Case Sensitivity section + }); + + describe('modelSupportsFileType', () => { + it('returns true for supported model-file combinations', () => { + expect(modelSupportsFileType('openai', 'gpt-4o-audio-preview', 'audio')).toBe(true); + expect(modelSupportsFileType('openai', 'gpt-5', 'pdf')).toBe(true); + }); + + it('returns false for unsupported model-file combinations', () => { + expect(modelSupportsFileType('openai', 'gpt-5', 'audio')).toBe(false); + }); + + it('returns true for openai-compatible models (uses provider capabilities)', () => { + expect(modelSupportsFileType('openai-compatible', 'custom-model', 'pdf')).toBe(true); + expect(modelSupportsFileType('openai-compatible', 'custom-model', 'image')).toBe(true); + expect(modelSupportsFileType('openai-compatible', 'custom-model', 'audio')).toBe(true); + }); + + it('throws error for unknown model', () => { + expect(() => modelSupportsFileType('openai', 'unknown-model', 'pdf')).toThrow( + expect.objectContaining({ + code: LLMErrorCode.MODEL_UNKNOWN, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }) + ); + }); + }); + + describe('validateModelFileSupport', () => { + it('validates supported files correctly', () => { + const result = validateModelFileSupport('openai', 'gpt-4o-audio-preview', 'audio/mp3'); + expect(result.isSupported).toBe(true); + expect(result.fileType).toBe('audio'); + expect(result.error).toBeUndefined(); + }); + + it('rejects unsupported files with descriptive error', () => { + const result = validateModelFileSupport('openai', 'gpt-5', 'audio/mp3'); + expect(result.isSupported).toBe(false); + expect(result.fileType).toBe('audio'); + expect(result.error).toBe("Model 'gpt-5' (openai) does not support audio files"); + }); + + it('handles unknown MIME types', () => { + const result = validateModelFileSupport('openai', 'gpt-5', 'application/unknown'); + expect(result.isSupported).toBe(false); + expect(result.fileType).toBeUndefined(); + expect(result.error).toBe('Unsupported file type: application/unknown'); + }); + + it('handles parameterized MIME types correctly', () => { + // Test audio/webm;codecs=opus (the original issue) + const webmResult = validateModelFileSupport( + 'openai', + 'gpt-4o-audio-preview', + 'audio/webm;codecs=opus' + ); + expect(webmResult.isSupported).toBe(true); + expect(webmResult.fileType).toBe('audio'); + expect(webmResult.error).toBeUndefined(); + + // Test other parameterized MIME types + const mp3Result = validateModelFileSupport( + 'openai', + 'gpt-4o-audio-preview', + 'audio/mpeg;layer=3' + ); + expect(mp3Result.isSupported).toBe(true); + expect(mp3Result.fileType).toBe('audio'); + + const pdfResult = validateModelFileSupport( + 'openai', + 'gpt-5', + 'application/pdf;version=1.4' + ); + expect(pdfResult.isSupported).toBe(true); + expect(pdfResult.fileType).toBe('pdf'); + + // Test that unsupported base types with parameters still fail correctly + const unknownResult = validateModelFileSupport( + 'openai', + 'gpt-5', + 'application/unknown;param=value' + ); + expect(unknownResult.isSupported).toBe(false); + expect(unknownResult.error).toBe( + 'Unsupported file type: application/unknown;param=value' + ); + }); + + it('accepts files for openai-compatible provider', () => { + const pdfResult = validateModelFileSupport( + 'openai-compatible', + 'custom-model', + 'application/pdf' + ); + expect(pdfResult.isSupported).toBe(true); + expect(pdfResult.fileType).toBe('pdf'); + expect(pdfResult.error).toBeUndefined(); + + const imageResult = validateModelFileSupport( + 'openai-compatible', + 'custom-model', + 'image/jpeg' + ); + expect(imageResult.isSupported).toBe(true); + expect(imageResult.fileType).toBe('image'); + expect(imageResult.error).toBeUndefined(); + }); + + // Case sensitivity already tested in main Case Sensitivity section + }); +}); + +describe('Provider-Specific Tests', () => { + describe('OpenAI provider', () => { + it('has correct capabilities and models', () => { + expect(getSupportedProviders()).toContain('openai'); + expect(getSupportedModels('openai')).toContain('o4-mini'); + expect(getDefaultModelForProvider('openai')).toBe('gpt-5-mini'); + expect(supportsBaseURL('openai')).toBe(false); + expect(requiresBaseURL('openai')).toBe(false); + expect(acceptsAnyModel('openai')).toBe(false); + }); + }); + + describe('Anthropic provider', () => { + it('has correct capabilities and models', () => { + expect(getSupportedProviders()).toContain('anthropic'); + expect(getSupportedModels('anthropic')).toContain('claude-4-sonnet-20250514'); + expect(getSupportedModels('anthropic')).toContain('claude-haiku-4-5-20251001'); + expect(getDefaultModelForProvider('anthropic')).toBe('claude-haiku-4-5-20251001'); + expect(supportsBaseURL('anthropic')).toBe(false); + expect(requiresBaseURL('anthropic')).toBe(false); + expect(acceptsAnyModel('anthropic')).toBe(false); + }); + }); + + describe('Google provider', () => { + it('has correct capabilities and models', () => { + expect(getSupportedProviders()).toContain('google'); + expect(getSupportedModels('google')).toContain('gemini-3-flash-preview'); + expect(getDefaultModelForProvider('google')).toBe('gemini-3-flash-preview'); + expect(supportsBaseURL('google')).toBe(false); + expect(requiresBaseURL('google')).toBe(false); + expect(acceptsAnyModel('google')).toBe(false); + }); + }); + + describe('OpenAI-Compatible provider', () => { + it('has correct capabilities for custom endpoints', () => { + expect(getSupportedProviders()).toContain('openai-compatible'); + expect(getSupportedModels('openai-compatible')).toEqual([]); + expect(getDefaultModelForProvider('openai-compatible')).toBe(null); + expect(supportsBaseURL('openai-compatible')).toBe(true); + expect(requiresBaseURL('openai-compatible')).toBe(true); + expect(acceptsAnyModel('openai-compatible')).toBe(true); + }); + }); + + describe('Groq provider', () => { + it('has correct capabilities and models', () => { + expect(getSupportedProviders()).toContain('groq'); + expect(getSupportedModels('groq')).toContain('llama-3.3-70b-versatile'); + expect(getDefaultModelForProvider('groq')).toBe('llama-3.3-70b-versatile'); + expect(supportsBaseURL('groq')).toBe(false); + expect(requiresBaseURL('groq')).toBe(false); + expect(acceptsAnyModel('groq')).toBe(false); + }); + }); + + describe('XAI provider', () => { + it('has correct capabilities and models', () => { + expect(getSupportedProviders()).toContain('xai'); + expect(getSupportedModels('xai')).toContain('grok-4'); + expect(getDefaultModelForProvider('xai')).toBe('grok-4'); + expect(supportsBaseURL('xai')).toBe(false); + expect(requiresBaseURL('xai')).toBe(false); + expect(acceptsAnyModel('xai')).toBe(false); + }); + }); + + describe('Cohere provider', () => { + it('has correct capabilities and models', () => { + expect(getSupportedProviders()).toContain('cohere'); + expect(getSupportedModels('cohere')).toContain('command-a-03-2025'); + expect(getDefaultModelForProvider('cohere')).toBe('command-a-03-2025'); + expect(supportsBaseURL('cohere')).toBe(false); + expect(requiresBaseURL('cohere')).toBe(false); + expect(acceptsAnyModel('cohere')).toBe(false); + }); + + it('validates all cohere models correctly', () => { + const cohereModels = [ + 'command-a-03-2025', + 'command-r-plus', + 'command-r', + 'command-r7b', + ]; + cohereModels.forEach((model) => { + expect(isValidProviderModel('cohere', model)).toBe(true); + }); + expect(isValidProviderModel('cohere', 'non-existent')).toBe(false); + }); + + it('returns correct maxInputTokens for cohere models', () => { + expect(getMaxInputTokensForModel('cohere', 'command-a-03-2025', mockLogger)).toBe( + 256000 + ); + expect(getMaxInputTokensForModel('cohere', 'command-r-plus', mockLogger)).toBe(128000); + expect(getMaxInputTokensForModel('cohere', 'command-r', mockLogger)).toBe(128000); + expect(getMaxInputTokensForModel('cohere', 'command-r7b', mockLogger)).toBe(128000); + }); + }); + + describe('OpenRouter provider', () => { + it('has correct capabilities for gateway routing', () => { + expect(getSupportedProviders()).toContain('openrouter'); + expect(getSupportedModels('openrouter')).toEqual([]); + expect(getDefaultModelForProvider('openrouter')).toBe(null); + expect(supportsBaseURL('openrouter')).toBe(false); // Fixed endpoint, auto-injected in resolver + expect(requiresBaseURL('openrouter')).toBe(false); // Auto-injected + expect(acceptsAnyModel('openrouter')).toBe(true); + }); + }); + + describe('LiteLLM provider', () => { + it('has correct capabilities for proxy routing', () => { + expect(getSupportedProviders()).toContain('litellm'); + expect(getSupportedModels('litellm')).toEqual([]); + expect(getDefaultModelForProvider('litellm')).toBe(null); + expect(supportsBaseURL('litellm')).toBe(true); + expect(requiresBaseURL('litellm')).toBe(true); // User must provide proxy URL + expect(acceptsAnyModel('litellm')).toBe(true); + }); + + it('supports all file types for user-hosted proxy', () => { + expect(getSupportedFileTypesForModel('litellm', 'any-model')).toEqual([ + 'pdf', + 'image', + 'audio', + ]); + }); + }); + + describe('Glama provider', () => { + it('has correct capabilities for gateway routing', () => { + expect(getSupportedProviders()).toContain('glama'); + expect(getSupportedModels('glama')).toEqual([]); + expect(getDefaultModelForProvider('glama')).toBe(null); + expect(supportsBaseURL('glama')).toBe(false); // Fixed endpoint, auto-injected + expect(requiresBaseURL('glama')).toBe(false); + expect(acceptsAnyModel('glama')).toBe(true); + }); + + it('supports all file types for gateway', () => { + expect(getSupportedFileTypesForModel('glama', 'openai/gpt-4o')).toEqual([ + 'pdf', + 'image', + 'audio', + ]); + }); + }); + + describe('Bedrock provider', () => { + it('has correct capabilities', () => { + expect(getSupportedProviders()).toContain('bedrock'); + expect(getSupportedModels('bedrock').length).toBeGreaterThan(0); + expect(getDefaultModelForProvider('bedrock')).toBe( + 'anthropic.claude-sonnet-4-5-20250929-v1:0' + ); + expect(supportsBaseURL('bedrock')).toBe(false); + expect(requiresBaseURL('bedrock')).toBe(false); + expect(acceptsAnyModel('bedrock')).toBe(false); + }); + }); +}); + +describe('Bedrock Region Prefix Handling', () => { + describe('stripBedrockRegionPrefix', () => { + it('strips eu. prefix', () => { + expect(stripBedrockRegionPrefix('eu.anthropic.claude-sonnet-4-5-20250929-v1:0')).toBe( + 'anthropic.claude-sonnet-4-5-20250929-v1:0' + ); + }); + + it('strips us. prefix', () => { + expect(stripBedrockRegionPrefix('us.anthropic.claude-sonnet-4-5-20250929-v1:0')).toBe( + 'anthropic.claude-sonnet-4-5-20250929-v1:0' + ); + }); + + it('strips global. prefix', () => { + expect( + stripBedrockRegionPrefix('global.anthropic.claude-sonnet-4-5-20250929-v1:0') + ).toBe('anthropic.claude-sonnet-4-5-20250929-v1:0'); + }); + + it('returns model unchanged when no prefix', () => { + expect(stripBedrockRegionPrefix('anthropic.claude-sonnet-4-5-20250929-v1:0')).toBe( + 'anthropic.claude-sonnet-4-5-20250929-v1:0' + ); + }); + + it('returns non-bedrock models unchanged', () => { + expect(stripBedrockRegionPrefix('gpt-5-mini')).toBe('gpt-5-mini'); + expect(stripBedrockRegionPrefix('claude-sonnet-4-5-20250929')).toBe( + 'claude-sonnet-4-5-20250929' + ); + }); + }); + + describe('registry lookups with prefixed models', () => { + const bedrockModel = 'anthropic.claude-sonnet-4-5-20250929-v1:0'; + + it('isValidProviderModel works with prefixed models', () => { + expect(isValidProviderModel('bedrock', bedrockModel)).toBe(true); + expect(isValidProviderModel('bedrock', `eu.${bedrockModel}`)).toBe(true); + expect(isValidProviderModel('bedrock', `us.${bedrockModel}`)).toBe(true); + expect(isValidProviderModel('bedrock', `global.${bedrockModel}`)).toBe(true); + }); + + it('getProviderFromModel works with prefixed models', () => { + expect(getProviderFromModel(bedrockModel)).toBe('bedrock'); + expect(getProviderFromModel(`eu.${bedrockModel}`)).toBe('bedrock'); + expect(getProviderFromModel(`us.${bedrockModel}`)).toBe('bedrock'); + expect(getProviderFromModel(`global.${bedrockModel}`)).toBe('bedrock'); + }); + + it('getMaxInputTokensForModel works with prefixed models', () => { + const expected = getMaxInputTokensForModel('bedrock', bedrockModel, mockLogger); + expect(getMaxInputTokensForModel('bedrock', `eu.${bedrockModel}`, mockLogger)).toBe( + expected + ); + expect(getMaxInputTokensForModel('bedrock', `us.${bedrockModel}`, mockLogger)).toBe( + expected + ); + expect(getMaxInputTokensForModel('bedrock', `global.${bedrockModel}`, mockLogger)).toBe( + expected + ); + }); + + it('getSupportedFileTypesForModel works with prefixed models', () => { + const expected = getSupportedFileTypesForModel('bedrock', bedrockModel); + expect(getSupportedFileTypesForModel('bedrock', `eu.${bedrockModel}`)).toEqual( + expected + ); + expect(getSupportedFileTypesForModel('bedrock', `us.${bedrockModel}`)).toEqual( + expected + ); + expect(getSupportedFileTypesForModel('bedrock', `global.${bedrockModel}`)).toEqual( + expected + ); + }); + + it('getModelPricing works with prefixed models', () => { + const expected = getModelPricing('bedrock', bedrockModel); + expect(getModelPricing('bedrock', `eu.${bedrockModel}`)).toEqual(expected); + expect(getModelPricing('bedrock', `us.${bedrockModel}`)).toEqual(expected); + expect(getModelPricing('bedrock', `global.${bedrockModel}`)).toEqual(expected); + }); + + it('getModelDisplayName works with prefixed models', () => { + const expected = getModelDisplayName(bedrockModel, 'bedrock'); + expect(getModelDisplayName(`eu.${bedrockModel}`, 'bedrock')).toBe(expected); + expect(getModelDisplayName(`us.${bedrockModel}`, 'bedrock')).toBe(expected); + expect(getModelDisplayName(`global.${bedrockModel}`, 'bedrock')).toBe(expected); + }); + }); +}); + +describe('resolveModelOrigin', () => { + describe('for gateway providers (openrouter, dexto)', () => { + it('resolves OpenRouter format Anthropic models to native format', () => { + // OpenRouter format → native model name via openrouterId reverse lookup + const result = resolveModelOrigin('anthropic/claude-opus-4.5', 'openrouter'); + expect(result).toEqual({ + provider: 'anthropic', + model: 'claude-opus-4-5-20251101', + }); + }); + + it('resolves OpenRouter format models for dexto provider', () => { + const result = resolveModelOrigin('anthropic/claude-haiku-4.5', 'dexto'); + expect(result).toEqual({ + provider: 'anthropic', + model: 'claude-haiku-4-5-20251001', + }); + }); + + it('handles case-insensitive OpenRouter format lookups', () => { + const result = resolveModelOrigin('ANTHROPIC/CLAUDE-OPUS-4.5', 'openrouter'); + expect(result).toEqual({ + provider: 'anthropic', + model: 'claude-opus-4-5-20251101', + }); + }); + + it('returns extracted model name for OpenRouter format without openrouterId mapping', () => { + // For models without an explicit openrouterId, falls back to extracting the model name + const result = resolveModelOrigin('openai/gpt-5-mini', 'openrouter'); + expect(result).toEqual({ + provider: 'openai', + model: 'gpt-5-mini', + }); + }); + + it('resolves native model names without prefix', () => { + const result = resolveModelOrigin('claude-haiku-4-5-20251001', 'openrouter'); + expect(result).toEqual({ + provider: 'anthropic', + model: 'claude-haiku-4-5-20251001', + }); + }); + + it('resolves native model names for OpenAI', () => { + const result = resolveModelOrigin('o4-mini', 'openrouter'); + expect(result).toEqual({ + provider: 'openai', + model: 'o4-mini', + }); + }); + + it('returns null for unknown vendor-prefixed models (groq uses native format)', () => { + // Groq models in our registry don't have vendor prefix (e.g., 'llama-3.3-70b-versatile') + // But OpenRouter uses 'meta-llama/llama-3.3-70b-versatile' + // Since the prefixed version isn't in our registry, this returns null + const result = resolveModelOrigin('meta-llama/llama-3.3-70b-versatile', 'openrouter'); + expect(result).toBeNull(); + }); + + it('resolves groq models using native format', () => { + // Use the native format that's in our registry + const result = resolveModelOrigin('llama-3.3-70b-versatile', 'openrouter'); + expect(result).toEqual({ + provider: 'groq', + model: 'llama-3.3-70b-versatile', + }); + }); + + it('returns null for unknown custom models', () => { + const result = resolveModelOrigin('custom/unknown-model', 'openrouter'); + expect(result).toBeNull(); + }); + + it('handles xAI models with x-ai prefix', () => { + const result = resolveModelOrigin('x-ai/grok-4', 'openrouter'); + expect(result).toEqual({ + provider: 'xai', + model: 'grok-4', + }); + }); + }); + + describe('for non-gateway providers', () => { + it('returns the model as-is for providers without supportsAllRegistryModels', () => { + // OpenAI doesn't support all registry models + const result = resolveModelOrigin('some-model', 'openai'); + expect(result).toEqual({ + provider: 'openai', + model: 'some-model', + }); + }); + + it('returns the model as-is for anthropic provider', () => { + const result = resolveModelOrigin('claude-opus-4-5-20251101', 'anthropic'); + expect(result).toEqual({ + provider: 'anthropic', + model: 'claude-opus-4-5-20251101', + }); + }); + }); +}); + +describe('hasAllRegistryModelsSupport', () => { + it('returns true for dexto provider', () => { + expect(hasAllRegistryModelsSupport('dexto')).toBe(true); + }); + + it('returns true for openrouter provider', () => { + expect(hasAllRegistryModelsSupport('openrouter')).toBe(true); + }); + + it('returns false for native providers', () => { + expect(hasAllRegistryModelsSupport('openai')).toBe(false); + expect(hasAllRegistryModelsSupport('anthropic')).toBe(false); + expect(hasAllRegistryModelsSupport('google')).toBe(false); + expect(hasAllRegistryModelsSupport('groq')).toBe(false); + }); +}); + +describe('getAllModelsForProvider', () => { + it('returns only provider models for native providers', () => { + const openaiModels = getAllModelsForProvider('openai'); + expect(openaiModels.length).toBeGreaterThan(0); + // Native provider models should not have originalProvider + expect( + openaiModels.every( + (m) => !('originalProvider' in m) || m.originalProvider === undefined + ) + ).toBe(true); + }); + + it('returns models from all accessible providers for gateway providers', () => { + const dextoModels = getAllModelsForProvider('dexto'); + + // Should have models from multiple providers + const providers = new Set(dextoModels.map((m) => m.originalProvider)); + expect(providers.size).toBeGreaterThan(1); + + // Should include models from openai, anthropic, google, etc. + expect(providers.has('openai')).toBe(true); + expect(providers.has('anthropic')).toBe(true); + expect(providers.has('google')).toBe(true); + }); + + it('includes originalProvider for gateway provider models', () => { + const dextoModels = getAllModelsForProvider('dexto'); + expect(dextoModels.every((m) => m.originalProvider !== undefined)).toBe(true); + }); + + it('does not include models from other gateway providers', () => { + const dextoModels = getAllModelsForProvider('dexto'); + const providers = new Set(dextoModels.map((m) => m.originalProvider)); + + // Should not include dexto or openrouter + expect(providers.has('dexto')).toBe(false); + expect(providers.has('openrouter')).toBe(false); + }); +}); + +describe('isModelValidForProvider', () => { + it('returns true for native provider models', () => { + expect(isModelValidForProvider('openai', 'gpt-5-mini')).toBe(true); + expect(isModelValidForProvider('anthropic', 'claude-haiku-4-5-20251001')).toBe(true); + }); + + it('returns false for unknown models on native providers', () => { + expect(isModelValidForProvider('openai', 'unknown-model')).toBe(false); + }); + + it('returns true for any model on gateway providers', () => { + // Gateway providers with supportsAllRegistryModels can access models from other providers + expect(isModelValidForProvider('dexto', 'gpt-5-mini')).toBe(true); + expect(isModelValidForProvider('dexto', 'claude-haiku-4-5-20251001')).toBe(true); + expect(isModelValidForProvider('openrouter', 'gpt-5-mini')).toBe(true); + }); + + it('returns true for OpenRouter format models on gateway providers', () => { + expect(isModelValidForProvider('dexto', 'anthropic/claude-opus-4.5')).toBe(true); + expect(isModelValidForProvider('openrouter', 'openai/gpt-5-mini')).toBe(true); + }); +}); + +describe('transformModelNameForProvider', () => { + describe('targeting gateway providers', () => { + it('transforms Anthropic models using openrouterId', () => { + const result = transformModelNameForProvider( + 'claude-haiku-4-5-20251001', + 'anthropic', + 'dexto' + ); + expect(result).toBe('anthropic/claude-haiku-4.5'); + }); + + it('transforms OpenAI models using openrouterId', () => { + const result = transformModelNameForProvider('gpt-5-mini', 'openai', 'dexto'); + expect(result).toBe('openai/gpt-5-mini'); + }); + + it('transforms Google models using openrouterId', () => { + const result = transformModelNameForProvider( + 'gemini-3-flash-preview', + 'google', + 'openrouter' + ); + expect(result).toBe('google/gemini-3-flash-preview'); + }); + + it('transforms xAI models with x-ai prefix', () => { + const result = transformModelNameForProvider('grok-4', 'xai', 'dexto'); + expect(result).toBe('x-ai/grok-4'); + }); + + it('does not transform models already in OpenRouter format', () => { + const result = transformModelNameForProvider( + 'anthropic/claude-opus-4.5', + 'anthropic', + 'dexto' + ); + expect(result).toBe('anthropic/claude-opus-4.5'); + }); + + it('does not transform groq models (no prefix needed)', () => { + const result = transformModelNameForProvider( + 'llama-3.3-70b-versatile', + 'groq', + 'dexto' + ); + expect(result).toBe('llama-3.3-70b-versatile'); + }); + + it('does not transform when original provider is a gateway', () => { + const result = transformModelNameForProvider( + 'anthropic/claude-opus-4.5', + 'dexto', + 'openrouter' + ); + expect(result).toBe('anthropic/claude-opus-4.5'); + }); + }); + + describe('targeting native providers', () => { + it('returns model unchanged when target is not a gateway', () => { + const result = transformModelNameForProvider('gpt-5-mini', 'openai', 'openai'); + expect(result).toBe('gpt-5-mini'); + }); + + it('returns model unchanged for anthropic target', () => { + const result = transformModelNameForProvider( + 'claude-haiku-4-5-20251001', + 'anthropic', + 'anthropic' + ); + expect(result).toBe('claude-haiku-4-5-20251001'); + }); + }); + + describe('error handling', () => { + it('throws for model without openrouterId mapping', () => { + // Create a scenario where we try to transform a model that doesn't have openrouterId + // This would be a bug in our registry - all gateway-accessible models should have openrouterId + expect(() => + transformModelNameForProvider('fake-model-no-mapping', 'openai', 'dexto') + ).toThrow(/has no openrouterId mapping/); + }); + }); +}); + +describe('Gateway provider integration with lookup functions', () => { + describe('getMaxInputTokensForModel', () => { + it('resolves gateway provider to native provider for token lookup', () => { + // dexto + anthropic model should resolve to anthropic's token limit + const dextoResult = getMaxInputTokensForModel('dexto', 'claude-haiku-4-5-20251001'); + const anthropicResult = getMaxInputTokensForModel( + 'anthropic', + 'claude-haiku-4-5-20251001' + ); + expect(dextoResult).toBe(anthropicResult); + }); + + it('handles OpenRouter format models', () => { + const result = getMaxInputTokensForModel('dexto', 'anthropic/claude-haiku-4.5'); + const expected = getMaxInputTokensForModel('anthropic', 'claude-haiku-4-5-20251001'); + expect(result).toBe(expected); + }); + }); + + describe('getSupportedFileTypesForModel', () => { + it('resolves gateway provider to native provider for file type lookup', () => { + const dextoResult = getSupportedFileTypesForModel('dexto', 'claude-haiku-4-5-20251001'); + const anthropicResult = getSupportedFileTypesForModel( + 'anthropic', + 'claude-haiku-4-5-20251001' + ); + expect(dextoResult).toEqual(anthropicResult); + }); + }); + + describe('getModelPricing', () => { + it('resolves gateway provider to native provider for pricing lookup', () => { + const dextoResult = getModelPricing('dexto', 'claude-haiku-4-5-20251001'); + const anthropicResult = getModelPricing('anthropic', 'claude-haiku-4-5-20251001'); + expect(dextoResult).toEqual(anthropicResult); + }); + + it('returns undefined for unknown models on gateway', () => { + const result = getModelPricing('dexto', 'unknown-custom-model'); + expect(result).toBeUndefined(); + }); + }); + + describe('getModelDisplayName', () => { + it('resolves gateway provider to native provider for display name lookup', () => { + const dextoResult = getModelDisplayName('claude-haiku-4-5-20251001', 'dexto'); + const anthropicResult = getModelDisplayName('claude-haiku-4-5-20251001', 'anthropic'); + expect(dextoResult).toBe(anthropicResult); + }); + + it('handles OpenRouter format models', () => { + const result = getModelDisplayName('anthropic/claude-haiku-4.5', 'dexto'); + expect(result).toBe('Claude 4.5 Haiku'); + }); + }); +}); diff --git a/dexto/packages/core/src/llm/registry.ts b/dexto/packages/core/src/llm/registry.ts new file mode 100644 index 00000000..27ddff84 --- /dev/null +++ b/dexto/packages/core/src/llm/registry.ts @@ -0,0 +1,2353 @@ +/** + * LLM Model Registry + * + * TODO: maxOutputTokens - Currently we rely on @ai-sdk/anthropic (and other provider SDKs) + * to set appropriate maxOutputTokens defaults per model. As of v2.0.56, @ai-sdk/anthropic + * has getModelCapabilities() which sets correct limits (e.g., 64000 for claude-haiku-4-5). + * + * If we need finer control or want to support models before the SDK does, we could: + * 1. Add maxOutputTokens to ModelInfo interface + * 2. Create getMaxOutputTokensForModel() / getEffectiveMaxOutputTokens() helpers + * 3. Pass explicit maxOutputTokens in TurnExecutor when config doesn't specify one + * + * For now, keeping SDK dependency is simpler and auto-updates with SDK releases. + */ + +import { LLMConfig } from './schemas.js'; +import { LLMError } from './errors.js'; +import { LLMErrorCode } from './error-codes.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import { + LLM_PROVIDERS, + type LLMProvider, + type SupportedFileType, + type TokenUsage, +} from './types.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { getOpenRouterModelContextLength } from './providers/openrouter-model-registry.js'; + +/** + * Pricing metadata for a model (USD per 1M tokens). + * Optional; when omitted, pricing is unknown. + */ +export interface ModelPricing { + inputPerM: number; + outputPerM: number; + cacheReadPerM?: number; + cacheWritePerM?: number; + currency?: 'USD'; + unit?: 'per_million_tokens'; +} + +export interface ModelInfo { + name: string; + maxInputTokens: number; + default?: boolean; + supportedFileTypes: SupportedFileType[]; // Required - every model must explicitly specify file support + displayName?: string; + // Pricing metadata (USD per 1M tokens). Optional; when omitted, pricing is unknown. + pricing?: ModelPricing; + /** + * OpenRouter model ID for use with gateway providers (dexto, openrouter). + * Only needed when the OpenRouter ID differs from the native model ID. + * For most OpenAI/Google/xAI models, the ID is just `{provider}/{name}`. + * For Anthropic, the IDs differ significantly (e.g., `claude-haiku-4-5-20251001` → `anthropic/claude-haiku-4.5`). + */ + openrouterId?: string; +} + +// Central list of supported file type identifiers used across server/UI +// Re-exported constants are defined in types.ts for single source of truth +// (imported above): LLM_PROVIDERS, SUPPORTED_FILE_TYPES + +// Central MIME type to file type mapping +export const MIME_TYPE_TO_FILE_TYPE: Record = { + 'application/pdf': 'pdf', + 'audio/mp3': 'audio', + 'audio/mpeg': 'audio', + 'audio/wav': 'audio', + 'audio/x-wav': 'audio', + 'audio/wave': 'audio', + 'audio/webm': 'audio', + 'audio/ogg': 'audio', + 'audio/m4a': 'audio', + 'audio/aac': 'audio', + // Common image MIME types + 'image/jpeg': 'image', + 'image/jpg': 'image', + 'image/png': 'image', + 'image/webp': 'image', + 'image/gif': 'image', +}; + +// Helper function to get array of allowed MIME types +export function getAllowedMimeTypes(): string[] { + return Object.keys(MIME_TYPE_TO_FILE_TYPE); +} + +export interface ProviderInfo { + models: ModelInfo[]; + baseURLSupport: 'none' | 'optional' | 'required'; // Cleaner single field + supportedFileTypes: SupportedFileType[]; // Provider-level default, used when model doesn't specify + supportsCustomModels?: boolean; // Allow arbitrary model IDs beyond fixed list + /** + * When true, this provider can access all models from all other providers in the registry. + * Used for gateway providers like 'dexto' that route to multiple upstream providers. + * Model names are transformed to the gateway's format (e.g., 'gpt-5-mini' → 'openai/gpt-5-mini'). + */ + supportsAllRegistryModels?: boolean; + /** + * OpenRouter prefix for this provider's models (e.g., 'openai', 'anthropic', 'x-ai'). + * Used by gateway providers to parse and route prefixed model names. + * - If set: provider's models can be accessed via gateway as `{prefix}/{model}` + * - If undefined: provider is not accessible via gateways (local, gateway providers themselves) + */ + openrouterPrefix?: string; +} + +/** Fallback when we cannot determine the model's input-token limit */ +export const DEFAULT_MAX_INPUT_TOKENS = 128000; + +// Use imported constant LLM_PROVIDERS + +/** + * LLM Model Registry - Single Source of Truth for Supported models and their capabilities + * + * IMPORTANT: supportedFileTypes is the SINGLE SOURCE OF TRUTH for file upload capabilities: + * - Empty array [] = Model does NOT support file uploads (UI will hide all attach buttons) + * - Specific types ['image', 'pdf'] = Model supports ONLY those file types + * - DO NOT use empty arrays as "unknown" - research the model's actual capabilities + * - The web UI directly reflects these capabilities without fallback logic + */ +export const LLM_REGISTRY: Record = { + openai: { + models: [ + // GPT-5.2 series (latest, released Dec 2025) + { + name: 'gpt-5.2-chat-latest', + displayName: 'GPT-5.2 Instant', + openrouterId: 'openai/gpt-5.2-chat', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.75, + outputPerM: 14.0, + cacheReadPerM: 0.175, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5.2', + displayName: 'GPT-5.2 Thinking', + openrouterId: 'openai/gpt-5.2', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.75, + outputPerM: 14.0, + cacheReadPerM: 0.175, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5.2-pro', + displayName: 'GPT-5.2 Pro', + openrouterId: 'openai/gpt-5.2-pro', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 21.0, + outputPerM: 168.0, + cacheReadPerM: 2.1, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5.2-codex', + displayName: 'GPT-5.2 Codex', + openrouterId: 'openai/gpt-5.2-codex', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.75, + outputPerM: 14.0, + cacheReadPerM: 0.175, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // GPT-5.1 series + { + name: 'gpt-5.1-chat-latest', + displayName: 'GPT-5.1 Instant', + openrouterId: 'openai/gpt-5.1-chat', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.25, + outputPerM: 10.0, + cacheReadPerM: 0.125, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5.1', + displayName: 'GPT-5.1 Thinking', + openrouterId: 'openai/gpt-5.1', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.25, + outputPerM: 10.0, + cacheReadPerM: 0.125, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5.1-codex', + displayName: 'GPT-5.1 Codex', + openrouterId: 'openai/gpt-5.1-codex', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.25, + outputPerM: 10.0, + cacheReadPerM: 0.125, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5.1-codex-mini', + displayName: 'GPT-5.1 Codex Mini', + openrouterId: 'openai/gpt-5.1-codex-mini', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.25, + outputPerM: 2.0, + cacheReadPerM: 0.025, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // { + // name: 'gpt-5.1-codex-max', + // displayName: 'GPT-5.1 Codex Max', + // maxInputTokens: 400000, + // supportedFileTypes: ['pdf', 'image'], + // pricing: { + // inputPerM: 1.25, + // outputPerM: 10.0, + // cacheReadPerM: 0.125, + // currency: 'USD', + // unit: 'per_million_tokens', + // }, + // }, + { + name: 'gpt-5-pro', + displayName: 'GPT-5 Pro', + openrouterId: 'openai/gpt-5-pro', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 15.0, + outputPerM: 120.0, + cacheReadPerM: 1.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5', + displayName: 'GPT-5', + openrouterId: 'openai/gpt-5', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.25, + outputPerM: 10.0, + cacheReadPerM: 0.125, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5-mini', + displayName: 'GPT-5 Mini', + openrouterId: 'openai/gpt-5-mini', + maxInputTokens: 400000, + default: true, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.25, + outputPerM: 2.0, + cacheReadPerM: 0.025, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5-nano', + displayName: 'GPT-5 Nano', + openrouterId: 'openai/gpt-5-nano', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.05, + outputPerM: 0.4, + cacheReadPerM: 0.005, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-5-codex', + displayName: 'GPT-5 Codex', + openrouterId: 'openai/gpt-5-codex', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.25, + outputPerM: 10.0, + cacheReadPerM: 0.125, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-4.1', + displayName: 'GPT-4.1', + openrouterId: 'openai/gpt-4.1', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 2.0, + outputPerM: 8.0, + cacheReadPerM: 0.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-4.1-mini', + displayName: 'GPT-4.1 Mini', + openrouterId: 'openai/gpt-4.1-mini', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.4, + outputPerM: 1.6, + cacheReadPerM: 0.1, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-4.1-nano', + displayName: 'GPT-4.1 Nano', + openrouterId: 'openai/gpt-4.1-nano', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.1, + outputPerM: 0.4, + cacheReadPerM: 0.025, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-4o', + displayName: 'GPT-4o', + openrouterId: 'openai/gpt-4o', + maxInputTokens: 128000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 2.5, + outputPerM: 10.0, + cacheReadPerM: 1.25, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-4o-mini', + displayName: 'GPT-4o Mini', + openrouterId: 'openai/gpt-4o-mini', + maxInputTokens: 128000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.15, + outputPerM: 0.6, + cacheReadPerM: 0.075, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gpt-4o-audio-preview', + displayName: 'GPT-4o Audio Preview', + openrouterId: 'openai/gpt-4o-audio-preview', + maxInputTokens: 128000, + supportedFileTypes: ['audio'], + pricing: { + inputPerM: 2.5, + outputPerM: 10.0, + cacheReadPerM: 1.25, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'o4-mini', + displayName: 'O4 Mini', + openrouterId: 'openai/o4-mini', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.1, + outputPerM: 4.4, + cacheReadPerM: 0.275, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'o3', + displayName: 'O3', + openrouterId: 'openai/o3', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 2.0, + outputPerM: 8.0, + cacheReadPerM: 0.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'o3-mini', + displayName: 'O3 Mini', + openrouterId: 'openai/o3-mini', + maxInputTokens: 200000, + supportedFileTypes: [], + pricing: { + inputPerM: 1.1, + outputPerM: 4.4, + cacheReadPerM: 0.55, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'o1', + displayName: 'O1', + openrouterId: 'openai/o1', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 15.0, + outputPerM: 60.0, + cacheReadPerM: 7.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + ], + baseURLSupport: 'none', + supportedFileTypes: [], // No defaults - models must explicitly specify support + openrouterPrefix: 'openai', + }, + 'openai-compatible': { + models: [], // Empty - accepts any model name for custom endpoints + baseURLSupport: 'required', + supportedFileTypes: ['pdf', 'image', 'audio'], // Allow all types for custom endpoints - user assumes responsibility for model capabilities + supportsCustomModels: true, + }, + anthropic: { + models: [ + { + name: 'claude-haiku-4-5-20251001', + displayName: 'Claude 4.5 Haiku', + openrouterId: 'anthropic/claude-haiku-4.5', + maxInputTokens: 200000, + default: true, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.0, + outputPerM: 5.0, + cacheWritePerM: 1.25, + cacheReadPerM: 0.1, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-sonnet-4-5-20250929', + displayName: 'Claude 4.5 Sonnet', + openrouterId: 'anthropic/claude-sonnet-4.5', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-opus-4-5-20251101', + displayName: 'Claude 4.5 Opus', + openrouterId: 'anthropic/claude-opus-4.5', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 5.0, + outputPerM: 25.0, + cacheWritePerM: 6.25, + cacheReadPerM: 0.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-opus-4-1-20250805', + displayName: 'Claude 4.1 Opus', + openrouterId: 'anthropic/claude-opus-4.1', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 15.0, + outputPerM: 75.0, + cacheWritePerM: 18.75, + cacheReadPerM: 1.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-4-opus-20250514', + displayName: 'Claude 4 Opus', + openrouterId: 'anthropic/claude-opus-4', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 15.0, + outputPerM: 75.0, + cacheWritePerM: 18.75, + cacheReadPerM: 1.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-4-sonnet-20250514', + displayName: 'Claude 4 Sonnet', + openrouterId: 'anthropic/claude-sonnet-4', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-3-7-sonnet-20250219', + displayName: 'Claude 3.7 Sonnet', + openrouterId: 'anthropic/claude-3.7-sonnet', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-3-5-sonnet-20240620', + displayName: 'Claude 3.5 Sonnet', + openrouterId: 'anthropic/claude-3.5-sonnet', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-3-5-haiku-20241022', + displayName: 'Claude 3.5 Haiku', + openrouterId: 'anthropic/claude-3.5-haiku', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.8, + outputPerM: 4, + cacheWritePerM: 1, + cacheReadPerM: 0.08, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + ], + baseURLSupport: 'none', + supportedFileTypes: [], // No defaults - models must explicitly specify support + openrouterPrefix: 'anthropic', + }, + google: { + models: [ + { + name: 'gemini-3-flash-preview', + displayName: 'Gemini 3 Flash Preview', + openrouterId: 'google/gemini-3-flash-preview', + maxInputTokens: 1048576, + default: true, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 0.5, + outputPerM: 3.0, + cacheReadPerM: 0.05, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-3-pro-preview', + displayName: 'Gemini 3 Pro Preview', + openrouterId: 'google/gemini-3-pro-preview', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 2.0, + outputPerM: 12.0, + cacheReadPerM: 0.2, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-3-pro-image-preview', + displayName: 'Gemini 3 Pro Image Preview', + openrouterId: 'google/gemini-3-pro-image-preview', + maxInputTokens: 1048576, + supportedFileTypes: ['image'], + pricing: { + inputPerM: 2.0, + outputPerM: 120.0, + cacheReadPerM: 0.2, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-2.5-pro', + displayName: 'Gemini 2.5 Pro', + openrouterId: 'google/gemini-2.5-pro', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 1.25, + outputPerM: 10.0, + cacheReadPerM: 0.31, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-2.5-flash', + displayName: 'Gemini 2.5 Flash', + openrouterId: 'google/gemini-2.5-flash', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 0.3, + outputPerM: 2.5, + cacheReadPerM: 0.03, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-2.5-flash-lite', + displayName: 'Gemini 2.5 Flash Lite', + openrouterId: 'google/gemini-2.5-flash-lite', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 0.1, + outputPerM: 0.4, + cacheReadPerM: 0.025, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-2.0-flash', + displayName: 'Gemini 2.0 Flash', + openrouterId: 'google/gemini-2.0-flash-001', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 0.15, + outputPerM: 0.6, + cacheReadPerM: 0.025, + cacheWritePerM: 1.0, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-2.0-flash-lite', + displayName: 'Gemini 2.0 Flash Lite', + openrouterId: 'google/gemini-2.0-flash-lite-001', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 0.075, + outputPerM: 0.3, + cacheReadPerM: 0.01875, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + ], + baseURLSupport: 'none', + supportedFileTypes: [], // No defaults - models must explicitly specify support + openrouterPrefix: 'google', + }, + // https://console.groq.com/docs/models + groq: { + models: [ + { + name: 'gemma-2-9b-it', + displayName: 'Gemma 2 9B Instruct', + maxInputTokens: 8192, + supportedFileTypes: [], + pricing: { + inputPerM: 0.2, + outputPerM: 0.2, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'openai/gpt-oss-20b', + displayName: 'GPT OSS 20B 128k', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.1, + outputPerM: 0.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'openai/gpt-oss-120b', + displayName: 'GPT OSS 120B 128k', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.15, + outputPerM: 0.75, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'moonshotai/kimi-k2-instruct', + displayName: 'Kimi K2 1T 128k', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 1.0, + outputPerM: 3.0, + cacheReadPerM: 0.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'meta-llama/llama-4-scout-17b-16e-instruct', + displayName: 'Llama 4 Scout (17Bx16E) 128k', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.11, + outputPerM: 0.34, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'meta-llama/llama-4-maverick-17b-128e-instruct', + displayName: 'Llama 4 Maverick (17Bx128E) 128k', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.2, + outputPerM: 0.6, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'deepseek-r1-distill-llama-70b', + displayName: 'DeepSeek R1 Distill Llama 70B 128k', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.75, + outputPerM: 0.9, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'qwen/qwen3-32b', + displayName: 'Qwen3 32B 131k', + maxInputTokens: 131000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.29, + outputPerM: 0.59, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'llama-3.3-70b-versatile', + displayName: 'Llama 3.3 70B Versatile', + maxInputTokens: 128000, + default: true, + supportedFileTypes: [], + pricing: { + inputPerM: 0.59, + outputPerM: 0.79, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + ], + baseURLSupport: 'none', + supportedFileTypes: [], // Groq currently doesn't support file uploads + }, + // https://docs.x.ai/docs/models + // Note: XAI API only supports image uploads (JPG/PNG up to 20MB), not PDFs + xai: { + models: [ + { + name: 'grok-4', + displayName: 'Grok 4', + openrouterId: 'x-ai/grok-4', + maxInputTokens: 256000, + default: true, + supportedFileTypes: ['image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheReadPerM: 0.75, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'grok-3', + displayName: 'Grok 3', + openrouterId: 'x-ai/grok-3', + maxInputTokens: 131072, + supportedFileTypes: ['image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheReadPerM: 0.75, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'grok-3-mini', + displayName: 'Grok 3 Mini', + openrouterId: 'x-ai/grok-3-mini', + maxInputTokens: 131072, + supportedFileTypes: ['image'], + pricing: { + inputPerM: 0.3, + outputPerM: 0.5, + cacheReadPerM: 0.075, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'grok-code-fast-1', + displayName: 'Grok Code Fast', + openrouterId: 'x-ai/grok-code-fast-1', + maxInputTokens: 131072, + supportedFileTypes: [], + pricing: { + inputPerM: 0.2, + outputPerM: 1.5, + cacheReadPerM: 0.02, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + ], + baseURLSupport: 'none', + supportedFileTypes: [], // XAI currently doesn't support file uploads + openrouterPrefix: 'x-ai', + }, + // https://docs.cohere.com/reference/models + cohere: { + models: [ + { + name: 'command-a-03-2025', + displayName: 'Command A (03-2025)', + openrouterId: 'cohere/command-a', + maxInputTokens: 256000, + default: true, + supportedFileTypes: [], + pricing: { + inputPerM: 2.5, + outputPerM: 10.0, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'command-r-plus', + displayName: 'Command R+', + openrouterId: 'cohere/command-r-plus-08-2024', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 2.5, + outputPerM: 10.0, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'command-r', + displayName: 'Command R', + openrouterId: 'cohere/command-r-08-2024', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.15, + outputPerM: 0.6, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'command-r7b', + displayName: 'Command R7B', + openrouterId: 'cohere/command-r7b-12-2024', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.0375, + outputPerM: 0.15, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + ], + baseURLSupport: 'none', + supportedFileTypes: [], // Cohere currently doesn't support file uploads + openrouterPrefix: 'cohere', + }, + // https://openrouter.ai/docs + // OpenRouter is a unified API gateway providing access to 100+ models from various providers. + // Model validation is handled dynamically via openrouter-model-registry.ts + openrouter: { + models: [], // Empty - accepts any model name (validated against OpenRouter's catalog) + baseURLSupport: 'none', // Fixed endpoint - baseURL auto-injected in resolver, no user override allowed + supportedFileTypes: ['pdf', 'image', 'audio'], // Allow all types - user assumes responsibility for model capabilities + supportsCustomModels: true, + supportsAllRegistryModels: true, // Can serve models from all other providers + }, + // https://docs.litellm.ai/ + // LiteLLM is an OpenAI-compatible proxy that unifies 100+ LLM providers. + // User must host their own LiteLLM proxy and provide the baseURL. + litellm: { + models: [], // Empty - accepts any model name (user's proxy determines available models) + baseURLSupport: 'required', // User must provide their LiteLLM proxy URL + supportedFileTypes: ['pdf', 'image', 'audio'], // Allow all types - user assumes responsibility for model capabilities + supportsCustomModels: true, + }, + // https://glama.ai/ + // Glama is an OpenAI-compatible gateway providing unified access to multiple LLM providers. + // Fixed endpoint: https://glama.ai/api/gateway/openai/v1 + glama: { + models: [], // Empty - accepts any model name (format: provider/model e.g., openai/gpt-4o) + baseURLSupport: 'none', // Fixed endpoint - baseURL auto-injected + supportedFileTypes: ['pdf', 'image', 'audio'], // Allow all types - user assumes responsibility for model capabilities + supportsCustomModels: true, + }, + // https://cloud.google.com/vertex-ai + // Google Vertex AI - GCP-hosted gateway for Gemini and Claude models + // Supports both Google's Gemini models and Anthropic's Claude via partnership + // + // Setup instructions: + // 1. Create a Google Cloud account and project + // 2. Enable the Vertex AI API: gcloud services enable aiplatform.googleapis.com + // 3. Enable desired Claude models (requires Anthropic Model Garden) + // 4. Install Google Cloud CLI: https://cloud.google.com/sdk/docs/install + // 5. Configure ADC: gcloud auth application-default login + // 6. Set env vars: GOOGLE_VERTEX_PROJECT (required), GOOGLE_VERTEX_LOCATION (optional) + // + // TODO: Add dynamic model fetching via publishers.models.list API + // - Requires: projectId, region, ADC auth + // - Endpoints: GET projects/{project}/locations/{location}/publishers/{google,anthropic}/models + // - Note: API doesn't return aliases (e.g., gemini-2.0-flash), only versioned IDs + // - Docs: https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/list + // - Models: https://cloud.google.com/vertex-ai/generative-ai/docs/models + vertex: { + models: [ + // Gemini 3 models on Vertex AI (Preview) + { + name: 'gemini-3-flash-preview', + displayName: 'Gemini 3 Flash (Vertex)', + maxInputTokens: 1048576, + default: true, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 0.5, + outputPerM: 3.0, + cacheReadPerM: 0.05, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-3-pro-preview', + displayName: 'Gemini 3 Pro (Vertex)', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 2.0, + outputPerM: 12.0, + cacheReadPerM: 0.2, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // Gemini 2.x models on Vertex AI + { + name: 'gemini-2.5-pro', + displayName: 'Gemini 2.5 Pro (Vertex)', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 1.25, + outputPerM: 10.0, + cacheReadPerM: 0.31, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-2.5-flash', + displayName: 'Gemini 2.5 Flash (Vertex)', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 0.15, + outputPerM: 0.6, + cacheReadPerM: 0.0375, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'gemini-2.0-flash', + displayName: 'Gemini 2.0 Flash (Vertex)', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 0.1, + outputPerM: 0.4, + cacheReadPerM: 0.025, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // Claude 4.5 models on Vertex AI (via Anthropic partnership) + // Note: Claude model IDs use @ suffix format on Vertex + { + name: 'claude-opus-4-5@20251101', + displayName: 'Claude 4.5 Opus (Vertex)', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 5.0, + outputPerM: 25.0, + cacheWritePerM: 6.25, + cacheReadPerM: 0.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-sonnet-4-5@20250929', + displayName: 'Claude 4.5 Sonnet (Vertex)', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-haiku-4-5@20251001', + displayName: 'Claude 4.5 Haiku (Vertex)', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.0, + outputPerM: 5.0, + cacheWritePerM: 1.25, + cacheReadPerM: 0.1, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // Claude 4.1 and 4.0 models on Vertex AI + { + name: 'claude-opus-4-1@20250805', + displayName: 'Claude 4.1 Opus (Vertex)', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 15.0, + outputPerM: 75.0, + cacheWritePerM: 18.75, + cacheReadPerM: 1.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-opus-4@20250514', + displayName: 'Claude 4 Opus (Vertex)', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 15.0, + outputPerM: 75.0, + cacheWritePerM: 18.75, + cacheReadPerM: 1.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-sonnet-4@20250514', + displayName: 'Claude 4 Sonnet (Vertex)', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // Claude 3.x models on Vertex AI + { + name: 'claude-3-7-sonnet@20250219', + displayName: 'Claude 3.7 Sonnet (Vertex)', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-3-5-sonnet-v2@20241022', + displayName: 'Claude 3.5 Sonnet v2 (Vertex)', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'claude-3-5-haiku@20241022', + displayName: 'Claude 3.5 Haiku (Vertex)', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.8, + outputPerM: 4.0, + cacheWritePerM: 1.0, + cacheReadPerM: 0.08, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + ], + baseURLSupport: 'none', // Auto-constructed from projectId and region + supportedFileTypes: ['pdf', 'image', 'audio'], + }, + // Amazon Bedrock - AWS-hosted gateway for Claude, Nova, and more + // Auth: AWS credentials (env vars) or Bedrock API key (AWS_BEARER_TOKEN_BEDROCK) + // + // Cross-region inference: Auto-added for anthropic.* and amazon.* models + // supportsCustomModels: true allows users to add custom model IDs beyond the fixed list + bedrock: { + supportsCustomModels: true, + models: [ + // Claude 4.5 models (latest) + { + name: 'anthropic.claude-sonnet-4-5-20250929-v1:0', + displayName: 'Claude 4.5 Sonnet', + maxInputTokens: 200000, + default: true, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'anthropic.claude-haiku-4-5-20251001-v1:0', + displayName: 'Claude 4.5 Haiku', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.0, + outputPerM: 5.0, + cacheWritePerM: 1.25, + cacheReadPerM: 0.1, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'anthropic.claude-opus-4-5-20251101-v1:0', + displayName: 'Claude 4.5 Opus', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 5.0, + outputPerM: 25.0, + cacheWritePerM: 6.25, + cacheReadPerM: 0.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // Amazon Nova models + { + name: 'amazon.nova-premier-v1:0', + displayName: 'Nova Premier', + maxInputTokens: 1000000, + supportedFileTypes: ['image'], + pricing: { + inputPerM: 2.5, + outputPerM: 12.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'amazon.nova-pro-v1:0', + displayName: 'Nova Pro', + maxInputTokens: 300000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.8, + outputPerM: 3.2, + cacheReadPerM: 0.2, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'amazon.nova-lite-v1:0', + displayName: 'Nova Lite', + maxInputTokens: 300000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 0.06, + outputPerM: 0.24, + cacheReadPerM: 0.015, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'amazon.nova-micro-v1:0', + displayName: 'Nova Micro', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.035, + outputPerM: 0.14, + cacheReadPerM: 0.00875, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // OpenAI GPT-OSS + { + name: 'openai.gpt-oss-120b-1:0', + displayName: 'GPT-OSS 120B', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.15, + outputPerM: 0.6, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'openai.gpt-oss-20b-1:0', + displayName: 'GPT-OSS 20B', + maxInputTokens: 128000, + supportedFileTypes: [], + pricing: { + inputPerM: 0.07, + outputPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // Qwen + { + name: 'qwen.qwen3-coder-30b-a3b-v1:0', + displayName: 'Qwen3 Coder 30B', + maxInputTokens: 262144, + supportedFileTypes: [], + pricing: { + inputPerM: 0.15, + outputPerM: 0.6, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'qwen.qwen3-coder-480b-a35b-v1:0', + displayName: 'Qwen3 Coder 480B', + maxInputTokens: 262144, + supportedFileTypes: [], + pricing: { + inputPerM: 0.22, + outputPerM: 1.8, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + ], + baseURLSupport: 'none', // Auto-constructed from region + supportedFileTypes: ['pdf', 'image'], + }, + // Native local model execution via node-llama-cpp + // Runs GGUF models directly on the machine using Metal/CUDA/Vulkan acceleration + // Models are downloaded from HuggingFace and stored in ~/.dexto/models/ + local: { + models: [], // Populated dynamically from local model registry + baseURLSupport: 'none', // No external server needed + supportedFileTypes: ['image'], // Vision support depends on model capabilities + supportsCustomModels: true, // Allow any GGUF model path + }, + // Ollama server integration + // Uses Ollama's OpenAI-compatible API for local model inference + // Requires Ollama to be installed and running (default: http://localhost:11434) + ollama: { + models: [], // Populated dynamically from Ollama API + baseURLSupport: 'optional', // Default: http://localhost:11434, can be customized + supportedFileTypes: ['image'], // Vision support depends on model + supportsCustomModels: true, // Accept any Ollama model name + }, + // Dexto Gateway - OpenAI-compatible proxy through api.dexto.ai + // Routes to OpenRouter with per-request billing (balance decrement) + // Requires DEXTO_API_KEY from `dexto login` + // + // This is a first-class provider that users explicitly select. + // Model IDs are in OpenRouter format (e.g., 'anthropic/claude-sonnet-4.5') + dexto: { + models: [ + // Claude models (Anthropic via OpenRouter) + { + name: 'anthropic/claude-haiku-4.5', + displayName: 'Claude 4.5 Haiku', + maxInputTokens: 200000, + default: true, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.0, + outputPerM: 5.0, + cacheWritePerM: 1.25, + cacheReadPerM: 0.1, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'anthropic/claude-sonnet-4.5', + displayName: 'Claude 4.5 Sonnet', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 3.0, + outputPerM: 15.0, + cacheWritePerM: 3.75, + cacheReadPerM: 0.3, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'anthropic/claude-opus-4.5', + displayName: 'Claude 4.5 Opus', + maxInputTokens: 200000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 5.0, + outputPerM: 25.0, + cacheWritePerM: 6.25, + cacheReadPerM: 0.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // OpenAI models (via OpenRouter) + { + name: 'openai/gpt-5.2', + displayName: 'GPT-5.2', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.75, + outputPerM: 14.0, + cacheReadPerM: 0.175, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'openai/gpt-5.2-codex', + displayName: 'GPT-5.2 Codex', + maxInputTokens: 400000, + supportedFileTypes: ['pdf', 'image'], + pricing: { + inputPerM: 1.75, + outputPerM: 14.0, + cacheReadPerM: 0.175, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // Google models (via OpenRouter) + { + name: 'google/gemini-3-pro-preview', + displayName: 'Gemini 3 Pro', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 2.0, + outputPerM: 12.0, + cacheReadPerM: 0.2, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + { + name: 'google/gemini-3-flash-preview', + displayName: 'Gemini 3 Flash', + maxInputTokens: 1048576, + supportedFileTypes: ['pdf', 'image', 'audio'], + pricing: { + inputPerM: 0.5, + outputPerM: 3.0, + cacheReadPerM: 0.05, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + // Free models (via OpenRouter) + { + name: 'qwen/qwen3-coder:free', + displayName: 'Qwen3 Coder (Free)', + maxInputTokens: 262000, + supportedFileTypes: [], + // Free - no pricing + }, + { + name: 'deepseek/deepseek-r1-0528:free', + displayName: 'DeepSeek R1 (Free)', + maxInputTokens: 163840, + supportedFileTypes: [], + // Free - no pricing + }, + // Other models (via OpenRouter) + { + name: 'z-ai/glm-4.7', + displayName: 'GLM 4.7', + maxInputTokens: 202752, + supportedFileTypes: [], + pricing: { + inputPerM: 0.4, + outputPerM: 1.5, + currency: 'USD', + unit: 'per_million_tokens', + }, + }, + ], + baseURLSupport: 'none', // Fixed endpoint: https://api.dexto.ai/v1 + supportedFileTypes: ['pdf', 'image', 'audio'], // Same as OpenRouter + supportsCustomModels: true, // Accept any OpenRouter model ID beyond the preset list + supportsAllRegistryModels: true, // Can serve models from all other providers via OpenRouter + }, +}; + +/** + * Strips Bedrock cross-region inference profile prefix (eu., us., global.) from model ID. + * This allows registry lookups to work regardless of whether the user specified a prefix. + * @param model The model ID, potentially with a region prefix + * @returns The model ID without the region prefix + */ +export function stripBedrockRegionPrefix(model: string): string { + if (model.startsWith('eu.') || model.startsWith('us.')) { + return model.slice(3); + } + if (model.startsWith('global.')) { + return model.slice(7); + } + return model; +} + +/** + * Gets the default model for a given provider from the registry. + * @param provider The name of the provider. + * @returns The default model for the provider, or null if no default model is found. + */ +export function getDefaultModelForProvider(provider: LLMProvider): string | null { + const providerInfo = LLM_REGISTRY[provider]; + return providerInfo.models.find((m) => m.default)?.name || null; +} + +/** + * Gets the list of supported providers. + * @returns An array of supported provider names. + */ +export function getSupportedProviders(): LLMProvider[] { + return [...LLM_PROVIDERS]; +} + +/** + * Gets the list of supported models for a given provider. + * @param provider The name of the provider. + * @returns An array of supported model names for the provider. + */ +export function getSupportedModels(provider: LLMProvider): string[] { + const providerInfo = LLM_REGISTRY[provider]; + return providerInfo.models.map((m) => m.name); +} + +/** + * Retrieves the maximum input token limit for a given provider and model from the registry. + * For gateway providers with supportsAllRegistryModels, looks up the model in its original provider. + * @param provider The name of the provider (e.g., 'openai', 'anthropic', 'google'). + * @param model The specific model name. + * @param logger Optional logger instance for logging. Optional because it's used in zod schema + * @returns The maximum input token limit for the model. + * @throws {LLMError} If the model is not found in the registry. + */ +export function getMaxInputTokensForModel( + provider: LLMProvider, + model: string, + logger?: IDextoLogger +): number { + // Resolve gateway providers to the original provider + const resolved = resolveToNativeProvider(provider, model); + const providerInfo = LLM_REGISTRY[resolved.provider]; + + const normalizedModel = stripBedrockRegionPrefix(resolved.model).toLowerCase(); + const modelInfo = providerInfo.models.find((m) => m.name.toLowerCase() === normalizedModel); + if (!modelInfo) { + const supportedModels = getSupportedModels(resolved.provider).join(', '); + logger?.error( + `Model '${resolved.model}' not found for provider '${resolved.provider}' in LLM registry. Supported models: ${supportedModels}` + ); + throw LLMError.unknownModel(resolved.provider, resolved.model); + } + + logger?.debug( + `Found max tokens for ${resolved.provider}/${resolved.model}: ${modelInfo.maxInputTokens}` + ); + return modelInfo.maxInputTokens; +} + +/** + * Validates if a provider and model combination is supported. + * Both parameters are required - structural validation (missing values) is handled by Zod schemas. + * @param provider The provider name. + * @param model The model name. + * @returns True if the combination is valid, false otherwise. + */ +export function isValidProviderModel(provider: LLMProvider, model: string): boolean { + const providerInfo = LLM_REGISTRY[provider]; + const normalizedModel = stripBedrockRegionPrefix(model).toLowerCase(); + return providerInfo.models.some((m) => m.name.toLowerCase() === normalizedModel); +} + +/** + * Infers the LLM provider from the model name by searching the registry. + * Matches the model name (case-insensitive) against all registered models. + * Returns the provider name if found, or 'unknown' if not found. + * + * @param model The model name (e.g., 'gpt-5-mini', 'claude-sonnet-4-5-20250929') + * @returns The inferred provider name ('openai', 'anthropic', etc.), or 'unknown' if no match is found. + */ +export function getProviderFromModel(model: string): LLMProvider { + // Handle OpenRouter format models (e.g., 'anthropic/claude-opus-4.5') + if (model.includes('/')) { + const [prefix, ...rest] = model.split('/'); + const modelName = rest.join('/'); + if (prefix) { + const normalizedPrefix = prefix.toLowerCase(); + // Check if prefix matches a known provider's openrouterPrefix (case-insensitive) + for (const provider of LLM_PROVIDERS) { + const providerPrefix = getOpenrouterPrefix(provider); + if (providerPrefix?.toLowerCase() === normalizedPrefix) { + // Verify model exists in this provider's registry before returning + const providerInfo = LLM_REGISTRY[provider]; + const normalizedModelName = stripBedrockRegionPrefix(modelName).toLowerCase(); + const existsInProvider = providerInfo.models.some( + (m) => + m.name.toLowerCase() === normalizedModelName || + m.openrouterId?.toLowerCase() === model.toLowerCase() + ); + if (existsInProvider) { + return provider; + } + // Model not found in matched provider - fall through to registry scan + break; + } + } + } + } + + const normalizedModel = stripBedrockRegionPrefix(model).toLowerCase(); + for (const provider of LLM_PROVIDERS) { + const info = LLM_REGISTRY[provider]; + if (info.models.some((m) => m.name.toLowerCase() === normalizedModel)) { + return provider; + } + } + throw LLMError.modelProviderUnknown(model); +} + +/** + * Returns a flat array of all supported model names from all providers. + */ +export function getAllSupportedModels(): string[] { + return Object.values(LLM_REGISTRY).flatMap((info) => info.models.map((m) => m.name)); +} + +/** + * Checks if a provider supports custom baseURL. + * @param provider The name of the provider. + * @returns True if the provider supports custom baseURL, false otherwise. + */ +export function supportsBaseURL(provider: LLMProvider): boolean { + const providerInfo = LLM_REGISTRY[provider]; + return providerInfo.baseURLSupport !== 'none'; +} + +/** + * Checks if a provider requires a custom baseURL. + * @param provider The name of the provider. + * @returns True if the provider requires a custom baseURL, false otherwise. + */ +export function requiresBaseURL(provider: LLMProvider): boolean { + const providerInfo = LLM_REGISTRY[provider]; + return providerInfo.baseURLSupport === 'required'; +} + +/** + * Checks if a provider accepts any model name (i.e., has empty models list). + * @param provider The name of the provider. + * @returns True if the provider accepts any model name, false otherwise. + */ +export function acceptsAnyModel(provider: LLMProvider): boolean { + const providerInfo = LLM_REGISTRY[provider]; + return providerInfo.models.length === 0; +} + +/** + * Checks if a provider supports custom model IDs beyond its fixed model list. + * This is set explicitly on providers that allow users to add arbitrary model IDs. + * @param provider The name of the provider. + * @returns True if the provider supports custom models, false otherwise. + */ +export function supportsCustomModels(provider: LLMProvider): boolean { + const providerInfo = LLM_REGISTRY[provider]; + return providerInfo.supportsCustomModels === true; +} + +/** + * Checks if a provider supports all registry models from all other providers. + * @param provider The name of the provider. + * @returns True if the provider supports all registry models, false otherwise. + */ +export function hasAllRegistryModelsSupport(provider: LLMProvider): boolean { + const providerInfo = LLM_REGISTRY[provider]; + return providerInfo.supportsAllRegistryModels === true; +} + +/** + * Gets the OpenRouter prefix for a provider from the registry. + * Returns undefined if the provider doesn't have a prefix (e.g., groq models already have vendor prefixes). + */ +function getOpenrouterPrefix(provider: LLMProvider): string | undefined { + return LLM_REGISTRY[provider].openrouterPrefix; +} + +/** + * Providers whose models are accessible via gateway providers with supportsAllRegistryModels. + * Derived from providers that have openrouterPrefix OR whose models don't need transformation (groq). + */ +const GATEWAY_ACCESSIBLE_PROVIDERS: LLMProvider[] = ( + Object.entries(LLM_REGISTRY) as [LLMProvider, ProviderInfo][] +) + .filter( + ([provider, info]) => + // Has openrouterPrefix (needs transformation) + info.openrouterPrefix !== undefined || + // Special case: groq models already have vendor prefixes, no transformation needed + provider === 'groq' + ) + .map(([provider]) => provider); + +/** + * Gets all models available for a provider, including inherited models + * when supportsAllRegistryModels is true. + * @param provider The name of the provider. + * @returns Array of ModelInfo with additional originalProvider field for inherited models. + */ +export function getAllModelsForProvider( + provider: LLMProvider +): Array { + const providerInfo = LLM_REGISTRY[provider]; + + // If provider doesn't support all registry models, return its own models + if (!providerInfo.supportsAllRegistryModels) { + return providerInfo.models.map((m) => ({ ...m })); + } + + // Collect models from all gateway-accessible providers + const allModels: Array = []; + + for (const sourceProvider of GATEWAY_ACCESSIBLE_PROVIDERS) { + const sourceInfo = LLM_REGISTRY[sourceProvider]; + for (const model of sourceInfo.models) { + allModels.push({ + ...model, + originalProvider: sourceProvider, + }); + } + } + + return allModels; +} + +/** + * Transforms a model name to the format required by a gateway provider (dexto/openrouter). + * Uses the explicit openrouterId mapping from the registry - no fallback guessing. + * + * Transformation is needed when: + * - Target is a gateway (dexto/openrouter) + * - Original provider is a "native" provider (anthropic, openai, google, etc.) + * + * No transformation needed when: + * - Target is not a gateway + * - Original provider is already a gateway (dexto/openrouter) - model is already in correct format + * - Model already contains a slash (already in OpenRouter format) + * - Provider models have vendor prefixes (groq's meta-llama/) + * + * @param model The model name to transform. + * @param originalProvider The provider the model originally belongs to. + * @param targetProvider The provider to transform the model name for. + * @returns The transformed model name. + * @throws {LLMError} If model requires transformation but has no openrouterId mapping. + */ +export function transformModelNameForProvider( + model: string, + originalProvider: LLMProvider, + targetProvider: LLMProvider +): string { + // Only transform when targeting gateway providers (those with supportsAllRegistryModels) + if (!hasAllRegistryModelsSupport(targetProvider)) { + return model; + } + + // If original provider is already a gateway, model is already in correct format + if (hasAllRegistryModelsSupport(originalProvider)) { + return model; + } + + // If model already has a slash, assume it's already in OpenRouter format + if (model.includes('/')) { + return model; + } + + // For providers without openrouterPrefix (like groq whose models already have vendor prefixes), + // no transformation needed + const prefix = getOpenrouterPrefix(originalProvider); + if (!prefix) { + return model; + } + + // Look up the explicit openrouterId mapping - no fallback + // Use case-insensitive matching for consistency with other registry lookups + const providerInfo = LLM_REGISTRY[originalProvider]; + if (providerInfo) { + const normalizedModel = model.toLowerCase(); + const modelInfo = providerInfo.models.find((m) => m.name.toLowerCase() === normalizedModel); + if (modelInfo?.openrouterId) { + return modelInfo.openrouterId; + } + } + + // No mapping found - this is a bug in our registry + throw new DextoRuntimeError( + LLMErrorCode.MODEL_UNKNOWN, + ErrorScope.LLM, + ErrorType.SYSTEM, + `Model '${model}' from provider '${originalProvider}' has no openrouterId mapping. ` + + `All models that can be used via gateway providers must have explicit openrouterId in the registry.`, + { model, originalProvider, targetProvider } + ); +} + +/** + * Finds the original provider for a model when accessed through a gateway provider. + * This is needed to look up model metadata (pricing, file types, etc.) from the original registry. + * @param model The model name (may include provider prefix like 'openai/gpt-5-mini'). + * @param gatewayProvider The gateway provider being used (e.g., 'dexto'). + * @returns The original provider and normalized model name, or null if not found. + */ +export function resolveModelOrigin( + model: string, + gatewayProvider: LLMProvider +): { provider: LLMProvider; model: string } | null { + // If the gateway doesn't support all registry models, model belongs to the gateway itself + if (!hasAllRegistryModelsSupport(gatewayProvider)) { + return { provider: gatewayProvider, model }; + } + + // Check if model has a provider prefix (e.g., 'openai/gpt-5-mini') + if (model.includes('/')) { + const [prefix, ...rest] = model.split('/'); + const modelName = rest.join('/'); + + // Find provider by prefix (case-insensitive) + if (prefix) { + const normalizedPrefix = prefix.toLowerCase(); + for (const provider of LLM_PROVIDERS) { + const providerPrefix = getOpenrouterPrefix(provider); + if (providerPrefix?.toLowerCase() === normalizedPrefix) { + // Reverse lookup: find native model name via openrouterId + // e.g., 'anthropic/claude-opus-4.5' → 'claude-opus-4-5-20251101' + const providerInfo = LLM_REGISTRY[provider]; + const nativeModel = providerInfo?.models.find( + (m) => m.openrouterId?.toLowerCase() === model.toLowerCase() + ); + if (nativeModel) { + return { provider, model: nativeModel.name }; + } + // Fallback: return extracted model name (may be custom or already native) + return { provider, model: modelName }; + } + } + } + + // For models with vendor prefix (like meta-llama/llama-3.3-70b), check all accessible providers + // The full model name including prefix might be in the registry (e.g., groq models) + for (const sourceProvider of GATEWAY_ACCESSIBLE_PROVIDERS) { + const sourceInfo = LLM_REGISTRY[sourceProvider]; + if (sourceInfo.models.some((m) => m.name.toLowerCase() === model.toLowerCase())) { + return { provider: sourceProvider, model }; + } + } + } + + // No prefix - search all accessible providers for the model + for (const sourceProvider of GATEWAY_ACCESSIBLE_PROVIDERS) { + const sourceInfo = LLM_REGISTRY[sourceProvider]; + const normalizedModel = stripBedrockRegionPrefix(model).toLowerCase(); + if (sourceInfo.models.some((m) => m.name.toLowerCase() === normalizedModel)) { + return { provider: sourceProvider, model }; + } + } + + // Model not found in any registry - might be a custom model + return null; +} + +/** + * Resolves a gateway provider to its underlying native provider. + * For gateway providers (dexto, openrouter), finds the original provider that owns the model. + * For native providers, returns the input unchanged. + * + * @example + * resolveToNativeProvider('dexto', 'anthropic/claude-opus-4.5') → { provider: 'anthropic', model: 'claude-opus-4-5-20251101' } + * resolveToNativeProvider('openai', 'gpt-5-mini') → { provider: 'openai', model: 'gpt-5-mini' } + */ +function resolveToNativeProvider( + provider: LLMProvider, + model: string +): { provider: LLMProvider; model: string } { + if (hasAllRegistryModelsSupport(provider)) { + const origin = resolveModelOrigin(model, provider); + if (origin) { + return origin; + } + } + return { provider, model }; +} + +/** + * Checks if a model is valid for a provider, considering supportsAllRegistryModels. + * @param provider The provider to check. + * @param model The model name to validate. + * @returns True if the model is valid for the provider. + */ +export function isModelValidForProvider(provider: LLMProvider, model: string): boolean { + const providerInfo = LLM_REGISTRY[provider]; + + // Check provider's own models first + const normalizedModel = stripBedrockRegionPrefix(model).toLowerCase(); + if (providerInfo.models.some((m) => m.name.toLowerCase() === normalizedModel)) { + return true; + } + + // If provider supports custom models, any model is valid + if (providerInfo.supportsCustomModels) { + return true; + } + + // If provider supports all registry models, check if model exists in any accessible provider + if (providerInfo.supportsAllRegistryModels) { + const origin = resolveModelOrigin(model, provider); + return origin !== null; + } + + return false; +} + +/** + * Providers that don't require API keys. + * These include: + * - Native local providers (local for node-llama-cpp, ollama for Ollama server) + * - Local/self-hosted providers (openai-compatible for vLLM, LocalAI) + * - Proxies that handle auth internally (litellm) + * - Cloud auth providers (vertex uses ADC, bedrock uses AWS credentials) + */ +const API_KEY_OPTIONAL_PROVIDERS: Set = new Set([ + 'local', // Native node-llama-cpp execution - no auth needed + 'ollama', // Ollama server - no auth needed by default + 'openai-compatible', // vLLM, LocalAI - often no auth needed + 'litellm', // Self-hosted proxy - handles auth internally + 'vertex', // Uses Google Cloud ADC (Application Default Credentials) + 'bedrock', // Uses AWS credentials (access key + secret or IAM role) +]); + +/** + * Checks if a provider requires an API key. + * Returns false for: + * - Local providers (openai-compatible for Ollama, vLLM, LocalAI) + * - Self-hosted proxies (litellm) + * - Cloud auth providers (vertex, bedrock) + * + * @param provider The name of the provider. + * @returns True if the provider requires an API key, false otherwise. + */ +export function requiresApiKey(provider: LLMProvider): boolean { + return !API_KEY_OPTIONAL_PROVIDERS.has(provider); +} + +/** + * Gets the supported file types for a specific model. + * For gateway providers with supportsAllRegistryModels, looks up the model in its original provider. + * @param provider The name of the provider. + * @param model The name of the model. + * @returns Array of supported file types for the model. + * @throws {Error} If the model is not found in the registry. + */ +export function getSupportedFileTypesForModel( + provider: LLMProvider, + model: string +): SupportedFileType[] { + // Resolve gateway providers to the original provider + const resolved = resolveToNativeProvider(provider, model); + const providerInfo = LLM_REGISTRY[resolved.provider]; + + // For providers that accept any model name (openai-compatible, gateways with custom models) + if (acceptsAnyModel(resolved.provider)) { + return providerInfo.supportedFileTypes; + } + + // Find the specific model (strip Bedrock region prefix for lookup) + const normalizedModel = stripBedrockRegionPrefix(resolved.model).toLowerCase(); + const modelInfo = providerInfo.models.find((m) => m.name.toLowerCase() === normalizedModel); + if (!modelInfo) { + throw LLMError.unknownModel(resolved.provider, resolved.model); + } + + return modelInfo.supportedFileTypes; +} + +/** + * Checks if a specific model supports a specific file type. + * @param provider The name of the provider. + * @param model The name of the model. + * @param fileType The file type to check support for. + * @returns True if the model supports the file type, false otherwise. + */ +export function modelSupportsFileType( + provider: LLMProvider, + model: string, + fileType: SupportedFileType +): boolean { + const supportedTypes = getSupportedFileTypesForModel(provider, model); + return supportedTypes.includes(fileType); +} + +/** + * Validates if file data is supported by a specific model by checking the mimetype + * @param provider The LLM provider name. + * @param model The model name. + * @param mimeType The MIME type of the file to validate. + * @returns Object containing validation result and details. + */ +export function validateModelFileSupport( + provider: LLMProvider, + model: string, + mimeType: string +): { + isSupported: boolean; + fileType?: SupportedFileType; + error?: string; +} { + // Extract base MIME type by removing parameters (e.g., "audio/webm;codecs=opus" -> "audio/webm") + const baseMimeType = mimeType.toLowerCase().split(';')[0]?.trim() || mimeType.toLowerCase(); + const fileType = MIME_TYPE_TO_FILE_TYPE[baseMimeType]; + if (!fileType) { + return { + isSupported: false, + error: `Unsupported file type: ${mimeType}`, + }; + } + + try { + if (!modelSupportsFileType(provider, model, fileType)) { + return { + isSupported: false, + fileType, + error: `Model '${model}' (${provider}) does not support ${fileType} files`, + }; + } + + return { + isSupported: true, + fileType, + }; + } catch (error) { + return { + isSupported: false, + fileType, + error: + error instanceof Error + ? error.message + : 'Unknown error validating model file support', + }; + } +} + +/** + * Determines the effective maximum input token limit based on configuration. + * Priority: + * 1. Explicit `maxInputTokens` in config + * 2. Registry lookup for known provider/model. + * + * @param config The validated LLM configuration. + * @param logger Optional logger instance for logging. + * @returns The effective maximum input token count for the LLM. + * @throws {Error} + * If `baseURL` is set but `maxInputTokens` is missing (indicating a Zod validation inconsistency). + * Or if `baseURL` is not set, but model isn't found in registry. + * TODO: make more readable + */ +export function getEffectiveMaxInputTokens(config: LLMConfig, logger: IDextoLogger): number { + const configuredMaxInputTokens = config.maxInputTokens; + + // Priority 1: Explicit config override or required value with baseURL + if (configuredMaxInputTokens != null) { + // Case 1a: baseURL is set. maxInputTokens is required and validated by Zod. Trust it. + if (config.baseURL) { + logger.debug( + `Using maxInputTokens from configuration (with baseURL): ${configuredMaxInputTokens}` + ); + return configuredMaxInputTokens; + } + + // Case 1b: baseURL is NOT set, but maxInputTokens is provided (override). + // Sanity-check against registry limits. + try { + const registryMaxInputTokens = getMaxInputTokensForModel( + config.provider, + config.model, + logger + ); + if (configuredMaxInputTokens > registryMaxInputTokens) { + logger.warn( + `Provided maxInputTokens (${configuredMaxInputTokens}) for ${config.provider}/${config.model} exceeds the known limit (${registryMaxInputTokens}) for model ${config.model}. Capping to registry limit.` + ); + return registryMaxInputTokens; + } else { + logger.debug( + `Using valid maxInputTokens override from configuration: ${configuredMaxInputTokens} (Registry limit: ${registryMaxInputTokens})` + ); + return configuredMaxInputTokens; + } + } catch (error: any) { + // Handle registry lookup failures during override check + if (error instanceof DextoRuntimeError && error.code === LLMErrorCode.MODEL_UNKNOWN) { + logger.warn( + `Registry lookup failed during maxInputTokens override check for ${config.provider}/${config.model}: ${error.message}. ` + + `Proceeding with the provided maxInputTokens value (${configuredMaxInputTokens}), but it might be invalid.` + ); + // Return the user's value, assuming Zod validation passed for provider/model existence initially. + return configuredMaxInputTokens; + } else { + // Re-throw unexpected errors + logger.error( + `Unexpected error during registry lookup for maxInputTokens override check: ${error}` + ); + throw error; + } + } + } + + // Priority 2: OpenRouter - look up context length from cached model registry + if (config.provider === 'openrouter') { + const contextLength = getOpenRouterModelContextLength(config.model); + if (contextLength !== null) { + logger.debug( + `Using maxInputTokens from OpenRouter registry for ${config.model}: ${contextLength}` + ); + return contextLength; + } + // Cache miss or stale - fall through to default + logger.warn( + `OpenRouter model ${config.model} not found in cache, defaulting to ${DEFAULT_MAX_INPUT_TOKENS} tokens` + ); + return DEFAULT_MAX_INPUT_TOKENS; + } + + // Priority 3: baseURL is set but maxInputTokens is missing - default to 128k tokens + if (config.baseURL) { + logger.warn( + `baseURL is set but maxInputTokens is missing. Defaulting to ${DEFAULT_MAX_INPUT_TOKENS}. ` + + `Provide 'maxInputTokens' in configuration to avoid default fallback.` + ); + return DEFAULT_MAX_INPUT_TOKENS; + } + + // Priority 4: Check if provider accepts any model (like openai-compatible) + if (acceptsAnyModel(config.provider)) { + logger.debug( + `Provider ${config.provider} accepts any model, defaulting to ${DEFAULT_MAX_INPUT_TOKENS} tokens` + ); + return DEFAULT_MAX_INPUT_TOKENS; + } + + // Priority 5: No override, no baseURL - use registry. + try { + const registryMaxInputTokens = getMaxInputTokensForModel( + config.provider, + config.model, + logger + ); + logger.debug( + `Using maxInputTokens from registry for ${config.provider}/${config.model}: ${registryMaxInputTokens}` + ); + return registryMaxInputTokens; + } catch (error: any) { + // Handle registry lookup failures gracefully (e.g., typo in validated config) + if (error instanceof DextoRuntimeError && error.code === LLMErrorCode.MODEL_UNKNOWN) { + // For providers that support custom models, use default instead of throwing + if (supportsCustomModels(config.provider)) { + logger.debug( + `Custom model ${config.model} not in ${config.provider} registry, defaulting to ${DEFAULT_MAX_INPUT_TOKENS} tokens` + ); + return DEFAULT_MAX_INPUT_TOKENS; + } + // Log as error and throw a specific fatal error + logger.error( + `Registry lookup failed for ${config.provider}/${config.model}: ${error.message}. ` + + `Effective maxInputTokens cannot be determined.` + ); + throw LLMError.unknownModel(config.provider, config.model); + } else { + // Re-throw unexpected errors during registry lookup + logger.error(`Unexpected error during registry lookup for maxInputTokens: ${error}`); + throw error; + } + } +} + +/** + * Gets the pricing information for a specific model. + * For gateway providers with supportsAllRegistryModels, looks up the model in its original provider. + * + * Note: This returns the original provider's pricing. Gateway providers may have markup + * that should be applied separately if needed. + * + * @param provider The name of the provider. + * @param model The name of the model. + * @returns The pricing information for the model, or undefined if not available. + */ +export function getModelPricing(provider: LLMProvider, model: string): ModelPricing | undefined { + // Resolve gateway providers to the original provider + const resolved = resolveToNativeProvider(provider, model); + const providerInfo = LLM_REGISTRY[resolved.provider]; + + // For providers that accept any model name, no pricing available + if (acceptsAnyModel(resolved.provider)) { + return undefined; + } + + const normalizedModel = stripBedrockRegionPrefix(resolved.model).toLowerCase(); + const modelInfo = providerInfo.models.find((m) => m.name.toLowerCase() === normalizedModel); + return modelInfo?.pricing; +} + +/** + * Gets the display name for a model, falling back to the model ID if not found. + * For gateway providers with supportsAllRegistryModels, looks up the model in its original provider. + */ +export function getModelDisplayName(model: string, provider?: LLMProvider): string { + let inferredProvider: LLMProvider; + try { + inferredProvider = provider ?? getProviderFromModel(model); + } catch { + // Unknown model - fall back to model ID + return model; + } + + // Resolve gateway providers to the original provider + const resolved = resolveToNativeProvider(inferredProvider, model); + const providerInfo = LLM_REGISTRY[resolved.provider]; + + if (!providerInfo || acceptsAnyModel(resolved.provider)) { + return model; + } + + const normalizedModel = stripBedrockRegionPrefix(resolved.model).toLowerCase(); + const modelInfo = providerInfo.models.find((m) => m.name.toLowerCase() === normalizedModel); + return modelInfo?.displayName ?? model; +} + +// TODO: Add reasoningCapable as a property in the model registry instead of hardcoding here +/** + * Checks if a model supports configurable reasoning effort. + * Currently only OpenAI reasoning models (o1, o3, codex, gpt-5.x) support this. + * + * @param model The model name to check. + * @param provider Optional provider for context (defaults to detecting from model name). + * @returns True if the model supports reasoning effort configuration. + */ +export function isReasoningCapableModel(model: string, _provider?: LLMProvider): boolean { + const modelLower = model.toLowerCase(); + + // Codex models are optimized for complex coding with reasoning + if (modelLower.includes('codex')) { + return true; + } + + // o1 and o3 are dedicated reasoning models + if (modelLower.startsWith('o1') || modelLower.startsWith('o3') || modelLower.startsWith('o4')) { + return true; + } + + // GPT-5 series support reasoning effort + if ( + modelLower.includes('gpt-5') || + modelLower.includes('gpt-5.1') || + modelLower.includes('gpt-5.2') + ) { + return true; + } + + return false; +} + +/** + * Calculates the cost for a given token usage based on model pricing. + * + * @param usage Token usage counts. + * @param pricing Model pricing (per million tokens). + * @returns Cost in USD. + */ +export function calculateCost(usage: TokenUsage, pricing: ModelPricing): number { + const inputCost = ((usage.inputTokens ?? 0) * pricing.inputPerM) / 1_000_000; + const outputCost = ((usage.outputTokens ?? 0) * pricing.outputPerM) / 1_000_000; + const cacheReadCost = ((usage.cacheReadTokens ?? 0) * (pricing.cacheReadPerM ?? 0)) / 1_000_000; + const cacheWriteCost = + ((usage.cacheWriteTokens ?? 0) * (pricing.cacheWritePerM ?? 0)) / 1_000_000; + // Charge reasoning tokens at output rate + const reasoningCost = ((usage.reasoningTokens ?? 0) * pricing.outputPerM) / 1_000_000; + + return inputCost + outputCost + cacheReadCost + cacheWriteCost + reasoningCost; +} diff --git a/dexto/packages/core/src/llm/resolver.ts b/dexto/packages/core/src/llm/resolver.ts new file mode 100644 index 00000000..48aa163e --- /dev/null +++ b/dexto/packages/core/src/llm/resolver.ts @@ -0,0 +1,271 @@ +import { Result, hasErrors, splitIssues, ok, fail, zodToIssues } from '../utils/result.js'; +import { Issue, ErrorScope, ErrorType } from '@core/errors/types.js'; +import { LLMErrorCode } from './error-codes.js'; + +import { type ValidatedLLMConfig, type LLMUpdates, type LLMConfig } from './schemas.js'; +import { LLMConfigSchema } from './schemas.js'; +import { + getDefaultModelForProvider, + acceptsAnyModel, + getProviderFromModel, + isValidProviderModel, + getEffectiveMaxInputTokens, + supportsBaseURL, + supportsCustomModels, + hasAllRegistryModelsSupport, + transformModelNameForProvider, +} from './registry.js'; +import { + lookupOpenRouterModel, + refreshOpenRouterModelCache, +} from './providers/openrouter-model-registry.js'; +import type { LLMUpdateContext } from './types.js'; +import { resolveApiKeyForProvider } from '@core/utils/api-key-resolver.js'; +import type { IDextoLogger } from '@core/logger/v2/types.js'; + +// TODO: Consider consolidating validation into async Zod schema (superRefine supports async). +// Currently OpenRouter validation is here to avoid network calls during startup/serverless. +// If startup validation is desired, move to schema with safeParseAsync() and handle serverless separately. + +/** + * Convenience function that combines resolveLLM and validateLLM + */ +export async function resolveAndValidateLLMConfig( + previous: ValidatedLLMConfig, + updates: LLMUpdates, + logger: IDextoLogger +): Promise> { + const { candidate, warnings } = await resolveLLMConfig(previous, updates, logger); + + // If resolver produced any errors, fail immediately (don't try to validate a broken candidate) + if (hasErrors(warnings)) { + const { errors } = splitIssues(warnings); + return fail(errors); + } + const result = validateLLMConfig(candidate, warnings); + return result; +} + +/** + * Infers the LLM config from the provided updates + * @param previous - The previous LLM config + * @param updates - The updates to the LLM config + * @returns The resolved LLM config + */ +export async function resolveLLMConfig( + previous: ValidatedLLMConfig, + updates: LLMUpdates, + logger: IDextoLogger +): Promise<{ candidate: LLMConfig; warnings: Issue[] }> { + const warnings: Issue[] = []; + + // Provider inference (if not provided, infer from model or previous provider) + const provider = + updates.provider ?? + (updates.model + ? (() => { + try { + return getProviderFromModel(updates.model); + } catch { + return previous.provider; + } + })() + : previous.provider); + + // API key resolution + // (if not provided, previous API key if provider is the same) + // (if not provided, and provider is different, throw error) + const envKey = resolveApiKeyForProvider(provider); + const apiKey = + updates.apiKey ?? (provider !== previous.provider ? envKey : previous.apiKey) ?? ''; + if (!apiKey) { + warnings.push({ + code: LLMErrorCode.API_KEY_CANDIDATE_MISSING, + message: 'API key not provided or found in environment', + severity: 'warning', + scope: ErrorScope.LLM, + type: ErrorType.USER, + context: { provider }, + }); + } else if (typeof apiKey === 'string' && apiKey.length < 10) { + warnings.push({ + code: LLMErrorCode.API_KEY_INVALID, + message: 'API key looks unusually short', + severity: 'warning', + scope: ErrorScope.LLM, + type: ErrorType.USER, + context: { provider }, + }); + } + + // Model fallback + // if new provider doesn't support the new model, use the default model + // Skip fallback for providers that support custom models (they allow arbitrary model IDs) + let model = updates.model ?? previous.model; + if ( + provider !== previous.provider && + !acceptsAnyModel(provider) && + !supportsCustomModels(provider) && + !isValidProviderModel(provider, model) + ) { + model = getDefaultModelForProvider(provider) ?? previous.model; + warnings.push({ + code: LLMErrorCode.MODEL_INCOMPATIBLE, + message: `Model set to default '${model}' for provider '${provider}'`, + severity: 'warning', + scope: ErrorScope.LLM, + type: ErrorType.USER, + context: { provider, model }, + }); + } + + // Gateway model transformation + // When targeting a gateway provider (dexto/openrouter), transform native model names + // to OpenRouter format (e.g., "claude-sonnet-4-5-20250929" -> "anthropic/claude-sonnet-4.5") + if (hasAllRegistryModelsSupport(provider) && !model.includes('/')) { + try { + const originalProvider = getProviderFromModel(model); + model = transformModelNameForProvider(model, originalProvider, provider); + logger.debug( + `Transformed model for ${provider}: ${updates.model ?? previous.model} -> ${model}` + ); + } catch { + // Model not in registry - pass through as-is, gateway may accept custom model IDs + logger.debug( + `Model '${model}' not in registry, passing through to ${provider} without transformation` + ); + } + } + + // Token defaults - always use model's effective max unless explicitly provided + const maxInputTokens = + updates.maxInputTokens ?? + getEffectiveMaxInputTokens({ provider, model, apiKey: apiKey || previous.apiKey }, logger); + + // BaseURL resolution + // Note: OpenRouter baseURL is handled by the factory (fixed endpoint, no user override) + let baseURL: string | undefined; + if (updates.baseURL) { + baseURL = updates.baseURL; + } else if (supportsBaseURL(provider)) { + baseURL = previous.baseURL; + } else { + baseURL = undefined; + } + + // Vertex AI validation - requires GOOGLE_VERTEX_PROJECT for ADC authentication + // This upfront check provides immediate feedback rather than failing at first API call + if (provider === 'vertex') { + const projectId = process.env.GOOGLE_VERTEX_PROJECT; + if (!projectId || !projectId.trim()) { + warnings.push({ + code: LLMErrorCode.CONFIG_MISSING, + message: + 'GOOGLE_VERTEX_PROJECT environment variable is required for Vertex AI. ' + + 'Set it to your GCP project ID and ensure ADC is configured via `gcloud auth application-default login`', + severity: 'error', + scope: ErrorScope.LLM, + type: ErrorType.USER, + context: { provider, model }, + }); + } + } + + // Amazon Bedrock validation - requires AWS_REGION for the endpoint URL + // Auth can be either: + // 1. AWS_BEARER_TOKEN_BEDROCK (API key - simplest) + // 2. AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY (IAM credentials) + if (provider === 'bedrock') { + const region = process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION; + if (!region || !region.trim()) { + warnings.push({ + code: LLMErrorCode.CONFIG_MISSING, + message: + 'AWS_REGION environment variable is required for Amazon Bedrock. ' + + 'Also set either AWS_BEARER_TOKEN_BEDROCK (API key) or ' + + 'AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY (IAM credentials).', + severity: 'error', + scope: ErrorScope.LLM, + type: ErrorType.USER, + context: { provider, model }, + }); + } + } + + // OpenRouter model validation with cache refresh + if (provider === 'openrouter') { + let lookupStatus = lookupOpenRouterModel(model); + + if (lookupStatus === 'unknown') { + // Cache stale/empty - try to refresh before validating + try { + await refreshOpenRouterModelCache({ apiKey }); + lookupStatus = lookupOpenRouterModel(model); + } catch { + // Network failed - keep 'unknown' status, allow gracefully + logger.debug( + `OpenRouter model cache refresh failed, allowing model '${model}' without validation` + ); + } + } + + if (lookupStatus === 'invalid') { + // Model definitively not found in fresh cache - this is an error + warnings.push({ + code: LLMErrorCode.MODEL_INCOMPATIBLE, + message: `Model '${model}' not found in OpenRouter catalog. Check model ID at https://openrouter.ai/models`, + severity: 'error', + scope: ErrorScope.LLM, + type: ErrorType.USER, + context: { provider, model }, + }); + } + // 'unknown' after failed refresh = allow (network issue, graceful degradation) + } + + return { + candidate: { + provider, + model, + apiKey, + baseURL, + maxIterations: updates.maxIterations ?? previous.maxIterations, + maxInputTokens, + maxOutputTokens: updates.maxOutputTokens ?? previous.maxOutputTokens, + temperature: updates.temperature ?? previous.temperature, + }, + warnings, + }; +} + +// Passes the input candidate through the schema and returns a result +export function validateLLMConfig( + candidate: LLMConfig, + warnings: Issue[] +): Result { + // Final validation (business rules + shape) + const parsed = LLMConfigSchema.safeParse(candidate); + if (!parsed.success) { + return fail(zodToIssues(parsed.error, 'error')); + } + + // Schema validation now handles apiKey non-empty validation + + // Check for short API key (warning) + if (parsed.data.apiKey && parsed.data.apiKey.length < 10) { + warnings.push({ + code: LLMErrorCode.API_KEY_INVALID, + message: 'API key seems too short - please verify it is correct', + path: ['apiKey'], + severity: 'warning', + scope: ErrorScope.LLM, + type: ErrorType.USER, + context: { + provider: candidate.provider, + model: candidate.model, + }, + }); + } + + return ok(parsed.data, warnings); +} diff --git a/dexto/packages/core/src/llm/schemas.test.ts b/dexto/packages/core/src/llm/schemas.test.ts new file mode 100644 index 00000000..19fcfb5d --- /dev/null +++ b/dexto/packages/core/src/llm/schemas.test.ts @@ -0,0 +1,488 @@ +import { describe, it, expect, vi } from 'vitest'; + +// Mock logger to prevent initialization issues +vi.mock('@core/logger/index.js', () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); +import { z } from 'zod'; +import { LLMErrorCode } from './error-codes.js'; +import { + LLMConfigSchema, + LLMUpdatesSchema, + type LLMConfig, + type ValidatedLLMConfig, +} from './schemas.js'; +import { LLM_PROVIDERS } from './types.js'; +import { + getSupportedModels, + getMaxInputTokensForModel, + requiresBaseURL, + supportsBaseURL, + getDefaultModelForProvider, + acceptsAnyModel, +} from './registry.js'; +import type { LLMProvider } from './types.js'; + +// Test helpers +class LLMTestHelpers { + static getValidConfigForProvider(provider: LLMProvider): LLMConfig { + const models = getSupportedModels(provider); + const defaultModel = getDefaultModelForProvider(provider) || models[0] || 'custom-model'; + + const baseConfig = { + provider, + model: defaultModel, + apiKey: 'test-key', + }; + + if (requiresBaseURL(provider)) { + return { ...baseConfig, baseURL: 'https://api.test.com/v1' }; + } + + return baseConfig; + } + + static getProviderRequiringBaseURL(): LLMProvider | null { + return LLM_PROVIDERS.find((p) => requiresBaseURL(p)) || null; + } + + static getProviderNotSupportingBaseURL(): LLMProvider | null { + return LLM_PROVIDERS.find((p) => !supportsBaseURL(p)) || null; + } +} + +describe('LLMConfigSchema', () => { + describe('Basic Structure Validation', () => { + it('should accept valid minimal config', () => { + const config = LLMTestHelpers.getValidConfigForProvider('openai'); + const result = LLMConfigSchema.parse(config); + + expect(result.provider).toBe('openai'); + expect(result.model).toBeTruthy(); + expect(result.apiKey).toBe('test-key'); + }); + + it('should apply default values', () => { + const config = LLMTestHelpers.getValidConfigForProvider('openai'); + const result = LLMConfigSchema.parse(config); + + expect(result.maxIterations).toBeUndefined(); + }); + + it('should preserve explicit optional values', () => { + const config: LLMConfig = { + provider: 'openai', + model: 'gpt-5', + apiKey: 'test-key', + maxIterations: 25, + temperature: 0.7, + maxOutputTokens: 4000, + }; + + const result = LLMConfigSchema.parse(config); + expect(result.maxIterations).toBe(25); + expect(result.temperature).toBe(0.7); + expect(result.maxOutputTokens).toBe(4000); + }); + }); + + describe('Required Fields Validation', () => { + it('should require provider field', () => { + const config = { + model: 'gpt-5', + apiKey: 'test-key', + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['provider']); + }); + + it('should require model field', () => { + const config = { + provider: 'openai', + apiKey: 'test-key', + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['model']); + }); + + it('should require apiKey field', () => { + const config = { + provider: 'openai', + model: 'gpt-5', + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['apiKey']); + }); + }); + + describe('Provider Validation', () => { + it('should accept all registry providers', () => { + for (const provider of LLM_PROVIDERS) { + const config = LLMTestHelpers.getValidConfigForProvider(provider); + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.provider).toBe(provider); + } + } + }); + + it('should reject invalid providers', () => { + const config = { + provider: 'invalid-provider', + model: 'test-model', + apiKey: 'test-key', + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_enum_value); + expect(result.error?.issues[0]?.path).toEqual(['provider']); + }); + + it('should be case sensitive for providers', () => { + const config = { + provider: 'OpenAI', // Should be 'openai' + model: 'gpt-5', + apiKey: 'test-key', + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_enum_value); + expect(result.error?.issues[0]?.path).toEqual(['provider']); + }); + }); + + describe('Model Validation', () => { + it('should accept known models for each provider', () => { + for (const provider of LLM_PROVIDERS) { + const models = getSupportedModels(provider); + if (models.length === 0) continue; // Skip providers that accept any model + + // Test first few models to avoid excessive test runs + const modelsToTest = models.slice(0, 3); + for (const model of modelsToTest) { + const config: LLMConfig = { + provider, + model, + apiKey: 'test-key', + ...(requiresBaseURL(provider) && { baseURL: 'https://api.test.com/v1' }), + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(true); + } + } + }); + + it('should reject unknown models for providers with restricted models', () => { + // Find a provider that has specific model restrictions + const provider = LLM_PROVIDERS.find((p) => !acceptsAnyModel(p)); + if (!provider) return; // Skip if no providers have model restrictions + + const config: LLMConfig = { + provider, + model: 'unknown-model-xyz-123', + apiKey: 'test-key', + ...(requiresBaseURL(provider) && { baseURL: 'https://api.test.com/v1' }), + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['model']); + expect((result.error?.issues[0] as any).params?.code).toBe( + LLMErrorCode.MODEL_INCOMPATIBLE + ); + }); + }); + + describe('Temperature Validation', () => { + it('should accept valid temperature values', () => { + const validTemperatures = [0, 0.1, 0.5, 0.7, 1.0]; + + for (const temperature of validTemperatures) { + const config: LLMConfig = { + ...LLMTestHelpers.getValidConfigForProvider('openai'), + temperature, + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.temperature).toBe(temperature); + } + } + }); + + it('should reject invalid temperature values', () => { + const invalidTemperatures = [-0.1, -1, 1.1, 2]; + + for (const temperature of invalidTemperatures) { + const config: LLMConfig = { + ...LLMTestHelpers.getValidConfigForProvider('openai'), + temperature, + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['temperature']); + } + }); + }); + + describe('BaseURL Validation', () => { + it('should require baseURL for providers that need it', () => { + const provider = LLMTestHelpers.getProviderRequiringBaseURL(); + if (!provider) return; // Skip if no providers require baseURL + + const config = { + provider, + model: 'custom-model', + apiKey: 'test-key', + // Missing baseURL + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['baseURL']); + expect((result.error?.issues[0] as any).params?.code).toBe( + LLMErrorCode.BASE_URL_MISSING + ); + }); + + it('should accept baseURL for providers that require it', () => { + const provider = LLMTestHelpers.getProviderRequiringBaseURL(); + if (!provider) return; + + const config: LLMConfig = { + provider, + model: 'custom-model', + apiKey: 'test-key', + baseURL: 'https://api.custom.com/v1', + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(true); + }); + + it('should reject baseURL for providers that do not support it', () => { + const provider = LLMTestHelpers.getProviderNotSupportingBaseURL(); + if (!provider) return; // Skip if all providers support baseURL + + const config: LLMConfig = { + ...LLMTestHelpers.getValidConfigForProvider(provider), + baseURL: 'https://api.custom.com/v1', + }; + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['provider']); + expect((result.error?.issues[0] as any).params?.code).toBe( + LLMErrorCode.BASE_URL_INVALID + ); + }); + }); + + describe('MaxInputTokens Validation', () => { + it('should accept valid maxInputTokens within model limits', () => { + // Find a provider with specific models to test token limits + const provider = LLM_PROVIDERS.find((p) => !acceptsAnyModel(p)); + if (!provider) return; + + const models = getSupportedModels(provider); + const model = models[0]!; + const maxTokens = getMaxInputTokensForModel(provider, model); + + const config: LLMConfig = { + provider, + model, + apiKey: 'test-key', + maxInputTokens: Math.floor(maxTokens / 2), // Well within limit + ...(requiresBaseURL(provider) && { baseURL: 'https://api.test.com/v1' }), + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(true); + }); + + it('should reject maxInputTokens exceeding model limits', () => { + const provider = LLM_PROVIDERS.find((p) => !acceptsAnyModel(p)); + if (!provider) return; + + const models = getSupportedModels(provider); + const model = models[0]!; + const maxTokens = getMaxInputTokensForModel(provider, model); + + const config: LLMConfig = { + provider, + model, + apiKey: 'test-key', + maxInputTokens: maxTokens + 1000, // Exceed limit + ...(requiresBaseURL(provider) && { baseURL: 'https://api.test.com/v1' }), + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['maxInputTokens']); + expect((result.error?.issues[0] as any).params?.code).toBe( + LLMErrorCode.TOKENS_EXCEEDED + ); + }); + + it('should allow maxInputTokens for providers that accept any model', () => { + const provider = LLMTestHelpers.getProviderRequiringBaseURL(); + if (!provider || !acceptsAnyModel(provider)) return; + + const config: LLMConfig = { + provider, + model: 'custom-model', + apiKey: 'test-key', + baseURL: 'https://api.custom.com/v1', + maxInputTokens: 50000, + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(true); + }); + }); + + describe('Edge Cases', () => { + it('should reject empty string values', () => { + const testCases = [ + { provider: '', model: 'gpt-5', apiKey: 'key' }, + { provider: 'openai', model: '', apiKey: 'key' }, + { provider: 'openai', model: 'gpt-5', apiKey: '' }, + ]; + + for (const config of testCases) { + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + } + }); + + it('should reject whitespace-only values', () => { + const testCases = [ + { provider: ' ', model: 'gpt-5', apiKey: 'key' }, + { provider: 'openai', model: ' ', apiKey: 'key' }, + { provider: 'openai', model: 'gpt-5', apiKey: ' ' }, + ]; + + for (const config of testCases) { + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + } + }); + + it('should handle type coercion for numeric fields', () => { + const config: any = { + ...LLMTestHelpers.getValidConfigForProvider('openai'), + maxIterations: '25', // String that should coerce to number + temperature: '0.7', // String that should coerce to number + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.maxIterations).toBe(25); + expect(result.data.temperature).toBe(0.7); + } + }); + + it('should reject invalid numeric coercion', () => { + const config: any = { + ...LLMTestHelpers.getValidConfigForProvider('openai'), + maxIterations: 'not-a-number', + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues[0]?.path).toEqual(['maxIterations']); + } + }); + }); + + describe('Strict Validation', () => { + it('should reject unknown fields', () => { + const config: any = { + ...LLMTestHelpers.getValidConfigForProvider('openai'), + unknownField: 'should-fail', + }; + + const result = LLMConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + + describe('Type Safety', () => { + it('should handle input and output types correctly', () => { + const input: LLMConfig = LLMTestHelpers.getValidConfigForProvider('openai'); + const result: ValidatedLLMConfig = LLMConfigSchema.parse(input); + + // Should have applied defaults + expect(result.maxIterations).toBeUndefined(); + + // Should preserve input values + expect(result.provider).toBe(input.provider); + expect(result.model).toBe(input.model); + expect(result.apiKey).toBe(input.apiKey); + }); + + it('should maintain type consistency', () => { + const config = LLMTestHelpers.getValidConfigForProvider('anthropic'); + const result = LLMConfigSchema.parse(config); + + // TypeScript should infer correct types + expect(typeof result.provider).toBe('string'); + expect(typeof result.model).toBe('string'); + expect(typeof result.apiKey).toBe('string'); + expect(result.maxIterations).toBeUndefined(); + }); + }); + + describe('LLMUpdatesSchema', () => { + describe('Update Requirements', () => { + it('should pass validation when model is provided', () => { + const updates = { model: 'gpt-5' }; + expect(() => LLMUpdatesSchema.parse(updates)).not.toThrow(); + }); + + it('should pass validation when provider is provided', () => { + const updates = { provider: 'openai' }; + expect(() => LLMUpdatesSchema.parse(updates)).not.toThrow(); + }); + + it('should pass validation when both model and provider are provided', () => { + const updates = { model: 'gpt-5', provider: 'openai' }; + expect(() => LLMUpdatesSchema.parse(updates)).not.toThrow(); + }); + + it('should reject empty updates object', () => { + const updates = {}; + expect(() => LLMUpdatesSchema.parse(updates)).toThrow(); + }); + + it('should reject updates with only non-key fields (no model/provider)', () => { + const updates = { maxIterations: 10 } as const; + expect(() => LLMUpdatesSchema.parse(updates)).toThrow(); + }); + + it('should pass validation when model/provider with other fields', () => { + const updates = { model: 'gpt-5', maxIterations: 10 }; + expect(() => LLMUpdatesSchema.parse(updates)).not.toThrow(); + }); + }); + }); +}); diff --git a/dexto/packages/core/src/llm/schemas.ts b/dexto/packages/core/src/llm/schemas.ts new file mode 100644 index 00000000..1009eaf2 --- /dev/null +++ b/dexto/packages/core/src/llm/schemas.ts @@ -0,0 +1,314 @@ +import { LLMErrorCode } from './error-codes.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { DextoRuntimeError } from '@core/errors/index.js'; +import { NonEmptyTrimmed, EnvExpandedString, OptionalURL } from '@core/utils/result.js'; +import { getPrimaryApiKeyEnvVar } from '@core/utils/api-key-resolver.js'; +import { z } from 'zod'; +import { + supportsBaseURL, + requiresBaseURL, + acceptsAnyModel, + supportsCustomModels, + getSupportedModels, + isValidProviderModel, + getMaxInputTokensForModel, + requiresApiKey, +} from './registry.js'; +import { LLM_PROVIDERS } from './types.js'; + +/** + * Options for LLM config validation + */ +export interface LLMValidationOptions { + /** + * When true, enforces API key and baseURL requirements. + * When false (relaxed mode), allows missing API keys/baseURLs for interactive configuration. + * + * Use strict mode for: + * - Server/API mode (headless, needs full config) + * - MCP mode (headless) + * + * Use relaxed mode for: + * - Web UI (user can configure via settings) + * - CLI (user can configure interactively) + * + * @default true + */ + strict?: boolean; +} + +/** + * Default-free field definitions for LLM configuration. + * Used to build both the full config schema (with defaults) and the updates schema (no defaults). + */ +const LLMConfigFields = { + provider: z + .enum(LLM_PROVIDERS) + .describe("LLM provider (e.g., 'openai', 'anthropic', 'google', 'groq')"), + + model: NonEmptyTrimmed.describe('Specific model name for the selected provider'), + + // Expand $ENV refs and trim; final validation happens with provider context + // Optional for providers that don't need API keys (Ollama, vLLM, etc.) + apiKey: EnvExpandedString() + .optional() + .describe('API key for provider; can be given directly or via $ENV reference'), + + maxIterations: z.coerce.number().int().positive().describe('Max iterations for agentic loops'), + + baseURL: OptionalURL.describe( + 'Base URL for provider (e.g., https://api.openai.com/v1). Only certain providers support this.' + ), + + maxInputTokens: z.coerce + .number() + .int() + .positive() + .optional() + .describe('Max input tokens for history; required for unknown models'), + + maxOutputTokens: z.coerce + .number() + .int() + .positive() + .optional() + .describe('Max tokens for model output'), + + temperature: z.coerce + .number() + .min(0) + .max(1) + .optional() + .describe('Randomness: 0 deterministic, 1 creative'), + + allowedMediaTypes: z + .array(z.string()) + .optional() + .describe( + 'MIME type patterns for media expansion (e.g., "image/*", "application/pdf"). ' + + 'If omitted, uses model capabilities from registry. Supports wildcards.' + ), + + // Provider-specific options + + /** + * OpenAI reasoning effort level for reasoning-capable models (o1, o3, codex, gpt-5.x). + * Controls how many reasoning tokens the model generates before producing a response. + * - 'none': No reasoning, fastest responses + * - 'minimal': Barely any reasoning, very fast responses + * - 'low': Light reasoning, fast responses + * - 'medium': Balanced reasoning (OpenAI's recommended daily driver) + * - 'high': Thorough reasoning for complex tasks + * - 'xhigh': Extra high reasoning for quality-critical, non-latency-sensitive tasks + */ + reasoningEffort: z + .enum(['none', 'minimal', 'low', 'medium', 'high', 'xhigh']) + .optional() + .describe( + 'OpenAI reasoning effort level for reasoning models (o1, o3, codex). ' + + "Options: 'none', 'minimal', 'low', 'medium' (recommended), 'high', 'xhigh'" + ), +} as const; +/** Business rules + compatibility checks */ + +// Base LLM config object schema (before validation/branding) - can be extended +export const LLMConfigBaseSchema = z + .object({ + provider: LLMConfigFields.provider, + model: LLMConfigFields.model, + // apiKey is optional at schema level - validated based on provider in superRefine + apiKey: LLMConfigFields.apiKey, + // Apply defaults only for complete config validation + maxIterations: z.coerce.number().int().positive().optional(), + baseURL: LLMConfigFields.baseURL, + maxInputTokens: LLMConfigFields.maxInputTokens, + maxOutputTokens: LLMConfigFields.maxOutputTokens, + temperature: LLMConfigFields.temperature, + allowedMediaTypes: LLMConfigFields.allowedMediaTypes, + // Provider-specific options + reasoningEffort: LLMConfigFields.reasoningEffort, + }) + .strict(); + +/** + * Creates an LLM config schema with configurable validation strictness. + * + * @param options.strict - When true (default), enforces API key and baseURL requirements. + * When false, allows missing credentials for interactive configuration. + */ +export function createLLMConfigSchema(options: LLMValidationOptions = {}) { + const { strict = true } = options; + + return LLMConfigBaseSchema.superRefine((data, ctx) => { + const baseURLIsSet = data.baseURL != null && data.baseURL.trim() !== ''; + const maxInputTokensIsSet = data.maxInputTokens != null; + + // API key validation with provider context + // In relaxed mode, skip API key validation to allow launching app for interactive config + // Skip validation for providers that don't require API keys: + // - openai-compatible: local providers like Ollama, vLLM, LocalAI + // - litellm: self-hosted proxy handles auth internally + // - vertex: uses Google Cloud ADC + // - bedrock: uses AWS credentials + if (strict && requiresApiKey(data.provider) && !data.apiKey?.trim()) { + const primaryVar = getPrimaryApiKeyEnvVar(data.provider); + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['apiKey'], + message: `Missing API key for provider '${data.provider}' – set $${primaryVar}`, + params: { + code: LLMErrorCode.API_KEY_MISSING, + scope: ErrorScope.LLM, + type: ErrorType.USER, + provider: data.provider, + envVar: primaryVar, + }, + }); + } + + if (baseURLIsSet) { + if (!supportsBaseURL(data.provider)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['provider'], + message: + `Provider '${data.provider}' does not support baseURL. ` + + `Use an 'openai-compatible' provider if you need a custom base URL.`, + params: { + code: LLMErrorCode.BASE_URL_INVALID, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }, + }); + } + } else if (strict && requiresBaseURL(data.provider)) { + // In relaxed mode, skip baseURL requirement validation + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['baseURL'], + message: `Provider '${data.provider}' requires a 'baseURL'.`, + params: { + code: LLMErrorCode.BASE_URL_MISSING, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }, + }); + } + + // Model and token validation always runs (not affected by strict mode) + if (!baseURLIsSet || supportsBaseURL(data.provider)) { + // Skip model validation for providers that accept any model OR support custom models + if (!acceptsAnyModel(data.provider) && !supportsCustomModels(data.provider)) { + const supportedModelsList = getSupportedModels(data.provider); + if (!isValidProviderModel(data.provider, data.model)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['model'], + message: + `Model '${data.model}' is not supported for provider '${data.provider}'. ` + + `Supported: ${supportedModelsList.join(', ')}`, + params: { + code: LLMErrorCode.MODEL_INCOMPATIBLE, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }, + }); + } + } + + // Skip token cap validation for providers that accept any model OR support custom models + if ( + maxInputTokensIsSet && + !acceptsAnyModel(data.provider) && + !supportsCustomModels(data.provider) + ) { + try { + const cap = getMaxInputTokensForModel(data.provider, data.model); + if (data.maxInputTokens! > cap) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['maxInputTokens'], + message: + `Max input tokens for model '${data.model}' is ${cap}. ` + + `You provided ${data.maxInputTokens}`, + params: { + code: LLMErrorCode.TOKENS_EXCEEDED, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }, + }); + } + } catch (error: unknown) { + if ( + error instanceof DextoRuntimeError && + error.code === LLMErrorCode.MODEL_UNKNOWN + ) { + // Model not found in registry + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['model'], + message: error.message, + params: { + code: error.code, + scope: error.scope, + type: error.type, + }, + }); + } else { + // Unexpected error + const message = + error instanceof Error ? error.message : 'Unknown error occurred'; + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['model'], + message, + params: { + code: LLMErrorCode.REQUEST_INVALID_SCHEMA, + scope: ErrorScope.LLM, + type: ErrorType.SYSTEM, + }, + }); + } + } + } + } + // Note: OpenRouter model validation happens in resolver.ts during switchLLM only + // to avoid network calls during startup/serverless cold starts + }) // Brand the validated type so it can be distinguished at compile time + .brand<'ValidatedLLMConfig'>(); +} + +/** + * Default LLM config schema with strict validation (backwards compatible). + * Use createLLMConfigSchema({ strict: false }) for relaxed validation. + */ +export const LLMConfigSchema = createLLMConfigSchema({ strict: true }); + +/** + * Relaxed LLM config schema that allows missing API keys and baseURLs. + * Use this for interactive modes (CLI, WebUI) where users can configure later. + */ +export const LLMConfigSchemaRelaxed = createLLMConfigSchema({ strict: false }); + +// Input type and output types for the zod schema +export type LLMConfig = z.input; +export type ValidatedLLMConfig = z.output; +// PATCH-like schema for updates (switch flows) + +// TODO: when moving to zod v4 we might be able to set this as strict +export const LLMUpdatesSchema = z + .object({ ...LLMConfigFields }) + .partial() + .superRefine((data, ctx) => { + // Require at least one meaningful change field: model or provider + if (!data.model && !data.provider) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'At least model or provider must be specified for LLM switch', + path: [], + }); + } + }); +export type LLMUpdates = z.input; +// Re-export context type from llm module +export type { LLMUpdateContext } from '../llm/types.js'; diff --git a/dexto/packages/core/src/llm/services/factory.ts b/dexto/packages/core/src/llm/services/factory.ts new file mode 100644 index 00000000..fb198780 --- /dev/null +++ b/dexto/packages/core/src/llm/services/factory.ts @@ -0,0 +1,267 @@ +import { ToolManager } from '../../tools/tool-manager.js'; +import { ValidatedLLMConfig } from '../schemas.js'; +import { LLMError } from '../errors.js'; +import { createOpenAI } from '@ai-sdk/openai'; +import { createGoogleGenerativeAI } from '@ai-sdk/google'; +import { createAnthropic } from '@ai-sdk/anthropic'; +import { createGroq } from '@ai-sdk/groq'; +import { createXai } from '@ai-sdk/xai'; +import { createVertex } from '@ai-sdk/google-vertex'; +import { createVertexAnthropic } from '@ai-sdk/google-vertex/anthropic'; +import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock'; +import { VercelLLMService } from './vercel.js'; +import { LanguageModel } from 'ai'; +import { SessionEventBus } from '../../events/index.js'; +import { createCohere } from '@ai-sdk/cohere'; +import { createLocalLanguageModel } from '../providers/local/ai-sdk-adapter.js'; +import type { IConversationHistoryProvider } from '../../session/history/types.js'; +import type { SystemPromptManager } from '../../systemPrompt/manager.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { requiresApiKey } from '../registry.js'; +import { getPrimaryApiKeyEnvVar, resolveApiKeyForProvider } from '../../utils/api-key-resolver.js'; +import type { CompactionConfigInput } from '../../context/compaction/schemas.js'; + +// Dexto Gateway headers for usage tracking +const DEXTO_GATEWAY_HEADERS = { + SESSION_ID: 'X-Dexto-Session-ID', + CLIENT_SOURCE: 'X-Dexto-Source', + CLIENT_VERSION: 'X-Dexto-Version', +} as const; + +/** + * Context for model creation, including session info for usage tracking. + */ +export interface DextoProviderContext { + /** Session ID for usage tracking */ + sessionId?: string; + /** Client source for usage attribution (cli, web, sdk) */ + clientSource?: 'cli' | 'web' | 'sdk'; +} + +/** + * Create a Vercel AI SDK LanguageModel from config. + * + * With explicit providers, the config's provider field directly determines + * where requests go. No auth-dependent routing - what you configure is what runs. + * + * @param llmConfig - LLM configuration from agent config + * @param context - Optional context for usage tracking (session ID, etc.) + * @returns Vercel AI SDK LanguageModel instance + */ +export function createVercelModel( + llmConfig: ValidatedLLMConfig, + context?: DextoProviderContext +): LanguageModel { + const { provider, model, baseURL } = llmConfig; + const apiKey = llmConfig.apiKey || resolveApiKeyForProvider(provider); + + // Runtime check: if provider requires API key but none is configured, fail with helpful message + if (requiresApiKey(provider) && !apiKey?.trim()) { + const envVar = getPrimaryApiKeyEnvVar(provider); + throw LLMError.apiKeyMissing(provider, envVar); + } + + switch (provider.toLowerCase()) { + case 'openai': { + // Regular OpenAI - strict compatibility, no baseURL + return createOpenAI({ apiKey: apiKey ?? '' })(model); + } + case 'openai-compatible': { + // OpenAI-compatible - requires baseURL, uses chat completions endpoint + // Must use .chat() as most compatible endpoints (like Ollama) don't support Responses API + const compatibleBaseURL = + baseURL?.replace(/\/$/, '') || process.env.OPENAI_BASE_URL?.replace(/\/$/, ''); + if (!compatibleBaseURL) { + throw LLMError.baseUrlMissing('openai-compatible'); + } + return createOpenAI({ apiKey: apiKey ?? '', baseURL: compatibleBaseURL }).chat(model); + } + case 'openrouter': { + // OpenRouter - unified API gateway for 100+ models (BYOK) + // Model IDs are in OpenRouter format (e.g., 'anthropic/claude-sonnet-4-5-20250929') + const orBaseURL = baseURL || 'https://openrouter.ai/api/v1'; + return createOpenAI({ apiKey: apiKey ?? '', baseURL: orBaseURL }).chat(model); + } + case 'litellm': { + // LiteLLM - OpenAI-compatible proxy for 100+ LLM providers + // User must provide their own LiteLLM proxy URL + if (!baseURL) { + throw LLMError.baseUrlMissing('litellm'); + } + return createOpenAI({ apiKey: apiKey ?? '', baseURL }).chat(model); + } + case 'glama': { + // Glama - OpenAI-compatible gateway for multiple LLM providers + // Fixed endpoint, no user configuration needed + const glamaBaseURL = 'https://glama.ai/api/gateway/openai/v1'; + return createOpenAI({ apiKey: apiKey ?? '', baseURL: glamaBaseURL }).chat(model); + } + case 'dexto': { + // Dexto Gateway - OpenAI-compatible proxy with per-request billing + // Routes through api.dexto.ai to OpenRouter, deducts from user balance + // Requires DEXTO_API_KEY from `dexto login` + // + // Model IDs are in OpenRouter format (e.g., 'anthropic/claude-sonnet-4-5-20250929') + // Users explicitly choose `provider: dexto` in their config + // + // Note: 402 "insufficient credits" errors are handled in turn-executor.ts mapProviderError() + const dextoBaseURL = 'https://api.dexto.ai/v1'; + + // Build headers for usage tracking + const headers: Record = { + [DEXTO_GATEWAY_HEADERS.CLIENT_SOURCE]: context?.clientSource ?? 'cli', + }; + if (context?.sessionId) { + headers[DEXTO_GATEWAY_HEADERS.SESSION_ID] = context.sessionId; + } + if (process.env.DEXTO_CLI_VERSION) { + headers[DEXTO_GATEWAY_HEADERS.CLIENT_VERSION] = process.env.DEXTO_CLI_VERSION; + } + + // Model is already in OpenRouter format - pass through directly + return createOpenAI({ apiKey: apiKey ?? '', baseURL: dextoBaseURL, headers }).chat( + model + ); + } + case 'vertex': { + // Google Vertex AI - supports both Gemini and Claude models + // Auth via Application Default Credentials (ADC) + // + // TODO: Integrate with agent config (llmConfig.vertex?.projectId) as primary, + // falling back to env vars. This would allow per-agent Vertex configuration. + const projectId = process.env.GOOGLE_VERTEX_PROJECT; + if (!projectId) { + throw LLMError.missingConfig( + 'vertex', + 'GOOGLE_VERTEX_PROJECT environment variable' + ); + } + const location = process.env.GOOGLE_VERTEX_LOCATION; + + // Route based on model type: Claude models use /anthropic subpath + if (model.includes('claude')) { + // Claude models on Vertex use the /anthropic subpath export + // Default to us-east5 for Claude (limited region availability) + return createVertexAnthropic({ + project: projectId, + location: location || 'us-east5', + })(model); + } + + // Gemini models use the main export + // Default to us-central1 for Gemini (widely available) + return createVertex({ + project: projectId, + location: location || 'us-central1', + })(model); + } + case 'bedrock': { + // Amazon Bedrock - AWS-hosted gateway for Claude, Nova, Llama, Mistral + // Auth via AWS credentials (env vars or credential provider) + // + // TODO: Add credentialProvider support for: + // - ~/.aws/credentials file profiles (fromIni) + // - AWS SSO sessions (fromSSO) + // - IAM roles on EC2/Lambda (fromNodeProviderChain) + // This would require adding @aws-sdk/credential-providers dependency + // and exposing a config option like llmConfig.bedrock?.credentialProvider + // + // Current implementation: SDK reads directly from env vars: + // - AWS_REGION (required) + // - AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY (required) + // - AWS_SESSION_TOKEN (optional, for temporary credentials) + const region = process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION; + if (!region) { + throw LLMError.missingConfig( + 'bedrock', + 'AWS_REGION or AWS_DEFAULT_REGION environment variable' + ); + } + + // Auto-detect cross-region inference profile prefix based on user's region + // Users can override by explicitly using prefixed model IDs (e.g., eu.anthropic.claude...) + let modelId = model; + const hasRegionPrefix = + model.startsWith('eu.') || model.startsWith('us.') || model.startsWith('global.'); + if (!hasRegionPrefix) { + const prefix = region.startsWith('eu-') ? 'eu.' : 'us.'; + modelId = `${prefix}${model}`; + } + + // SDK automatically reads AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN + return createAmazonBedrock({ region })(modelId); + } + case 'anthropic': + return createAnthropic({ apiKey: apiKey ?? '' })(model); + case 'google': + return createGoogleGenerativeAI({ apiKey: apiKey ?? '' })(model); + case 'groq': + return createGroq({ apiKey: apiKey ?? '' })(model); + case 'xai': + return createXai({ apiKey: apiKey ?? '' })(model); + case 'cohere': + return createCohere({ apiKey: apiKey ?? '' })(model); + case 'ollama': { + // Ollama - local model server with OpenAI-compatible API + // Uses the /v1 endpoint for AI SDK compatibility + // Default URL: http://localhost:11434 + const ollamaBaseURL = baseURL || 'http://localhost:11434/v1'; + // Ollama doesn't require an API key, but the SDK needs a non-empty string + return createOpenAI({ apiKey: 'ollama', baseURL: ollamaBaseURL }).chat(model); + } + case 'local': { + // Native node-llama-cpp execution via AI SDK adapter. + // Model is loaded lazily on first use. + return createLocalLanguageModel({ + modelId: model, + }); + } + default: + throw LLMError.unsupportedProvider(provider); + } +} + +/** + * Create an LLM service instance using the Vercel AI SDK. + * All providers are routed through the unified Vercel service. + * + * @param config LLM configuration from the config file + * @param toolManager Unified tool manager instance + * @param systemPromptManager Prompt manager for system prompts + * @param historyProvider History provider for conversation persistence + * @param sessionEventBus Session-level event bus for emitting LLM events + * @param sessionId Session ID + * @param resourceManager Resource manager for blob storage and resource access + * @param logger Logger instance for dependency injection + * @param compactionStrategy Optional compaction strategy for context management + * @param compactionConfig Optional compaction configuration for thresholds + * @returns VercelLLMService instance + */ +export function createLLMService( + config: ValidatedLLMConfig, + toolManager: ToolManager, + systemPromptManager: SystemPromptManager, + historyProvider: IConversationHistoryProvider, + sessionEventBus: SessionEventBus, + sessionId: string, + resourceManager: import('../../resources/index.js').ResourceManager, + logger: IDextoLogger, + compactionStrategy?: import('../../context/compaction/types.js').ICompactionStrategy | null, + compactionConfig?: CompactionConfigInput +): VercelLLMService { + const model = createVercelModel(config, { sessionId }); + + return new VercelLLMService( + toolManager, + model, + systemPromptManager, + historyProvider, + sessionEventBus, + config, + sessionId, + resourceManager, + logger, + compactionStrategy, + compactionConfig + ); +} diff --git a/dexto/packages/core/src/llm/services/index.ts b/dexto/packages/core/src/llm/services/index.ts new file mode 100644 index 00000000..d4702960 --- /dev/null +++ b/dexto/packages/core/src/llm/services/index.ts @@ -0,0 +1 @@ +export * from './types.js'; diff --git a/dexto/packages/core/src/llm/services/test-utils.integration.ts b/dexto/packages/core/src/llm/services/test-utils.integration.ts new file mode 100644 index 00000000..af1dcce3 --- /dev/null +++ b/dexto/packages/core/src/llm/services/test-utils.integration.ts @@ -0,0 +1,234 @@ +import { DextoAgent } from '../../agent/DextoAgent.js'; +import { + resolveApiKeyForProvider, + getPrimaryApiKeyEnvVar, + PROVIDER_API_KEY_MAP, +} from '../../utils/api-key-resolver.js'; +import type { LLMProvider } from '../types.js'; +import type { AgentConfig } from '../../agent/schemas.js'; + +/** + * Shared utilities for LLM service integration tests + */ + +export interface TestEnvironment { + agent: DextoAgent; + sessionId: string; + cleanup: () => Promise; +} + +/** + * Creates a test environment with real dependencies (no mocks) + * Uses DextoAgent to handle complex initialization properly + */ +export async function createTestEnvironment( + config: AgentConfig, + sessionId: string = 'test-session' +): Promise { + const agent = new DextoAgent(config); + await agent.start(); + + return { + agent, + sessionId, + cleanup: async () => { + if (agent.isStarted()) { + // Don't wait - just stop the agent immediately + // The agent.stop() will handle graceful shutdown + await agent.stop(); + } + }, + }; +} + +// Standard test cases have been moved inline to each test file +// This reduces complexity and makes tests more explicit + +/** + * Test configuration helpers that create full AgentConfig objects + */ +export const TestConfigs = { + /** + * Creates OpenAI test config + */ + createOpenAIConfig(): AgentConfig { + const provider: LLMProvider = 'openai'; + const apiKey = resolveApiKeyForProvider(provider); + if (!apiKey) { + throw new Error( + `${getPrimaryApiKeyEnvVar(provider)} environment variable is required for OpenAI integration tests` + ); + } + + return { + systemPrompt: 'You are a helpful assistant for testing purposes.', + llm: { + provider, + model: 'gpt-4o-mini', // Use cheapest non-reasoning model for testing + apiKey, + maxOutputTokens: 1000, // Enough for reasoning models (reasoning + answer) + temperature: 0, // Deterministic responses + maxIterations: 1, // Minimal tool iterations + }, + mcpServers: {}, + storage: { + cache: { type: 'in-memory' }, + database: { type: 'in-memory' }, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }, + sessions: { + maxSessions: 10, + sessionTTL: 60000, // 60s for tests + }, + logger: { + level: 'info', + transports: [{ type: 'console' }], + }, + toolConfirmation: { + mode: 'auto-approve', // Tests don't have interactive approval + timeout: 120000, + }, + elicitation: { + enabled: false, // Tests don't handle elicitation + timeout: 120000, + }, + }; + }, + + /** + * Creates Anthropic test config + */ + createAnthropicConfig(): AgentConfig { + const provider: LLMProvider = 'anthropic'; + const apiKey = resolveApiKeyForProvider(provider); + if (!apiKey) { + throw new Error( + `${getPrimaryApiKeyEnvVar(provider)} environment variable is required for Anthropic integration tests` + ); + } + + return { + systemPrompt: 'You are a helpful assistant for testing purposes.', + llm: { + provider, + model: 'claude-haiku-4-5-20251001', // Use cheapest model for testing + apiKey, + maxOutputTokens: 1000, // Enough for reasoning models (reasoning + answer) + temperature: 0, + maxIterations: 1, + }, + mcpServers: {}, + storage: { + cache: { type: 'in-memory' }, + database: { type: 'in-memory' }, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }, + sessions: { + maxSessions: 10, + sessionTTL: 60000, + }, + logger: { + level: 'info', + transports: [{ type: 'console' }], + }, + toolConfirmation: { + mode: 'auto-approve', // Tests don't have interactive approval + timeout: 120000, + }, + elicitation: { + enabled: false, // Tests don't handle elicitation + timeout: 120000, + }, + }; + }, + + /** + * Creates Vercel test config - parametric for different providers/models + */ + createVercelConfig(provider: LLMProvider = 'openai', model?: string): AgentConfig { + const apiKey = resolveApiKeyForProvider(provider); + // Only enforce API key check for providers that require it (exclude local, ollama, vertex with empty key maps) + if (!apiKey && providerRequiresApiKey(provider)) { + throw new Error( + `${getPrimaryApiKeyEnvVar(provider)} environment variable is required for Vercel integration tests with ${provider}` + ); + } + + // Default models for common providers + const defaultModels: Record = { + openai: 'gpt-4o-mini', + anthropic: 'claude-haiku-4-5-20251001', + google: 'gemini-2.0-flash', + groq: 'llama-3.1-8b-instant', + xai: 'grok-beta', + cohere: 'command-r', + 'openai-compatible': 'gpt-5-mini', + openrouter: 'anthropic/claude-3.5-haiku', // OpenRouter model format: provider/model + litellm: 'gpt-4', // LiteLLM model names follow the provider's convention + glama: 'openai/gpt-4o', // Glama model format: provider/model + vertex: 'gemini-2.5-pro', // Vertex AI uses ADC auth, not API keys + bedrock: 'anthropic.claude-3-5-haiku-20241022-v1:0', // Bedrock uses AWS credentials, not API keys + local: 'llama-3.2-3b-q4', // Native node-llama-cpp GGUF models + ollama: 'llama3.2', // Ollama server models + dexto: 'anthropic/claude-4.5-sonnet', // Dexto gateway (OpenRouter model format) + }; + + return { + systemPrompt: 'You are a helpful assistant for testing purposes.', + llm: { + provider, + model: model || defaultModels[provider], + apiKey, + maxOutputTokens: 1000, // Enough for reasoning models (reasoning + answer) + temperature: 0, + maxIterations: 1, + }, + mcpServers: {}, + storage: { + cache: { type: 'in-memory' }, + database: { type: 'in-memory' }, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }, + sessions: { + maxSessions: 10, + sessionTTL: 60000, + }, + logger: { + level: 'info', + transports: [{ type: 'console' }], + }, + toolConfirmation: { + mode: 'auto-approve', // Tests don't have interactive approval + timeout: 120000, + }, + elicitation: { + enabled: false, // Tests don't handle elicitation + timeout: 120000, + }, + }; + }, +} as const; + +/** + * Helper to check if a provider requires an API key + * Providers with empty arrays in PROVIDER_API_KEY_MAP don't require API keys (e.g., local, ollama, vertex) + */ +export function providerRequiresApiKey(provider: LLMProvider): boolean { + const envVars = PROVIDER_API_KEY_MAP[provider]; + return envVars && envVars.length > 0; +} + +/** + * Helper to check if API key is available for a provider + * Used to skip tests when API keys are not configured + */ +export function requiresApiKey(provider: LLMProvider): boolean { + return !!resolveApiKeyForProvider(provider); +} + +/** + * Cleanup helper + */ +export async function cleanupTestEnvironment(_env: TestEnvironment): Promise { + await _env.cleanup(); +} diff --git a/dexto/packages/core/src/llm/services/types.ts b/dexto/packages/core/src/llm/services/types.ts new file mode 100644 index 00000000..67359d7b --- /dev/null +++ b/dexto/packages/core/src/llm/services/types.ts @@ -0,0 +1,22 @@ +import { LanguageModel } from 'ai'; +import type { LLMProvider } from '../types.js'; + +/** + * Configuration object returned by LLMService.getConfig() + */ +export type LLMServiceConfig = { + provider: LLMProvider; + model: LanguageModel; + configuredMaxInputTokens?: number | null; + modelMaxInputTokens?: number | null; +}; + +/** + * Token usage statistics from LLM + */ +export interface LLMTokenUsage { + inputTokens: number; + outputTokens: number; + reasoningTokens?: number; + totalTokens: number; +} diff --git a/dexto/packages/core/src/llm/services/vercel.integration.test.ts b/dexto/packages/core/src/llm/services/vercel.integration.test.ts new file mode 100644 index 00000000..294d62a9 --- /dev/null +++ b/dexto/packages/core/src/llm/services/vercel.integration.test.ts @@ -0,0 +1,367 @@ +import { describe, test, expect } from 'vitest'; +import { + createTestEnvironment, + TestConfigs, + requiresApiKey, + cleanupTestEnvironment, +} from './test-utils.integration.js'; +import { ErrorScope, ErrorType } from '@core/errors/index.js'; +import { LLMErrorCode } from '../error-codes.js'; + +/** + * Vercel AI SDK LLM Service Integration Tests + * + * These tests verify the Vercel AI SDK service works correctly with real API calls. + * They test multiple providers through the Vercel AI SDK. + */ +describe('Vercel AI SDK LLM Service Integration', () => { + // Test with OpenAI through Vercel AI SDK by default + const defaultProvider = 'openai'; + const skipTests = !requiresApiKey(defaultProvider); + const t = skipTests ? test.skip : test.concurrent; + + // Normal operation tests + t( + 'generate works normally', + async () => { + const env = await createTestEnvironment( + TestConfigs.createVercelConfig(defaultProvider) + ); + try { + const response = await env.agent.run('Hello', undefined, undefined, env.sessionId); + + expect(response).toBeTruthy(); + expect(typeof response).toBe('string'); + expect(response.length).toBeGreaterThan(0); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'multi-turn generate works normally', + async () => { + const env = await createTestEnvironment( + TestConfigs.createVercelConfig(defaultProvider) + ); + try { + const response1 = await env.agent.run( + 'My name is Bob', + undefined, + undefined, + env.sessionId + ); + const response2 = await env.agent.run( + 'What is my name?', + undefined, + undefined, + env.sessionId + ); + + expect(response1).toBeTruthy(); + expect(response2).toBeTruthy(); + expect(typeof response1).toBe('string'); + expect(typeof response2).toBe('string'); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'stream works normally', + async () => { + const env = await createTestEnvironment( + TestConfigs.createVercelConfig(defaultProvider) + ); + try { + const response = await env.agent.run( + 'Hello', + undefined, + undefined, + env.sessionId, + true + ); + + expect(response).toBeTruthy(); + expect(typeof response).toBe('string'); + expect(response.length).toBeGreaterThan(0); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'multi-turn stream works normally', + async () => { + const env = await createTestEnvironment( + TestConfigs.createVercelConfig(defaultProvider) + ); + try { + const response1 = await env.agent.run( + 'I like pizza', + undefined, + undefined, + env.sessionId, + true + ); + const response2 = await env.agent.run( + 'What do I like?', + undefined, + undefined, + env.sessionId, + true + ); + + expect(response1).toBeTruthy(); + expect(response2).toBeTruthy(); + expect(typeof response1).toBe('string'); + expect(typeof response2).toBe('string'); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + t( + 'creating sessions works normally', + async () => { + const env = await createTestEnvironment( + TestConfigs.createVercelConfig(defaultProvider) + ); + try { + const newSession = await env.agent.createSession('test-vercel-session'); + const response = await env.agent.run( + 'Hello in new session', + undefined, + undefined, + newSession.id + ); + + expect(newSession).toBeTruthy(); + expect(newSession.id).toBe('test-vercel-session'); + expect(response).toBeTruthy(); + expect(typeof response).toBe('string'); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + // Multiple Provider Support through Vercel AI SDK + (requiresApiKey('anthropic') ? test.concurrent : test.skip)( + 'anthropic through vercel works normally', + async () => { + const anthropicConfig = TestConfigs.createVercelConfig('anthropic'); + const anthropicEnv = await createTestEnvironment(anthropicConfig); + + try { + const response = await anthropicEnv.agent.run( + 'Hello', + undefined, + undefined, + anthropicEnv.sessionId + ); + + expect(response).toBeTruthy(); + expect(typeof response).toBe('string'); + expect(response.length).toBeGreaterThan(0); + } finally { + await cleanupTestEnvironment(anthropicEnv); + } + }, + 60000 + ); + + (requiresApiKey('google') ? test.concurrent : test.skip)( + 'google through vercel works normally', + async () => { + const googleConfig = TestConfigs.createVercelConfig('google'); + const googleEnv = await createTestEnvironment(googleConfig); + + try { + const response = await googleEnv.agent.run( + 'Hello', + undefined, + undefined, + googleEnv.sessionId + ); + + expect(response).toBeTruthy(); + expect(typeof response).toBe('string'); + expect(response.length).toBeGreaterThan(0); + } finally { + await cleanupTestEnvironment(googleEnv); + } + }, + 60000 + ); + + // Error handling tests + t( + 'errors handled with correct error codes', + async () => { + // Test with unsupported file type to trigger validation error + const invalidFileData = Buffer.from('test data').toString('base64'); + + const env = await createTestEnvironment( + TestConfigs.createVercelConfig(defaultProvider) + ); + try { + await expect( + env.agent.run( + 'Process this file', + undefined, + { + data: invalidFileData, + mimeType: 'application/unknown-type', + filename: 'test.unknown', + }, + env.sessionId + ) + ).rejects.toMatchObject({ + issues: [ + expect.objectContaining({ + code: LLMErrorCode.INPUT_FILE_UNSUPPORTED, + scope: ErrorScope.LLM, + type: ErrorType.USER, + }), + ], + }); + } finally { + await cleanupTestEnvironment(env); + } + }, + 60000 + ); + + // Positive media/file tests (OpenAI via Vercel) + (requiresApiKey('openai') ? test.concurrent : test.skip)( + 'openai via vercel: image input works', + async () => { + const openaiConfig = TestConfigs.createVercelConfig('openai'); + const openaiEnv = await createTestEnvironment(openaiConfig); + let errorSeen = false; + const onError = () => { + errorSeen = true; + }; + try { + openaiEnv.agent.agentEventBus.on('llm:error', onError); + // 1x1 PNG (red pixel) base64 (no data URI), minimal cost + const imgBase64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=='; + + const res = await openaiEnv.agent.run( + 'What is in the image?', + { image: imgBase64, mimeType: 'image/png' }, + undefined, + openaiEnv.sessionId + ); + + expect(typeof res).toBe('string'); + expect(res.length).toBeGreaterThan(0); + expect(errorSeen).toBe(false); + } finally { + // cleanup listener + try { + openaiEnv.agent.agentEventBus.off('llm:error', onError); + } catch (_e) { + void 0; // ignore + } + await cleanupTestEnvironment(openaiEnv); + } + }, + 60000 + ); + + (requiresApiKey('openai') ? test.concurrent : test.skip)( + 'openai via vercel: pdf file input works', + async () => { + const openaiConfig = TestConfigs.createVercelConfig('openai'); + const openaiEnv = await createTestEnvironment(openaiConfig); + let errorSeen = false; + const onError = () => { + errorSeen = true; + }; + try { + openaiEnv.agent.agentEventBus.on('llm:error', onError); + // Valid tiny PDF (Hello World) base64 from OpenAI tests + const pdfBase64 = + 'JVBERi0xLjQKMSAwIG9iago8PAovVHlwZSAvQ2F0YWxvZwovUGFnZXMgMiAwIFIKPj4KZW5kb2JqCjIgMCBvYmoKPDwKL1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDEKPj4KZW5kb2JqCjMgMCBvYmoKPDwKL1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovTWVkaWFCb3ggWzAgMCA2MTIgNzkyXQovQ29udGVudHMgNCAwIFIKPj4KZW5kb2JqCjQgMCBvYmoKPDwKL0xlbmd0aCA0NAo+PgpzdHJlYW0KQlQKL0YxIDEyIFRmCjcyIDcyMCBUZAooSGVsbG8gV29ybGQpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDUKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDEwIDAwMDAwIG4gCjAwMDAwMDAwNzkgMDAwMDAgbiAKMDAwMDAwMDE3MyAwMDAwMCBuIAowMDAwMDAwMzAxIDAwMDAwIG4gCnRyYWlsZXIKPDwKL1NpemUgNQovUm9vdCAxIDAgUgo+PgpzdGFydHhyZWYKMzgwCiUlRU9G'; + + const res = await openaiEnv.agent.run( + 'Summarize this PDF', + undefined, + { data: pdfBase64, mimeType: 'application/pdf', filename: 'test.pdf' }, + openaiEnv.sessionId + ); + + expect(typeof res).toBe('string'); + expect(res.length).toBeGreaterThan(0); + expect(errorSeen).toBe(false); + } finally { + try { + openaiEnv.agent.agentEventBus.off('llm:error', onError); + } catch (_e) { + void 0; // ignore + } + await cleanupTestEnvironment(openaiEnv); + } + }, + 60000 + ); + + (requiresApiKey('openai') ? test.concurrent : test.skip)( + 'openai via vercel: streaming with image works', + async () => { + const openaiConfig = TestConfigs.createVercelConfig('openai'); + const openaiEnv = await createTestEnvironment(openaiConfig); + let errorSeen = false; + const onError = () => { + errorSeen = true; + }; + try { + openaiEnv.agent.agentEventBus.on('llm:error', onError); + const imgBase64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=='; + + const res = await openaiEnv.agent.run( + 'Describe this image in one sentence', + { image: imgBase64, mimeType: 'image/png' }, + undefined, + openaiEnv.sessionId, + true + ); + + expect(typeof res).toBe('string'); + expect(res.length).toBeGreaterThan(0); + expect(errorSeen).toBe(false); + } finally { + try { + openaiEnv.agent.agentEventBus.off('llm:error', onError); + } catch (_e) { + void 0; // ignore + } + await cleanupTestEnvironment(openaiEnv); + } + }, + 60000 + ); + + // Skip test warnings + if (skipTests) { + test('Vercel AI SDK integration tests skipped - no API key', () => { + console.warn( + `Vercel AI SDK integration tests skipped: ${defaultProvider.toUpperCase()}_API_KEY environment variable not found` + ); + expect(true).toBe(true); // Placeholder test + }); + } +}); diff --git a/dexto/packages/core/src/llm/services/vercel.ts b/dexto/packages/core/src/llm/services/vercel.ts new file mode 100644 index 00000000..2735151c --- /dev/null +++ b/dexto/packages/core/src/llm/services/vercel.ts @@ -0,0 +1,295 @@ +import { LanguageModel, type ModelMessage } from 'ai'; +import { ToolManager } from '../../tools/tool-manager.js'; +import { LLMServiceConfig } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { ToolSet } from '../../tools/types.js'; +import { ContextManager } from '../../context/manager.js'; +import { getEffectiveMaxInputTokens, getMaxInputTokensForModel } from '../registry.js'; +import type { ModelLimits } from '../../context/compaction/overflow.js'; +import type { CompactionConfigInput } from '../../context/compaction/schemas.js'; +import { ContentPart } from '../../context/types.js'; +import type { SessionEventBus } from '../../events/index.js'; +import type { IConversationHistoryProvider } from '../../session/history/types.js'; +import type { SystemPromptManager } from '../../systemPrompt/manager.js'; +import { VercelMessageFormatter } from '../formatters/vercel.js'; +import type { ValidatedLLMConfig } from '../schemas.js'; +import { InstrumentClass } from '../../telemetry/decorators.js'; +import { trace, context, propagation } from '@opentelemetry/api'; +import { TurnExecutor } from '../executor/turn-executor.js'; +import { MessageQueueService } from '../../session/message-queue.js'; +import type { ResourceManager } from '../../resources/index.js'; +import { DextoRuntimeError } from '../../errors/DextoRuntimeError.js'; +import { LLMErrorCode } from '../error-codes.js'; +import type { ContentInput } from '../../agent/types.js'; + +/** + * Vercel AI SDK implementation of LLMService + * + * This service delegates actual LLM execution to TurnExecutor, which handles: + * - Tool execution with multimodal support + * - Streaming with llm:chunk events + * - Message persistence via StreamProcessor + * - Reactive compaction on overflow + * - Tool output pruning + * - Message queue injection + * + * @see TurnExecutor for the main execution loop + * @see StreamProcessor for stream event handling + */ +@InstrumentClass({ + prefix: 'llm.vercel', + excludeMethods: ['getModelId', 'getAllTools', 'createTurnExecutor'], +}) +export class VercelLLMService { + private model: LanguageModel; + private config: ValidatedLLMConfig; + private toolManager: ToolManager; + private contextManager: ContextManager; + private sessionEventBus: SessionEventBus; + private readonly sessionId: string; + private logger: IDextoLogger; + private resourceManager: ResourceManager; + private messageQueue: MessageQueueService; + private compactionStrategy: + | import('../../context/compaction/types.js').ICompactionStrategy + | null; + private modelLimits: ModelLimits; + private compactionThresholdPercent: number; + + /** + * Helper to extract model ID from LanguageModel union type (string | LanguageModelV2) + */ + private getModelId(): string { + return typeof this.model === 'string' ? this.model : this.model.modelId; + } + + constructor( + toolManager: ToolManager, + model: LanguageModel, + systemPromptManager: SystemPromptManager, + historyProvider: IConversationHistoryProvider, + sessionEventBus: SessionEventBus, + config: ValidatedLLMConfig, + sessionId: string, + resourceManager: ResourceManager, + logger: IDextoLogger, + compactionStrategy?: import('../../context/compaction/types.js').ICompactionStrategy | null, + compactionConfig?: CompactionConfigInput + ) { + this.logger = logger.createChild(DextoLogComponent.LLM); + this.model = model; + this.config = config; + this.toolManager = toolManager; + this.sessionEventBus = sessionEventBus; + this.sessionId = sessionId; + this.resourceManager = resourceManager; + this.compactionStrategy = compactionStrategy ?? null; + this.compactionThresholdPercent = compactionConfig?.thresholdPercent ?? 0.9; + + // Create session-level message queue for mid-task user messages + this.messageQueue = new MessageQueueService(this.sessionEventBus, this.logger); + + // Create properly-typed ContextManager for Vercel + const formatter = new VercelMessageFormatter(this.logger); + const maxInputTokens = getEffectiveMaxInputTokens(config, this.logger); + + // Set model limits for compaction overflow detection + // - maxContextTokens overrides the model's context window + // - thresholdPercent is applied separately in isOverflow() to trigger before 100% + let effectiveContextWindow = maxInputTokens; + + // Apply maxContextTokens override if set (cap the context window) + if (compactionConfig?.maxContextTokens !== undefined) { + effectiveContextWindow = Math.min(maxInputTokens, compactionConfig.maxContextTokens); + this.logger.debug( + `Compaction: Using maxContextTokens override: ${compactionConfig.maxContextTokens} (model max: ${maxInputTokens})` + ); + } + + // NOTE: thresholdPercent is NOT applied here - it's only applied in isOverflow() + // to trigger compaction early (e.g., at 90% instead of 100%) + + this.modelLimits = { + contextWindow: effectiveContextWindow, + }; + + this.contextManager = new ContextManager( + config, + formatter, + systemPromptManager, + maxInputTokens, + historyProvider, + sessionId, + resourceManager, + this.logger + ); + + this.logger.debug( + `[VercelLLMService] Initialized for model: ${this.getModelId()}, provider: ${this.config.provider}, temperature: ${this.config.temperature}, maxOutputTokens: ${this.config.maxOutputTokens}` + ); + } + + getAllTools(): Promise { + return this.toolManager.getAllTools(); + } + + /** + * Create a TurnExecutor instance for executing the agent loop. + */ + private createTurnExecutor(externalSignal?: AbortSignal): TurnExecutor { + return new TurnExecutor( + this.model, + this.toolManager, + this.contextManager, + this.sessionEventBus, + this.resourceManager, + this.sessionId, + { + maxSteps: this.config.maxIterations, + maxOutputTokens: this.config.maxOutputTokens, + temperature: this.config.temperature, + baseURL: this.config.baseURL, + // Provider-specific options + reasoningEffort: this.config.reasoningEffort, + }, + { provider: this.config.provider, model: this.getModelId() }, + this.logger, + this.messageQueue, + this.modelLimits, + externalSignal, + this.compactionStrategy, + this.compactionThresholdPercent + ); + } + + /** + * Result from streaming a response. + */ + public static StreamResult: { text: string }; + + /** + * Stream a response for the given content. + * Primary method for running conversations with multi-image support. + * + * @param content - String or ContentPart[] (text, images, files) + * @param options - { signal?: AbortSignal } + * @returns Object with text response + */ + async stream( + content: ContentInput, + options?: { signal?: AbortSignal } + ): Promise<{ text: string }> { + // Get active span and context for telemetry + const activeSpan = trace.getActiveSpan(); + const currentContext = context.active(); + + const provider = this.config.provider; + const model = this.getModelId(); + + // Set on active span + if (activeSpan) { + activeSpan.setAttribute('llm.provider', provider); + activeSpan.setAttribute('llm.model', model); + } + + // Add to baggage for child span propagation + const existingBaggage = propagation.getBaggage(currentContext); + const baggageEntries: Record = {}; + + // Preserve existing baggage + if (existingBaggage) { + existingBaggage.getAllEntries().forEach(([key, entry]) => { + baggageEntries[key] = entry; + }); + } + + // Add LLM metadata + baggageEntries['llm.provider'] = { value: provider }; + baggageEntries['llm.model'] = { value: model }; + + const updatedContext = propagation.setBaggage( + currentContext, + propagation.createBaggage(baggageEntries) + ); + + // Execute rest of method in updated context + return await context.with(updatedContext, async () => { + // Normalize content to ContentPart[] for addUserMessage + const parts: ContentPart[] = + typeof content === 'string' ? [{ type: 'text', text: content }] : content; + + // Add user message with all content parts + await this.contextManager.addUserMessage(parts); + + // Create executor (uses session-level messageQueue, pass external abort signal) + const executor = this.createTurnExecutor(options?.signal); + + // Execute with streaming enabled + const contributorContext = { mcpManager: this.toolManager.getMcpManager() }; + const result = await executor.execute(contributorContext, true); + + return { + text: result.text ?? '', + }; + }); + } + + /** + * Get configuration information about the LLM service + * @returns Configuration object with provider and model information + */ + getConfig(): LLMServiceConfig { + const configuredMaxTokens = this.contextManager.getMaxInputTokens(); + let modelMaxInputTokens: number; + + // Fetching max tokens from LLM registry - default to configured max tokens if not found + // Max tokens may not be found if the model is supplied by user + try { + modelMaxInputTokens = getMaxInputTokensForModel( + this.config.provider, + this.getModelId(), + this.logger + ); + } catch (error) { + // if the model is not found in the LLM registry, log and default to configured max tokens + if (error instanceof DextoRuntimeError && error.code === LLMErrorCode.MODEL_UNKNOWN) { + modelMaxInputTokens = configuredMaxTokens; + this.logger.debug( + `Could not find model ${this.getModelId()} in LLM registry to get max tokens. Using configured max tokens: ${configuredMaxTokens}.` + ); + } else { + throw error; + } + } + return { + provider: this.config.provider, + model: this.model, + configuredMaxInputTokens: configuredMaxTokens, + modelMaxInputTokens: modelMaxInputTokens, + }; + } + + /** + * Get the context manager for external access + */ + getContextManager(): ContextManager { + return this.contextManager; + } + + /** + * Get the message queue for external access (e.g., queueing messages while busy) + */ + getMessageQueue(): MessageQueueService { + return this.messageQueue; + } + + /** + * Get the compaction strategy for external access (e.g., session-native compaction) + */ + getCompactionStrategy(): + | import('../../context/compaction/types.js').ICompactionStrategy + | null { + return this.compactionStrategy; + } +} diff --git a/dexto/packages/core/src/llm/types.ts b/dexto/packages/core/src/llm/types.ts new file mode 100644 index 00000000..f66c028b --- /dev/null +++ b/dexto/packages/core/src/llm/types.ts @@ -0,0 +1,52 @@ +// Derive types from registry constants without creating runtime imports. +export const LLM_PROVIDERS = [ + 'openai', + 'openai-compatible', + 'anthropic', + 'google', + 'groq', + 'xai', + 'cohere', + 'openrouter', + 'litellm', + 'glama', + 'vertex', + 'bedrock', + 'local', // Native node-llama-cpp execution (GGUF models) + 'ollama', // Ollama server integration + 'dexto', // Dexto gateway - routes through api.dexto.ai/v1 with billing +] as const; +export type LLMProvider = (typeof LLM_PROVIDERS)[number]; + +export const SUPPORTED_FILE_TYPES = ['pdf', 'image', 'audio'] as const; +export type SupportedFileType = (typeof SUPPORTED_FILE_TYPES)[number]; + +/** + * Context interface for message formatters. + * Provides runtime information for model-aware processing. + */ + +export interface LLMContext { + /** LLM provider name (e.g., 'google', 'openai') */ + provider: LLMProvider; + + /** Specific LLM model name (e.g., 'gemini-2.5-flash', 'gpt-5') */ + model: string; +} + +// TODO: see how we can combine this with LLMContext +export interface LLMUpdateContext { + provider?: LLMProvider; + model?: string; + suggestedAction?: string; +} + +export interface TokenUsage { + inputTokens?: number; + outputTokens?: number; + reasoningTokens?: number; + totalTokens?: number; + // Cache tokens (Vercel AI SDK: cachedInputTokens, providerMetadata.anthropic.cacheCreationInputTokens) + cacheReadTokens?: number; + cacheWriteTokens?: number; +} diff --git a/dexto/packages/core/src/llm/validation.test.ts b/dexto/packages/core/src/llm/validation.test.ts new file mode 100644 index 00000000..d090a79d --- /dev/null +++ b/dexto/packages/core/src/llm/validation.test.ts @@ -0,0 +1,444 @@ +import { describe, test, expect } from 'vitest'; +import { validateInputForLLM } from './validation.js'; +import { LLMErrorCode } from './error-codes.js'; +import { createSilentMockLogger } from '../logger/v2/test-utils.js'; + +describe('validateInputForLLM', () => { + const mockLogger = createSilentMockLogger(); + + describe('text validation', () => { + test('should pass validation for valid text input', () => { + const result = validateInputForLLM( + { text: 'Hello, world!' }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + }); + + test('should pass validation for empty text when no other input provided', () => { + const result = validateInputForLLM( + { text: '' }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + }); + + test('should pass validation for undefined text', () => { + const result = validateInputForLLM( + {}, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + }); + }); + + describe('file validation', () => { + test('should pass validation for supported file type with model that supports PDF', () => { + const result = validateInputForLLM( + { + text: 'Analyze this file', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'application/pdf', + filename: 'document.pdf', + }, + }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + if (result.ok) { + expect(result.data.fileValidation?.isSupported).toBe(true); + } + }); + + test('should pass validation for supported audio file with model that supports audio', () => { + const result = validateInputForLLM( + { + text: 'Analyze this audio', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'audio/mp3', + filename: 'audio.mp3', + }, + }, + { provider: 'openai', model: 'gpt-4o-audio-preview' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + if (result.ok) { + expect(result.data.fileValidation?.isSupported).toBe(true); + } + }); + + test('should fail validation for unsupported file type (model without audio support)', () => { + const result = validateInputForLLM( + { + text: 'Analyze this audio', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'audio/mp3', + filename: 'audio.mp3', + }, + }, + { provider: 'openai', model: 'gpt-5' }, // This model doesn't support audio + mockLogger + ); + + expect(result.ok).toBe(false); + expect(result.issues.filter((i) => i.severity === 'error').length).toBeGreaterThan(0); + expect(result.issues.some((i) => i.code === LLMErrorCode.INPUT_FILE_UNSUPPORTED)).toBe( + true + ); + }); + + test('should fail validation for file not in allowed MIME types', () => { + const result = validateInputForLLM( + { + text: 'Analyze this file', + fileData: { + data: 'base64data', + mimeType: 'application/exe', + filename: 'malware.exe', + }, + }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(false); + expect( + result.issues.filter((i) => i.severity === 'error').map((i) => i.message) + ).toContain('Unsupported file type: application/exe'); + expect(result.issues.some((i) => i.code === LLMErrorCode.INPUT_FILE_UNSUPPORTED)).toBe( + true + ); + }); + + test('should fail validation for oversized file', () => { + const largeBase64 = 'A'.repeat(67108865); // > 64MB + + const result = validateInputForLLM( + { + text: 'Analyze this file', + fileData: { + data: largeBase64, + mimeType: 'application/pdf', + filename: 'large.pdf', + }, + }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(false); + expect( + result.issues.filter((i) => i.severity === 'error').map((i) => i.message) + ).toContain('File size too large (max 64MB)'); + }); + + test('should fail validation for invalid base64 format', () => { + const result = validateInputForLLM( + { + text: 'Analyze this file', + fileData: { + data: 'invalid-base64!@#', + mimeType: 'application/pdf', + filename: 'document.pdf', + }, + }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(false); + expect( + result.issues.filter((i) => i.severity === 'error').map((i) => i.message) + ).toContain('Invalid file data format'); + }); + + test('should fail validation when model is not specified for file', () => { + const result = validateInputForLLM( + { + text: 'Analyze this file', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'application/pdf', + filename: 'document.pdf', + }, + }, + { provider: 'openai' }, // No model specified + mockLogger + ); + + expect(result.ok).toBe(false); + expect( + result.issues.filter((i) => i.severity === 'error').map((i) => i.message) + ).toContain('Model must be specified for file capability validation'); + }); + + test('should fail validation for file without mimeType', () => { + const result = validateInputForLLM( + { + text: 'Analyze this file', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: '', // Empty MIME type should fail + filename: 'document.pdf', + }, + }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(false); + expect(result.issues.some((i) => i.code === LLMErrorCode.INPUT_FILE_UNSUPPORTED)).toBe( + true + ); + expect( + result.issues.filter((i) => i.severity === 'error').map((i) => i.message) + ).toContain('Unsupported file type: '); + }); + + test('should pass validation for parameterized MIME types', () => { + // Test audio/webm;codecs=opus (the original issue) + const webmResult = validateInputForLLM( + { + text: 'Analyze this audio', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 + mimeType: 'audio/webm;codecs=opus', + filename: 'recording.webm', + }, + }, + { provider: 'openai', model: 'gpt-4o-audio-preview' }, + mockLogger + ); + + expect(webmResult.ok).toBe(true); + + // Test application/pdf;version=1.4 + const pdfResult = validateInputForLLM( + { + text: 'Analyze this document', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 + mimeType: 'application/pdf;version=1.4', + filename: 'document.pdf', + }, + }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(pdfResult.ok).toBe(true); + }); + }); + + describe('image validation', () => { + test('should pass validation for image input', () => { + const result = validateInputForLLM( + { + text: 'Analyze this image', + imageData: { + image: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD', + mimeType: 'image/jpeg', + }, + }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + if (result.ok) { + expect(result.data.imageValidation).toBeDefined(); + expect(result.data.imageValidation?.isSupported).toBe(true); + } + }); + + test('should pass validation for image without mimeType', () => { + const result = validateInputForLLM( + { + text: 'Analyze this image', + imageData: { + image: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD', + }, + }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + }); + }); + + describe('combined input validation', () => { + test('should pass validation for text + image + file', () => { + const result = validateInputForLLM( + { + text: 'Analyze this content', + imageData: { + image: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD', + mimeType: 'image/jpeg', + }, + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'application/pdf', + filename: 'document.pdf', + }, + }, + { provider: 'openai', model: 'gpt-5' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + }); + + test('should fail validation when file input is invalid for model', () => { + const result = validateInputForLLM( + { + text: 'Analyze this content', + imageData: { + image: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD', + mimeType: 'image/jpeg', + }, + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'audio/mp3', + filename: 'audio.mp3', + }, + }, + { provider: 'openai', model: 'gpt-5' }, // This model doesn't support audio + mockLogger + ); + + expect(result.ok).toBe(false); + expect(result.issues.filter((i) => i.severity === 'error').length).toBeGreaterThan(0); + expect(result.issues.some((i) => i.code === LLMErrorCode.INPUT_FILE_UNSUPPORTED)).toBe( + true + ); + }); + }); + + describe('error handling', () => { + test('should handle unknown provider gracefully', () => { + const result = validateInputForLLM( + { + text: 'Analyze this file', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'application/pdf', + filename: 'document.pdf', + }, + }, + { provider: 'openai', model: 'unknown-model' }, + mockLogger + ); + + expect(result.ok).toBe(false); + expect(result.issues.filter((i) => i.severity === 'error').length).toBeGreaterThan(0); + }); + + test('should fail validation for unknown model', () => { + const result = validateInputForLLM( + { + text: 'Analyze this file', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'application/pdf', + filename: 'document.pdf', + }, + }, + { provider: 'openai', model: 'unknown-model' }, + mockLogger + ); + + // Fixed behavior: unknown models should fail validation + expect(result.ok).toBe(false); + expect(result.issues.some((i) => i.code === LLMErrorCode.INPUT_FILE_UNSUPPORTED)).toBe( + true + ); + expect(result.issues.filter((i) => i.severity === 'error').length).toBeGreaterThan(0); + }); + }); + + describe('different providers and models', () => { + test('should work with Anthropic provider and PDF files', () => { + const result = validateInputForLLM( + { + text: 'Hello Claude', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'application/pdf', + filename: 'document.pdf', + }, + }, + { provider: 'anthropic', model: 'claude-4-sonnet-20250514' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + if (result.ok) { + expect(result.data.fileValidation?.isSupported).toBe(true); + } + }); + + test('should work with Google provider and PDF files', () => { + const result = validateInputForLLM( + { + text: 'Hello Gemini', + fileData: { + data: 'SGVsbG8gV29ybGQ=', // Valid base64 for "Hello World" + mimeType: 'application/pdf', + filename: 'document.pdf', + }, + }, + { provider: 'google', model: 'gemini-2.0-flash' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + if (result.ok) { + expect(result.data.fileValidation?.isSupported).toBe(true); + } + }); + + test('should work with image validation (always supported currently)', () => { + const result = validateInputForLLM( + { + text: 'Hello Gemini', + imageData: { + image: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD', + mimeType: 'image/jpeg', + }, + }, + { provider: 'google', model: 'gemini-2.0-flash' }, + mockLogger + ); + + expect(result.ok).toBe(true); + expect(result.issues.filter((i) => i.severity === 'error')).toHaveLength(0); + if (result.ok) { + expect(result.data.imageValidation?.isSupported).toBe(true); + } + }); + }); +}); diff --git a/dexto/packages/core/src/llm/validation.ts b/dexto/packages/core/src/llm/validation.ts new file mode 100644 index 00000000..38d888b9 --- /dev/null +++ b/dexto/packages/core/src/llm/validation.ts @@ -0,0 +1,252 @@ +import { validateModelFileSupport, getAllowedMimeTypes } from './registry.js'; +import type { LLMProvider } from './types.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import type { ImageData, FileData } from '../context/types.js'; +import { Result, ok, fail } from '../utils/result.js'; +import { Issue, ErrorScope, ErrorType } from '@core/errors/types.js'; +import { LLMErrorCode } from './error-codes.js'; + +// TOOD: Refactor/simplify this file +export interface ValidationLLMConfig { + provider: LLMProvider; + model?: string; +} + +export interface ValidationContext { + provider?: string; + model?: string | undefined; + fileSize?: number; + maxFileSize?: number; + filename?: string | undefined; + mimeType?: string; + fileType?: string | undefined; + suggestedAction?: string; +} + +export interface ValidationData { + fileValidation?: { + isSupported: boolean; + fileType?: string; + error?: string; + }; + imageValidation?: { + isSupported: boolean; + error?: string; + }; +} + +/** + * Input interface for comprehensive validation + */ +export interface ValidationInput { + text?: string; + imageData?: ImageData | undefined; + fileData?: FileData | undefined; +} + +// Security constants +const MAX_FILE_SIZE = 67108864; // 64MB in base64 format +const MAX_IMAGE_SIZE = 20971520; // 20MB + +/** + * Validates all inputs (text, image, file) against LLM capabilities and security requirements. + * This is the single entry point for all input validation using pure Result pattern. + * @param input The input data to validate (text, image, file) + * @param config The LLM configuration (provider and model) + * @param logger The logger instance for logging + * @returns Comprehensive validation result + */ +export function validateInputForLLM( + input: ValidationInput, + config: ValidationLLMConfig, + logger: IDextoLogger +): Result { + const issues: Issue[] = []; + const validationData: ValidationData = {}; + + try { + const context: ValidationContext = { + provider: config.provider, + model: config.model, + }; + + // Validate file data if provided + if (input.fileData) { + const fileValidation = validateFileInput(input.fileData, config, logger); + validationData.fileValidation = fileValidation; + + if (!fileValidation.isSupported) { + issues.push({ + code: LLMErrorCode.INPUT_FILE_UNSUPPORTED, + message: fileValidation.error || 'File type not supported by current LLM', + scope: ErrorScope.LLM, + type: ErrorType.USER, + severity: 'error', + context: { + ...context, + fileType: fileValidation.fileType, + mimeType: input.fileData.mimeType, + filename: input.fileData.filename, + suggestedAction: 'Use a supported file type or different model', + }, + }); + } + } + + // Validate image data if provided + if (input.imageData) { + const imageValidation = validateImageInput(input.imageData, config, logger); + validationData.imageValidation = imageValidation; + + if (!imageValidation.isSupported) { + issues.push({ + code: LLMErrorCode.INPUT_IMAGE_UNSUPPORTED, + message: imageValidation.error || 'Image format not supported by current LLM', + scope: ErrorScope.LLM, + type: ErrorType.USER, + severity: 'error', + context: { + ...context, + suggestedAction: 'Use a supported image format or different model', + }, + }); + } + } + + // Basic text validation (currently permissive - empty text is allowed) + // TODO: Could be extended with more sophisticated text validation rules + // Note: Empty text is currently allowed as it may be valid in combination with images/files + + return issues.length === 0 ? ok(validationData, issues) : fail(issues); + } catch (error) { + logger.error(`Error during input validation: ${error}`); + return fail([ + { + code: LLMErrorCode.REQUEST_INVALID_SCHEMA, + message: 'Failed to validate input', + scope: ErrorScope.LLM, + type: ErrorType.SYSTEM, + severity: 'error', + context: { + provider: config.provider, + model: config.model, + suggestedAction: 'Check input format and try again', + }, + }, + ]); + } +} + +/** + * Validates file input including security checks and model capability validation. + * @param fileData The file data to validate + * @param config The LLM configuration + * @param logger The logger instance for logging + * @returns File validation result + */ +function validateFileInput( + fileData: FileData, + config: ValidationLLMConfig, + logger: IDextoLogger +): NonNullable { + logger.info(`Validating file input: ${fileData.mimeType}`); + + // Security validation: file size check (max 64MB for base64) + if (typeof fileData.data === 'string' && fileData.data.length > MAX_FILE_SIZE) { + return { + isSupported: false, + error: 'File size too large (max 64MB)', + }; + } + + // Security validation: MIME type allowlist + // Extract base MIME type by removing parameters (e.g., "audio/webm;codecs=opus" -> "audio/webm") + const baseMimeType = + fileData.mimeType.toLowerCase().split(';')[0]?.trim() || fileData.mimeType.toLowerCase(); + const allowedMimeTypes = getAllowedMimeTypes(); + if (!allowedMimeTypes.includes(baseMimeType)) { + return { + isSupported: false, + error: `Unsupported file type: ${fileData.mimeType}`, + }; + } + + // Security validation: base64 format check + if (typeof fileData.data === 'string') { + // Enhanced base64 validation: ensures proper length and padding + const base64Regex = /^[A-Za-z0-9+/]+={0,2}$/; + if (!base64Regex.test(fileData.data) || fileData.data.length % 4 !== 0) { + return { + isSupported: false, + error: 'Invalid file data format', + }; + } + } + + // Model-specific capability validation (only if model is specified) + if (config.model) { + return validateModelFileSupport(config.provider, config.model, fileData.mimeType); + } + + // If no model specified, we cannot validate capabilities + return { + isSupported: false, + error: 'Model must be specified for file capability validation', + }; +} + +/** + * Validates image input with size and format checks. + * @param imageData The image data to validate + * @param config The LLM configuration + * @param logger The logger instance for logging + * @returns Image validation result + */ +function validateImageInput( + imageData: ImageData, + config: ValidationLLMConfig, + logger: IDextoLogger +): NonNullable { + logger.info(`Validating image input: ${imageData.mimeType}`); + + // Check image size if available + if (typeof imageData.image === 'string' && imageData.image.length > MAX_IMAGE_SIZE) { + return { + isSupported: false, + error: `Image size too large (max ${MAX_IMAGE_SIZE / 1048576}MB)`, + }; + } + + // Resolve image MIME type from either explicit field or data URL + // Example: callers may only provide a data URL like + // image: "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD..." + // without setting imageData.mimeType. In that case, parse the MIME from the prefix. + let resolvedMime: string | undefined = imageData.mimeType?.toLowerCase(); + if (!resolvedMime && typeof imageData.image === 'string') { + const dataUrlMatch = /^data:([^;]+);base64,/i.exec(imageData.image); + if (dataUrlMatch && dataUrlMatch[1]) { + resolvedMime = dataUrlMatch[1].toLowerCase(); + } + } + + if (!resolvedMime) { + return { isSupported: false, error: 'Missing image MIME type' }; + } + + if (!config.model) { + return { + isSupported: false, + error: 'Model must be specified for image capability validation', + }; + } + + // Extract base MIME type by removing parameters (e.g., "image/jpeg;quality=85" -> "image/jpeg") + const baseMimeType = resolvedMime.split(';')[0]?.trim() || resolvedMime; + + // Delegate both allowed-MIME and model capability to registry helper + const res = validateModelFileSupport(config.provider, config.model, baseMimeType); + return { + isSupported: res.isSupported, + ...(res.error ? { error: res.error } : {}), + }; +} diff --git a/dexto/packages/core/src/logger/browser.ts b/dexto/packages/core/src/logger/browser.ts new file mode 100644 index 00000000..6483ac5f --- /dev/null +++ b/dexto/packages/core/src/logger/browser.ts @@ -0,0 +1,72 @@ +// Browser-safe console-backed logger implementation. +// Matches the public surface used by the app/CLI but avoids fs/path/winston. + +export interface LoggerOptions { + level?: string; + silent?: boolean; + logToConsole?: boolean; +} + +export class Logger { + private level: string; + private isSilent: boolean; + + constructor(options: LoggerOptions = {}) { + this.level = (options.level || 'info').toLowerCase(); + this.isSilent = options.silent ?? false; + } + + private out(fn: (...args: any[]) => void, args: any[]) { + if (!this.isSilent && typeof console !== 'undefined') fn(...args); + } + + error(message: any, meta?: any) { + this.out(console.error, [message, meta]); + } + warn(message: any, meta?: any) { + this.out(console.warn, [message, meta]); + } + info(message: any, meta?: any) { + this.out(console.info, [message, meta]); + } + http(message: any, meta?: any) { + this.out(console.info, [message, meta]); + } + verbose(message: any, meta?: any) { + this.out(console.debug, [message, meta]); + } + debug(message: any, meta?: any) { + this.out(console.debug, [message, meta]); + } + silly(message: any, meta?: any) { + this.out(console.debug, [message, meta]); + } + + displayAIResponse(response: any) { + this.out(console.log, [response]); + } + toolCall(toolName: string, args: any) { + this.out(console.log, ['Tool Call', toolName, args]); + } + toolResult(result: any) { + this.out(console.log, ['Tool Result', result]); + } + displayStartupInfo(info: Record) { + this.out(console.log, ['Startup', info]); + } + displayError(message: string, error?: Error) { + this.out(console.error, [message, error]); + } + + setLevel(level: string) { + this.level = level.toLowerCase(); + } + getLevel(): string { + return this.level; + } + getLogFilePath(): string | null { + return null; + } +} + +export const logger = new Logger(); diff --git a/dexto/packages/core/src/logger/factory.ts b/dexto/packages/core/src/logger/factory.ts new file mode 100644 index 00000000..e9efd7ce --- /dev/null +++ b/dexto/packages/core/src/logger/factory.ts @@ -0,0 +1,74 @@ +/** + * Logger Factory + * + * Creates logger instances from agent configuration. + * Bridges the gap between agent config (LoggerConfig) and the DextoLogger implementation. + */ + +import type { LoggerConfig } from './v2/schemas.js'; +import type { IDextoLogger, LogLevel } from './v2/types.js'; +import { DextoLogComponent } from './v2/types.js'; +import { DextoLogger } from './v2/dexto-logger.js'; +import { createTransport } from './v2/transport-factory.js'; + +export interface CreateLoggerOptions { + /** Logger configuration from agent config */ + config: LoggerConfig; + /** Agent ID for multi-agent isolation */ + agentId: string; + /** Component identifier (defaults to AGENT) */ + component?: DextoLogComponent; +} + +/** + * Helper to get effective log level from environment or config + * DEXTO_LOG_LEVEL environment variable takes precedence over config + */ +function getEffectiveLogLevel(configLevel: LogLevel): LogLevel { + const envLevel = process.env.DEXTO_LOG_LEVEL; + if (envLevel) { + const validLevels: LogLevel[] = ['debug', 'info', 'warn', 'error', 'silly']; + const normalizedLevel = envLevel.toLowerCase() as LogLevel; + if (validLevels.includes(normalizedLevel)) { + return normalizedLevel; + } + } + return configLevel; +} + +/** + * Create a logger instance from agent configuration + * + * @param options Logger creation options + * @returns Configured logger instance + * + * @example + * ```typescript + * const logger = createLogger({ + * config: validatedConfig.logger, + * agentId: 'my-agent', + * component: DextoLogComponent.AGENT + * }); + * + * logger.info('Agent started'); + * ``` + */ +export function createLogger(options: CreateLoggerOptions): IDextoLogger { + const { config, agentId, component = DextoLogComponent.AGENT } = options; + + // Override log level with DEXTO_LOG_LEVEL environment variable if present + const effectiveLevel = getEffectiveLogLevel(config.level); + + // Create transport instances from configs + const transports = config.transports.map((transportConfig) => { + return createTransport(transportConfig); + }); + + // Create and return logger instance + return new DextoLogger({ + level: effectiveLevel, + component, + agentId, + transports, + }); +} diff --git a/dexto/packages/core/src/logger/index.ts b/dexto/packages/core/src/logger/index.ts new file mode 100644 index 00000000..49f39923 --- /dev/null +++ b/dexto/packages/core/src/logger/index.ts @@ -0,0 +1,24 @@ +// Logger factory for dependency injection +export { createLogger } from './factory.js'; +export type { CreateLoggerOptions } from './factory.js'; + +// Multi-transport logger - v2 +export type { LogLevel, LogEntry, IDextoLogger, ILoggerTransport } from './v2/types.js'; +export { DextoLogComponent } from './v2/types.js'; +export { LoggerTransportSchema, LoggerConfigSchema } from './v2/schemas.js'; +export type { LoggerTransportConfig, LoggerConfig } from './v2/schemas.js'; +export type { DextoLoggerConfig } from './v2/dexto-logger.js'; +export { DextoLogger } from './v2/dexto-logger.js'; +export { createTransport, createTransports } from './v2/transport-factory.js'; +export type { ConsoleTransportConfig } from './v2/transports/console-transport.js'; +export { ConsoleTransport } from './v2/transports/console-transport.js'; +export type { FileTransportConfig } from './v2/transports/file-transport.js'; +export { FileTransport } from './v2/transports/file-transport.js'; + +// Error handling +export { LoggerError } from './v2/errors.js'; +export { LoggerErrorCode } from './v2/error-codes.js'; + +// Legacy logger (to be removed) +export type { LoggerOptions } from './logger.js'; +export { Logger, logger } from './logger.js'; diff --git a/dexto/packages/core/src/logger/logger.test.ts b/dexto/packages/core/src/logger/logger.test.ts new file mode 100644 index 00000000..41160897 --- /dev/null +++ b/dexto/packages/core/src/logger/logger.test.ts @@ -0,0 +1,130 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Logger } from '../logger/index.js'; +import * as fs from 'fs'; +import * as path from 'path'; +import { tmpdir } from 'os'; + +// Ensure console.log is mocked and environment is reset between tests +beforeEach(() => { + delete process.env.DEXTO_LOG_LEVEL; + delete process.env.DEXTO_LOG_TO_CONSOLE; + delete process.env.DEBUG; + vi.restoreAllMocks(); +}); + +afterEach(() => { + // Clean up any temporary log files + vi.restoreAllMocks(); +}); + +describe('Logger utilities', () => { + let spyLog: ReturnType; + let _spyStdErrWrite: ReturnType; + let tempDir: string; + + beforeEach(() => { + spyLog = vi.spyOn(console, 'log').mockImplementation(() => {}); + _spyStdErrWrite = vi.spyOn(process.stderr, 'write').mockImplementation(() => true) as any; + tempDir = fs.mkdtempSync(path.join(tmpdir(), 'dexto-logger-test-')); + }); + + afterEach(() => { + // Clean up temp directory + if (fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('getDefaultLogLevel falls back to "info"', () => { + const customLogPath = path.join(tempDir, 'test.log'); + const l = new Logger({ customLogPath }); + expect(l.getLevel()).toBe('info'); + }); + + it('respects DEXTO_LOG_LEVEL env var', () => { + process.env.DEXTO_LOG_LEVEL = 'debug'; + const customLogPath = path.join(tempDir, 'test.log'); + const l = new Logger({ customLogPath }); + expect(l.getLevel()).toBe('debug'); + }); + + it('setLevel updates level and rejects invalid levels', () => { + const customLogPath = path.join(tempDir, 'test.log'); + const l = new Logger({ level: 'info', customLogPath }); + l.setLevel('warn'); + expect(l.getLevel()).toBe('warn'); + // Invalid level should not change current level + l.setLevel('invalid'); + expect(l.getLevel()).toBe('warn'); + }); + + it('uses file logging by default', () => { + const customLogPath = path.join(tempDir, 'test.log'); + const l = new Logger({ customLogPath }); + expect(l.getLogFilePath()).toBe(customLogPath); + }); + + it('enables console logging when DEXTO_LOG_TO_CONSOLE=true', () => { + process.env.DEXTO_LOG_TO_CONSOLE = 'true'; + const customLogPath = path.join(tempDir, 'test.log'); + const l = new Logger({ customLogPath }); + + // Logger should still have file path set + expect(l.getLogFilePath()).toBe(customLogPath); + + // toolCall should log to console when console logging is enabled + l.toolCall('testTool', { foo: 'bar' }); + expect(spyLog).toHaveBeenCalledWith(expect.stringContaining('Tool Call')); + }); + + it('display methods always use console.log (UI display)', () => { + const customLogPath = path.join(tempDir, 'test.log'); + const l = new Logger({ customLogPath }); + + l.toolCall('testTool', { foo: 'bar' }); + // Display methods (toolCall, displayAIResponse, toolResult) always use console.log for UI + expect(spyLog).toHaveBeenCalledWith(expect.stringContaining('Tool Call')); + }); + + it('creates log file directory if it does not exist', async () => { + const logDir = path.join(tempDir, 'nested', 'log', 'dir'); + const customLogPath = path.join(logDir, 'test.log'); + + const l = new Logger({ customLogPath }); + + // Give async initialization time to complete + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(fs.existsSync(logDir)).toBe(true); + expect(l.getLogFilePath()).toBe(customLogPath); + }); + + it('actually writes logs to the file', async () => { + const customLogPath = path.join(tempDir, 'test.log'); + const l = new Logger({ customLogPath }); + + // Write a test log + l.info('Test log message'); + + // Give time for file write + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Check that file exists and contains the log + expect(fs.existsSync(customLogPath)).toBe(true); + const logContent = fs.readFileSync(customLogPath, 'utf8'); + expect(logContent).toContain('Test log message'); + expect(logContent).toContain('"level":"info"'); + }); + + it('uses synchronous path resolution for project detection', () => { + const customLogPath = path.join(tempDir, 'test.log'); + const l = new Logger({ customLogPath }); + + // Logger should initialize synchronously without errors + expect(l.getLogFilePath()).toBe(customLogPath); + + // Should be able to log immediately + l.info('Immediate log'); + expect(l.getLevel()).toBe('info'); + }); +}); diff --git a/dexto/packages/core/src/logger/logger.ts b/dexto/packages/core/src/logger/logger.ts new file mode 100644 index 00000000..3ac9169a --- /dev/null +++ b/dexto/packages/core/src/logger/logger.ts @@ -0,0 +1,552 @@ +/** + * TODO(logger-browser-safety): Make logging environment-safe without breaking consumers + * + * Problem + * - The core logger is Node-only (winston + fs/path + file IO). If any module that depends on this + * logger is included in a browser bundle (directly or via an overly broad root export), Web UI builds + * can fail. Today our Web UI only imports types and a small util, so bundlers tree‑shake away the + * Node logger, but this is fragile if future UI imports include runtime modules. + * + * Constraints/Goals + * - Keep file-based logging as the default for Node (excellent DX/UX). + * - Keep logs visible in the browser (console-based), never a no‑op. + * - Avoid touching "a lot of files" or changing call sites for consumers. + * - Keep `@dexto/core` ergonomic; browser consumers should not have to learn a separate API. + * + * Plan (incremental) + * 1) Introduce an environment-aware logging boundary: + * - Create a browser-safe logger implementation (console-based) that matches the logger API. + * - Expose logging via a subpath `@dexto/core/logger` and add conditional exports so that: + * - Browser resolves to the console logger + * - Node resolves to this file-based logger (current Winston version) + * - Alternative: use the `browser` field in package.json to alias the Node logger file to a + * browser-safe implementation at bundle time. Subpath conditional exports are preferred for clarity. + * + * 2) Keep the root export of `@dexto/core` browser-safe by default: + * - Continue to expose types and UI-safe helpers from the root. + * - Keep Node-only modules (storage/config/agent-registry/etc.) on explicit subpaths so browser + * builds do not pull them accidentally. + * + * 3) Optional API ergonomics: + * - Provide a top-level `Dexto` object (runtime orchestrator) that accepts a `logger` in its config + * and propagates it to sub-services (MCP manager, storage, etc.). This allows injection without + * consumers needing to import Node-only loggers. + * + * 4) UI safety now (tactical): + * - Ensure the Web UI imports types with `import type { ... } from '@dexto/core'` and uses API calls + * for runtime. This avoids bundling Node code until the logger split is implemented. + * + * Verification & Guardrails + * - Add a CI check that building a minimal Next/Vite app that imports root `@dexto/core` types succeeds. + * - Mark side-effect status appropriately and keep top-level Node-only side effects out of root paths. + */ + +/** + * TODO (Telemetry): Integrate OpenTelemetry structured logs with trace correlation + * + * Future Enhancement: + * - Replace or enhance Winston logger with OpenTelemetry Logs API + * - Automatically inject trace_id and span_id into all log messages + * - Enable correlation between traces and logs in observability backends + * - Support OpenTelemetry log exporters (OTLP, console, etc.) + * + * Benefits: + * - Unified observability: traces, metrics, and logs in one system + * - Click on trace in Jaeger → see correlated logs + * - Click on log → see full trace context + * + * Implementation: + * - Use @opentelemetry/api-logs package + * - Create OTel-aware logger that wraps or replaces Winston + * - Maintain existing logger API for backward compatibility + * See feature-plans/telemetry.md for details + */ +import * as winston from 'winston'; +import chalk from 'chalk'; +import boxen from 'boxen'; +import * as fs from 'fs'; +import * as path from 'path'; + +// Winston logger configuration +const logLevels = { + error: 0, + warn: 1, + info: 2, + http: 3, + verbose: 4, + debug: 5, + silly: 6, +}; + +// Available chalk colors for message formatting +type ChalkColor = + | 'black' + | 'red' + | 'green' + | 'yellow' + | 'blue' + | 'magenta' + | 'cyan' + | 'white' + | 'gray' + | 'grey' + | 'blackBright' + | 'redBright' + | 'greenBright' + | 'yellowBright' + | 'blueBright' + | 'magentaBright' + | 'cyanBright' + | 'whiteBright'; + +// Custom format for console output +const consoleFormat = winston.format.printf(({ level, message, timestamp, color }) => { + const levelColorMap: Record string> = { + error: chalk.red, + warn: chalk.yellow, + info: chalk.blue, + http: chalk.cyan, + verbose: chalk.magenta, + debug: chalk.gray, + silly: chalk.gray.dim, + }; + + const colorize = levelColorMap[level] || chalk.white; + + // Apply color to message if specified + const formattedMessage = + color && typeof color === 'string' && chalk[color as ChalkColor] + ? chalk[color as ChalkColor](message) + : message; + + return `${chalk.dim(timestamp)} ${colorize(level.toUpperCase())}: ${formattedMessage}`; +}); + +/** + * Logic to redact sensitive information from logs. + * This is useful for preventing sensitive information from being logged in production. + * On by default, we can set the environment variable REDACT_SECRETS to false to disable this behavior. + */ +const SHOULD_REDACT = process.env.REDACT_SECRETS !== 'false'; +const SENSITIVE_KEYS = ['apiKey', 'password', 'secret', 'token']; +const MASK_REGEX = new RegExp( + `(${SENSITIVE_KEYS.join('|')})(["']?\\s*[:=]\\s*)(["'])?.*?\\3`, + 'gi' +); +const maskFormat = winston.format((info) => { + if (SHOULD_REDACT && typeof info.message === 'string') { + info.message = info.message.replace(MASK_REGEX, '$1$2$3[REDACTED]$3'); + } + return info; +}); + +export interface LoggerOptions { + level?: string; + silent?: boolean; + logToConsole?: boolean; + customLogPath?: string; +} + +// Helper to get default log level from environment or fallback to 'info' +const getDefaultLogLevel = (): string => { + const envLevel = process.env.DEXTO_LOG_LEVEL; + if (envLevel && Object.keys(logLevels).includes(envLevel.toLowerCase())) { + return envLevel.toLowerCase(); + } + return 'info'; +}; + +export class Logger { + private logger: winston.Logger; + private isSilent: boolean = false; + private logFilePath: string | null = null; + private logToConsole: boolean = false; + + constructor(options: LoggerOptions = {}) { + this.isSilent = options.silent || false; + + // Initialize transports synchronously + this.initializeTransports(options); + + // Create logger with transports + this.logger = winston.createLogger({ + levels: logLevels, + level: options.level || getDefaultLogLevel(), + silent: options.silent || false, + format: winston.format.combine( + winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + maskFormat(), + winston.format.errors({ stack: true }), + winston.format.splat(), + winston.format.json() + ), + transports: this.createTransports(options), + }); + } + + private initializeTransports(options: LoggerOptions) { + // Check if console logging should be enabled for Winston logs + // Default to false (file-only logging), enable only when explicitly requested + const logToConsole = options.logToConsole ?? process.env.DEXTO_LOG_TO_CONSOLE === 'true'; + this.logToConsole = logToConsole; + + // Set up file logging path only if explicitly provided + // File logging is optional - CLI enrichment layer provides paths for v2 logger + if (options.customLogPath) { + this.logFilePath = options.customLogPath; + } else { + this.logFilePath = null; + } + } + + private createTransports(_options: LoggerOptions): winston.transport[] { + const transports: winston.transport[] = []; + + // Add console transport if enabled + if (this.logToConsole) { + transports.push( + new winston.transports.Console({ + format: winston.format.combine( + winston.format.timestamp({ format: 'HH:mm:ss' }), + maskFormat(), + consoleFormat + ), + }) + ); + } + + // Add file transport only if path is provided + if (this.logFilePath) { + try { + // Ensure log directory exists + const logDir = path.dirname(this.logFilePath); + fs.mkdirSync(logDir, { recursive: true }); + + transports.push( + new winston.transports.File({ + filename: this.logFilePath, + format: winston.format.combine( + winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + maskFormat(), + winston.format.errors({ stack: true }), + winston.format.json() + ), + // Add daily rotation + maxsize: 10 * 1024 * 1024, // 10MB + maxFiles: 7, // Keep 7 files + tailable: true, + }) + ); + } catch (error) { + // If file logging fails, fall back to console + console.error( + `Failed to initialize file logging: ${error}. Falling back to console.` + ); + if (!this.logToConsole) { + this.logToConsole = true; + transports.push( + new winston.transports.Console({ + format: winston.format.combine( + winston.format.timestamp({ format: 'HH:mm:ss' }), + maskFormat(), + consoleFormat + ), + }) + ); + } + } + } + + // Ensure at least one transport exists (console fallback) + if (transports.length === 0) { + this.logToConsole = true; + transports.push( + new winston.transports.Console({ + format: winston.format.combine( + winston.format.timestamp({ format: 'HH:mm:ss' }), + maskFormat(), + consoleFormat + ), + }) + ); + } + + return transports; + } + + // General logging methods with optional color parameter + error(message: string, meta?: any, color?: ChalkColor) { + // Handle Error objects specially to preserve stack traces + if (meta instanceof Error) { + this.logger.error(message, meta); + } else { + this.logger.error(message, { ...meta, color }); + } + } + + warn(message: string, meta?: any, color?: ChalkColor) { + if (meta instanceof Error) { + this.logger.warn(message, meta); + } else { + this.logger.warn(message, { ...meta, color }); + } + } + + info(message: string, meta?: any, color?: ChalkColor) { + if (meta instanceof Error) { + this.logger.info(message, meta); + } else { + this.logger.info(message, { ...meta, color }); + } + } + + http(message: string, meta?: any, color?: ChalkColor) { + if (meta instanceof Error) { + this.logger.http(message, meta); + } else { + this.logger.http(message, { ...meta, color }); + } + } + + verbose(message: string, meta?: any, color?: ChalkColor) { + if (meta instanceof Error) { + this.logger.verbose(message, meta); + } else { + this.logger.verbose(message, { ...meta, color }); + } + } + + debug(message: string | object, meta?: any, color?: ChalkColor) { + const formattedMessage = + typeof message === 'string' ? message : JSON.stringify(message, null, 2); + if (meta instanceof Error) { + this.logger.debug(formattedMessage, meta); + } else { + this.logger.debug(formattedMessage, { ...meta, color }); + } + } + + silly(message: string, meta?: any, color?: ChalkColor) { + if (meta instanceof Error) { + this.logger.silly(message, meta); + } else { + this.logger.silly(message, { ...meta, color }); + } + } + + // Display AI response in a box + displayAIResponse(response: any) { + if (this.isSilent) return; + + if (response.content) { + console.log( + boxen(chalk.white(response.content), { + padding: 1, + borderColor: 'yellow', + title: '🤖 AI Response', + titleAlignment: 'center', + }) + ); + } else { + console.log(chalk.yellow('AI is thinking...')); + } + } + + // Tool-related logging + toolCall(toolName: string, args: any) { + if (this.isSilent) return; + console.log( + boxen( + `${chalk.cyan('Tool Call')}: ${chalk.yellow(toolName)}\n${chalk.dim('Arguments')}:\n${chalk.white(JSON.stringify(args, null, 2))}`, + { padding: 1, borderColor: 'blue', title: '🔧 Tool Call', titleAlignment: 'center' } + ) + ); + } + + toolResult(result: any) { + if (this.isSilent) return; + let displayText = ''; + let isError = false; + let borderColor = 'green'; + let title = '✅ Tool Result'; + + // Check if result indicates an error + if (result?.error || result?.isError) { + isError = true; + borderColor = 'yellow'; + title = '⚠️ Tool Result (Error)'; + } + + // Handle different result formats + if (result?.content && Array.isArray(result.content)) { + // Standard MCP format with content array + result.content.forEach((item: any) => { + if (item.type === 'text') { + displayText += item.text; + } else if (item.type === 'image' && item.url) { + displayText += `[Image URL: ${item.url}]`; + } else if (item.type === 'image') { + displayText += `[Image Data: ${item.mimeType || 'unknown type'}]`; + } else if (item.type === 'markdown') { + displayText += item.markdown; + } else { + displayText += `[Unsupported content type: ${item.type}]`; + } + displayText += '\n'; + }); + } else if (result?.message) { + // Error message format + displayText = result.message; + isError = true; + borderColor = 'red'; + title = '❌ Tool Error'; + } else if (typeof result === 'string') { + // Plain string response - truncate if too long + if (result.length > 1000) { + displayText = `${result.slice(0, 500)}... [${result.length - 500} chars omitted]`; + } else { + displayText = result; + } + } else { + // Fallback for any other format - truncate if too long + try { + const resultStr = JSON.stringify(result, null, 2); + if (resultStr.length > 2000) { + displayText = `${resultStr.slice(0, 1000)}... [${resultStr.length - 1000} chars omitted]`; + } else { + displayText = resultStr; + } + } catch { + displayText = `[Unparseable result: ${typeof result}]`; + } + } + + // Format empty results + if (!displayText || displayText.trim() === '') { + displayText = '[Empty result]'; + } + + // Apply color based on error status + const textColor = isError ? chalk.yellow : chalk.green; + console.log( + boxen(textColor(displayText), { + padding: 1, + borderColor, + title, + titleAlignment: 'center', + }) + ); + } + + // Configuration + setLevel(level: string) { + if (Object.keys(logLevels).includes(level.toLowerCase())) { + this.logger.level = level.toLowerCase(); + // Ensure we do not bypass silent / file-only modes + if (!this.isSilent) { + console.log(`Log level set to: ${level}`); + } + } else { + this.error(`Invalid log level: ${level}. Using current level: ${this.logger.level}`); + } + } + + // Get the current log file path + getLogFilePath(): string | null { + return this.logFilePath; + } + + // Get current log level + getLevel(): string { + return this.logger.level; + } + + // CLI startup information display methods + displayStartupInfo(info: { + configPath?: string; + model?: string; + provider?: string; + connectedServers?: { count: number; names: string[] }; + failedConnections?: { [key: string]: string }; + toolStats?: { total: number; mcp: number; internal: number }; + sessionId?: string; + logLevel?: string; + logFile?: string; + }) { + if (this.isSilent) return; + + console.log(''); // Add spacing + + if (info.configPath) { + console.log(`📄 ${chalk.bold('Config:')} ${chalk.dim(info.configPath)}`); + } + + if (info.model && info.provider) { + console.log( + `🤖 ${chalk.bold('Current Model:')} ${chalk.cyan(info.model)} ${chalk.dim(`(${info.provider})`)}` + ); + } + + if (info.connectedServers) { + if (info.connectedServers.count > 0) { + const serverNames = info.connectedServers.names.join(', '); + console.log( + `🔗 ${chalk.bold('Connected Servers:')} ${chalk.green(info.connectedServers.count)} ${chalk.dim(`(${serverNames})`)}` + ); + } else { + console.log( + `🔗 ${chalk.bold('Connected Servers:')} ${chalk.yellow('0')} ${chalk.dim('(no MCP servers connected)')}` + ); + } + } + + if (info.failedConnections && Object.keys(info.failedConnections).length > 0) { + const failedNames = Object.keys(info.failedConnections); + console.log( + `❌ ${chalk.bold('Failed Connections:')} ${chalk.red(failedNames.length)} ${chalk.dim(`(${failedNames.join(', ')})`)}` + ); + // Show specific error details + for (const [serverName, error] of Object.entries(info.failedConnections)) { + console.log(` ${chalk.red('•')} ${chalk.dim(serverName)}: ${chalk.red(error)}`); + } + } + + if (info.toolStats) { + console.log( + `🛠️ ${chalk.bold('Available Tools:')} ${chalk.green(info.toolStats.total)} total ${chalk.dim(`(${info.toolStats.mcp} MCP, ${info.toolStats.internal} internal)`)}` + ); + } + + if (info.sessionId) { + console.log(`💬 ${chalk.bold('Session:')} ${chalk.blue(info.sessionId)}`); + } + + if (info.logLevel && info.logFile) { + console.log( + `📋 ${chalk.bold('Log Level:')} ${chalk.cyan(info.logLevel)} ${chalk.dim(`(file: ${info.logFile})`)}` + ); + } + } + + displayError(message: string, error?: Error) { + if (this.isSilent) return; + + const showStack = this.getLevel() === 'debug'; + const errorContent = + error?.stack && showStack + ? `${chalk.red('Error')}: ${chalk.red(message)}\n${chalk.dim(error.stack)}` + : `${chalk.red('Error')}: ${chalk.red(message)}`; + + console.log( + boxen(errorContent, { + padding: 1, + borderColor: 'red', + title: '❌ Error', + titleAlignment: 'center', + }) + ); + } +} + +// Export a default instance with log level from environment +export const logger = new Logger(); diff --git a/dexto/packages/core/src/logger/v2/dexto-logger.ts b/dexto/packages/core/src/logger/v2/dexto-logger.ts new file mode 100644 index 00000000..61205e29 --- /dev/null +++ b/dexto/packages/core/src/logger/v2/dexto-logger.ts @@ -0,0 +1,192 @@ +/** + * Dexto Logger + * + * Main logger implementation with multi-transport support. + * Supports structured logging, component-based categorization, and per-agent isolation. + */ + +import type { + IDextoLogger, + ILoggerTransport, + LogEntry, + LogLevel, + DextoLogComponent, +} from './types.js'; + +export interface DextoLoggerConfig { + /** Minimum log level to record */ + level: LogLevel; + /** Component identifier */ + component: DextoLogComponent; + /** Agent ID for multi-agent isolation */ + agentId: string; + /** Transport instances */ + transports: ILoggerTransport[]; + /** Shared level reference (internal - for child loggers to share parent's level) */ + _levelRef?: { value: LogLevel }; +} + +/** + * DextoLogger - Multi-transport logger with structured logging + */ +export class DextoLogger implements IDextoLogger { + /** Shared level reference - allows parent and all children to share the same level */ + private levelRef: { value: LogLevel }; + private component: DextoLogComponent; + private agentId: string; + private transports: ILoggerTransport[]; + + // Log level hierarchy for filtering + // Following Winston convention: lower number = more severe + // If level is 'debug', logs error(0), warn(1), info(2), debug(3) but not silly(4) + private static readonly LEVELS: Record = { + error: 0, + warn: 1, + info: 2, + debug: 3, + silly: 4, + }; + + constructor(config: DextoLoggerConfig) { + // Use shared level ref if provided (for child loggers), otherwise create new one + this.levelRef = config._levelRef ?? { value: config.level }; + this.component = config.component; + this.agentId = config.agentId; + this.transports = config.transports; + } + + debug(message: string, context?: Record): void { + if (this.shouldLog('debug')) { + this.log('debug', message, context); + } + } + + silly(message: string, context?: Record): void { + if (this.shouldLog('silly')) { + this.log('silly', message, context); + } + } + + info(message: string, context?: Record): void { + if (this.shouldLog('info')) { + this.log('info', message, context); + } + } + + warn(message: string, context?: Record): void { + if (this.shouldLog('warn')) { + this.log('warn', message, context); + } + } + + error(message: string, context?: Record): void { + if (this.shouldLog('error')) { + this.log('error', message, context); + } + } + + trackException(error: Error, context?: Record): void { + this.error(error.message, { + ...context, + errorName: error.name, + errorStack: error.stack, + errorType: error.constructor.name, + }); + } + + /** + * Internal log method that creates log entry and sends to transports + */ + private log(level: LogLevel, message: string, context?: Record): void { + const entry: LogEntry = { + level, + message, + timestamp: new Date().toISOString(), + component: this.component, + agentId: this.agentId, + context, + }; + + // Send to all transports + for (const transport of this.transports) { + try { + const result = transport.write(entry); + // Handle async transports - attach rejection handler to prevent unhandled promise rejections + if (result && typeof result === 'object' && 'catch' in result) { + (result as Promise).catch((error) => { + console.error('Logger transport error:', error); + }); + } + } catch (error) { + // Don't let transport errors break logging (handles sync errors) + console.error('Logger transport error:', error); + } + } + } + + /** + * Check if a log level should be recorded based on configured level + * Winston convention: log if level number <= configured level number + * So if configured is 'debug' (3), we log error(0), warn(1), info(2), debug(3) but not silly(4) + */ + private shouldLog(level: LogLevel): boolean { + return DextoLogger.LEVELS[level] <= DextoLogger.LEVELS[this.levelRef.value]; + } + + /** + * Set the log level dynamically + * Affects this logger and all child loggers (shared level reference) + */ + setLevel(level: LogLevel): void { + this.levelRef.value = level; + } + + /** + * Get the current log level + */ + getLevel(): LogLevel { + return this.levelRef.value; + } + + /** + * Get the log file path if file logging is configured + */ + getLogFilePath(): string | null { + // Find the FileTransport and get its path + for (const transport of this.transports) { + if ('getFilePath' in transport && typeof transport.getFilePath === 'function') { + return transport.getFilePath(); + } + } + return null; + } + + /** + * Create a child logger for a different component + * Shares the same transports and level reference but uses different component identifier + */ + createChild(component: DextoLogComponent): DextoLogger { + return new DextoLogger({ + level: this.levelRef.value, // Initial value (will be overridden by _levelRef) + component, + agentId: this.agentId, + transports: this.transports, + _levelRef: this.levelRef, // Share the same level reference + }); + } + + /** + * Cleanup all transports + */ + async destroy(): Promise { + for (const transport of this.transports) { + if (transport.destroy) { + try { + await transport.destroy(); + } catch (error) { + console.error('Error destroying transport:', error); + } + } + } + } +} diff --git a/dexto/packages/core/src/logger/v2/error-codes.ts b/dexto/packages/core/src/logger/v2/error-codes.ts new file mode 100644 index 00000000..3e8796db --- /dev/null +++ b/dexto/packages/core/src/logger/v2/error-codes.ts @@ -0,0 +1,15 @@ +/** + * Logger-specific error codes + * Covers transport initialization, configuration, and logging operations + */ +export enum LoggerErrorCode { + // Transport errors + TRANSPORT_NOT_IMPLEMENTED = 'logger_transport_not_implemented', + TRANSPORT_UNKNOWN_TYPE = 'logger_transport_unknown_type', + TRANSPORT_INITIALIZATION_FAILED = 'logger_transport_initialization_failed', + TRANSPORT_WRITE_FAILED = 'logger_transport_write_failed', + + // Configuration errors + INVALID_CONFIG = 'logger_invalid_config', + INVALID_LOG_LEVEL = 'logger_invalid_log_level', +} diff --git a/dexto/packages/core/src/logger/v2/errors.ts b/dexto/packages/core/src/logger/v2/errors.ts new file mode 100644 index 00000000..3ba8a8b2 --- /dev/null +++ b/dexto/packages/core/src/logger/v2/errors.ts @@ -0,0 +1,97 @@ +import { DextoRuntimeError } from '../../errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '../../errors/types.js'; +import { LoggerErrorCode } from './error-codes.js'; + +/** + * Logger error factory with typed methods for creating logger-specific errors + * Each method creates a properly typed error with LOGGER scope + */ +export class LoggerError { + /** + * Transport not yet implemented + */ + static transportNotImplemented( + transportType: string, + availableTransports: string[] + ): DextoRuntimeError { + return new DextoRuntimeError( + LoggerErrorCode.TRANSPORT_NOT_IMPLEMENTED, + ErrorScope.LOGGER, + ErrorType.USER, + `${transportType} transport not yet implemented. Available transports: ${availableTransports.join(', ')}`, + { transportType, availableTransports } + ); + } + + /** + * Unknown transport type + */ + static unknownTransportType(transportType: string): DextoRuntimeError { + return new DextoRuntimeError( + LoggerErrorCode.TRANSPORT_UNKNOWN_TYPE, + ErrorScope.LOGGER, + ErrorType.USER, + `Unknown transport type: ${transportType}`, + { transportType } + ); + } + + /** + * Transport initialization failed + */ + static transportInitializationFailed( + transportType: string, + reason: string, + details?: Record + ): DextoRuntimeError { + return new DextoRuntimeError( + LoggerErrorCode.TRANSPORT_INITIALIZATION_FAILED, + ErrorScope.LOGGER, + ErrorType.SYSTEM, + `Failed to initialize ${transportType} transport: ${reason}`, + { transportType, reason, ...(details ?? {}) } + ); + } + + /** + * Transport write operation failed + */ + static transportWriteFailed(transportType: string, error: unknown): DextoRuntimeError { + return new DextoRuntimeError( + LoggerErrorCode.TRANSPORT_WRITE_FAILED, + ErrorScope.LOGGER, + ErrorType.SYSTEM, + `Transport write failed for ${transportType}`, + { + transportType, + originalError: error instanceof Error ? error.message : String(error), + } + ); + } + + /** + * Invalid logger configuration + */ + static invalidConfig(message: string, context?: Record): DextoRuntimeError { + return new DextoRuntimeError( + LoggerErrorCode.INVALID_CONFIG, + ErrorScope.LOGGER, + ErrorType.USER, + `Invalid logger configuration: ${message}`, + context + ); + } + + /** + * Invalid log level + */ + static invalidLogLevel(level: string, validLevels: string[]): DextoRuntimeError { + return new DextoRuntimeError( + LoggerErrorCode.INVALID_LOG_LEVEL, + ErrorScope.LOGGER, + ErrorType.USER, + `Invalid log level '${level}'. Valid levels: ${validLevels.join(', ')}`, + { level, validLevels } + ); + } +} diff --git a/dexto/packages/core/src/logger/v2/schemas.ts b/dexto/packages/core/src/logger/v2/schemas.ts new file mode 100644 index 00000000..62815d9a --- /dev/null +++ b/dexto/packages/core/src/logger/v2/schemas.ts @@ -0,0 +1,108 @@ +/** + * Logger Configuration Schemas + * + * Zod schemas for logger configuration with multi-transport support. + * Supports console, file, and optional remote (Upstash) transports. + */ + +import { z } from 'zod'; + +/** + * Silent transport configuration (no-op, discards all logs) + */ +const SilentTransportSchema = z + .object({ + type: z.literal('silent'), + }) + .strict() + .describe('Silent transport that discards all logs (useful for sub-agents)'); + +/** + * Console transport configuration + */ +const ConsoleTransportSchema = z + .object({ + type: z.literal('console'), + colorize: z.boolean().default(true).describe('Enable colored output'), + }) + .strict() + .describe('Console transport for terminal output'); + +/** + * File transport configuration with rotation support + */ +const FileTransportSchema = z + .object({ + type: z.literal('file'), + path: z.string().describe('Absolute path to log file'), + maxSize: z + .number() + .positive() + .default(10 * 1024 * 1024) + .describe('Max file size in bytes before rotation (default: 10MB)'), + maxFiles: z + .number() + .int() + .positive() + .default(5) + .describe('Max number of rotated files to keep (default: 5)'), + }) + .strict() + .describe('File transport with rotation support'); + +/** + * Upstash Redis transport configuration (optional remote logging) + */ +const UpstashTransportSchema = z + .object({ + type: z.literal('upstash'), + url: z.string().url().describe('Upstash Redis REST URL'), + token: z.string().describe('Upstash Redis REST token'), + listName: z.string().default('dexto-logs').describe('Redis list name for log entries'), + maxListLength: z + .number() + .int() + .positive() + .default(10000) + .describe('Max entries in Redis list (default: 10000)'), + batchSize: z + .number() + .int() + .positive() + .default(100) + .describe('Number of log entries to batch before sending (default: 100)'), + }) + .strict() + .describe('Upstash Redis transport for remote logging'); + +/** + * Transport configuration (discriminated union) + */ +export const LoggerTransportSchema = z.discriminatedUnion('type', [ + SilentTransportSchema, + ConsoleTransportSchema, + FileTransportSchema, + UpstashTransportSchema, +]); + +export type LoggerTransportConfig = z.output; + +/** + * Logger configuration schema + */ +export const LoggerConfigSchema = z + .object({ + level: z + .enum(['debug', 'info', 'warn', 'error', 'silly']) + .default('error') + .describe('Minimum log level to record'), + transports: z + .array(LoggerTransportSchema) + .min(1) + .default([{ type: 'console', colorize: true }]) + .describe('Log output destinations'), + }) + .strict() + .describe('Logger configuration with multi-transport support'); + +export type LoggerConfig = z.output; diff --git a/dexto/packages/core/src/logger/v2/test-utils.ts b/dexto/packages/core/src/logger/v2/test-utils.ts new file mode 100644 index 00000000..7494921e --- /dev/null +++ b/dexto/packages/core/src/logger/v2/test-utils.ts @@ -0,0 +1,50 @@ +/** + * Test utilities for logger mocking + * + * Provides a reusable mock logger for tests that need IDextoLogger. + */ + +import { vi } from 'vitest'; +import type { IDextoLogger, LogLevel } from './types.js'; + +/** + * Creates a mock logger that satisfies IDextoLogger interface. + * All methods are vi.fn() mocks that can be spied on. + */ +export function createMockLogger(): IDextoLogger { + const mockLogger: IDextoLogger = { + debug: vi.fn(), + silly: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(() => mockLogger), + destroy: vi.fn(), + setLevel: vi.fn(), + getLevel: vi.fn((): LogLevel => 'info'), + getLogFilePath: vi.fn(() => null), + }; + return mockLogger; +} + +/** + * Creates a silent mock logger with no-op functions. + * Useful when you don't need to spy on logger calls. + */ +export function createSilentMockLogger(): IDextoLogger { + const mockLogger: IDextoLogger = { + debug: () => {}, + silly: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + trackException: () => {}, + createChild: () => mockLogger, + destroy: async () => {}, + setLevel: () => {}, + getLevel: () => 'info', + getLogFilePath: () => null, + }; + return mockLogger; +} diff --git a/dexto/packages/core/src/logger/v2/transport-factory.ts b/dexto/packages/core/src/logger/v2/transport-factory.ts new file mode 100644 index 00000000..fdbd37b9 --- /dev/null +++ b/dexto/packages/core/src/logger/v2/transport-factory.ts @@ -0,0 +1,53 @@ +/** + * Transport Factory + * + * Creates transport instances from configuration. + * Used by CLI enrichment layer to instantiate transports. + */ + +import type { ILoggerTransport } from './types.js'; +import type { LoggerTransportConfig } from './schemas.js'; +import { SilentTransport } from './transports/silent-transport.js'; +import { ConsoleTransport } from './transports/console-transport.js'; +import { FileTransport } from './transports/file-transport.js'; +import { LoggerError } from './errors.js'; + +/** + * Create a transport instance from configuration + * @param config Transport configuration from schema + * @returns Transport instance + */ +export function createTransport(config: LoggerTransportConfig): ILoggerTransport { + switch (config.type) { + case 'silent': + return new SilentTransport(); + + case 'console': + return new ConsoleTransport({ + colorize: config.colorize, + }); + + case 'file': + return new FileTransport({ + path: config.path, + maxSize: config.maxSize, + maxFiles: config.maxFiles, + }); + + case 'upstash': + // TODO: Implement UpstashTransport in Phase B (optional) + throw LoggerError.transportNotImplemented('upstash', ['silent', 'console', 'file']); + + default: + throw LoggerError.unknownTransportType((config as any).type); + } +} + +/** + * Create multiple transports from configuration array + * @param configs Array of transport configurations + * @returns Array of transport instances + */ +export function createTransports(configs: LoggerTransportConfig[]): ILoggerTransport[] { + return configs.map(createTransport); +} diff --git a/dexto/packages/core/src/logger/v2/transports/console-transport.ts b/dexto/packages/core/src/logger/v2/transports/console-transport.ts new file mode 100644 index 00000000..4483bd1b --- /dev/null +++ b/dexto/packages/core/src/logger/v2/transports/console-transport.ts @@ -0,0 +1,67 @@ +/** + * Console Transport + * + * Logs to stdout/stderr with optional color support. + * Uses chalk for color formatting. + */ + +import chalk from 'chalk'; +import type { ILoggerTransport, LogEntry, LogLevel } from '../types.js'; + +export interface ConsoleTransportConfig { + colorize?: boolean; +} + +/** + * Console transport for terminal output + */ +export class ConsoleTransport implements ILoggerTransport { + private colorize: boolean; + + constructor(config: ConsoleTransportConfig = {}) { + this.colorize = config.colorize ?? true; + } + + write(entry: LogEntry): void { + const timestamp = new Date(entry.timestamp).toLocaleTimeString(); + const component = `[${entry.component}:${entry.agentId}]`; + const levelLabel = `[${entry.level.toUpperCase()}]`; + + let message = `${timestamp} ${levelLabel} ${component} ${entry.message}`; + + if (this.colorize) { + const colorFn = this.getColorForLevel(entry.level); + message = colorFn(message); + } + + // Add structured context if present + if (entry.context && Object.keys(entry.context).length > 0) { + message += '\n' + JSON.stringify(entry.context, null, 2); + } + + // Use stderr for errors and warnings, stdout for others + if (entry.level === 'error' || entry.level === 'warn') { + console.error(message); + } else { + console.log(message); + } + } + + /** + * Get chalk color function for log level + */ + private getColorForLevel(level: LogLevel): (text: string) => string { + switch (level) { + case 'debug': + return chalk.gray; + case 'info': + return chalk.cyan; + case 'warn': + return chalk.yellow; + case 'error': + return chalk.red; + default: + return (s: string) => s; + } + } +} diff --git a/dexto/packages/core/src/logger/v2/transports/file-transport.ts b/dexto/packages/core/src/logger/v2/transports/file-transport.ts new file mode 100644 index 00000000..36719db8 --- /dev/null +++ b/dexto/packages/core/src/logger/v2/transports/file-transport.ts @@ -0,0 +1,197 @@ +/** + * File Transport + * + * Logs to a file with automatic rotation based on file size. + * Keeps a configurable number of rotated log files. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { ILoggerTransport, LogEntry } from '../types.js'; + +export interface FileTransportConfig { + /** Absolute path to log file */ + path: string; + /** Max file size in bytes before rotation (default: 10MB) */ + maxSize?: number; + /** Max number of rotated files to keep (default: 5) */ + maxFiles?: number; +} + +/** + * File transport with size-based rotation + */ +export class FileTransport implements ILoggerTransport { + private filePath: string; + private maxSize: number; + private maxFiles: number; + private writeStream: fs.WriteStream | null = null; + private currentSize: number = 0; + private isRotating: boolean = false; + private pendingLogs: string[] = []; + + constructor(config: FileTransportConfig) { + this.filePath = config.path; + this.maxSize = config.maxSize ?? 10 * 1024 * 1024; // 10MB default + this.maxFiles = config.maxFiles ?? 5; // Keep 5 files default + + // Ensure log directory exists + const dir = path.dirname(this.filePath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + // Get current file size if it exists + if (fs.existsSync(this.filePath)) { + const stats = fs.statSync(this.filePath); + this.currentSize = stats.size; + } + + // Create write stream + this.createWriteStream(); + } + + private createWriteStream(): void { + this.writeStream = fs.createWriteStream(this.filePath, { + flags: 'a', // Append mode + encoding: 'utf8', + }); + + this.writeStream.on('error', (error) => { + console.error('FileTransport write stream error:', error); + }); + } + + write(entry: LogEntry): void { + // Format log entry as JSON line + const line = JSON.stringify(entry) + '\n'; + const lineSize = Buffer.byteLength(line, 'utf8'); + + // Buffer logs if not ready or rotating (prevents log loss) + if (!this.writeStream || this.isRotating) { + this.pendingLogs.push(line); + return; + } + + // Check if rotation is needed + if (this.currentSize + lineSize > this.maxSize) { + // Buffer this log and trigger async rotation + this.pendingLogs.push(line); + void this.rotate(); // Fire and forget - logs are buffered + return; + } + + // Write to file immediately + this.writeStream.write(line); + this.currentSize += lineSize; + } + + /** + * Rotate log files asynchronously + * Renames current file to .1, shifts existing rotated files up (.1 -> .2, etc.) + * Deletes oldest file if maxFiles is exceeded, then flushes buffered logs + */ + private async rotate(): Promise { + if (this.isRotating) { + return; + } + + this.isRotating = true; + + try { + // Close current write stream asynchronously + if (this.writeStream) { + await new Promise((resolve) => { + this.writeStream!.end(() => resolve()); + }); + this.writeStream = null; + } + + // Use async file operations to avoid blocking event loop + const promises = []; + + // Delete oldest rotated file if it exists + const oldestFile = `${this.filePath}.${this.maxFiles}`; + try { + await fs.promises.access(oldestFile); + promises.push(fs.promises.unlink(oldestFile)); + } catch { + // File doesn't exist, skip + } + + await Promise.all(promises); + + // Shift existing rotated files up by one + for (let i = this.maxFiles - 1; i >= 1; i--) { + const oldFile = `${this.filePath}.${i}`; + const newFile = `${this.filePath}.${i + 1}`; + + try { + await fs.promises.access(oldFile); + await fs.promises.rename(oldFile, newFile); + } catch { + // File doesn't exist, skip + } + } + + // Rename current file to .1 + try { + await fs.promises.access(this.filePath); + await fs.promises.rename(this.filePath, `${this.filePath}.1`); + } catch { + // File doesn't exist, skip + } + + // Reset size counter and create new stream + this.currentSize = 0; + this.createWriteStream(); + + // Flush buffered logs after rotation completes + await this.flushPendingLogs(); + } catch (error) { + console.error('FileTransport rotation error:', error); + } finally { + this.isRotating = false; + } + } + + /** + * Flush buffered logs to the write stream + * Called after rotation completes to prevent log loss + */ + private async flushPendingLogs(): Promise { + while (this.pendingLogs.length > 0 && this.writeStream) { + const line = this.pendingLogs.shift()!; + const lineSize = Buffer.byteLength(line, 'utf8'); + + // Check if we need to rotate again + if (this.currentSize + lineSize > this.maxSize) { + // Put the log back and trigger another rotation + this.pendingLogs.unshift(line); + await this.rotate(); + break; + } + + // Write to stream + this.writeStream.write(line); + this.currentSize += lineSize; + } + } + + /** + * Get the log file path + */ + getFilePath(): string { + return this.filePath; + } + + /** + * Cleanup resources + */ + destroy(): void { + if (this.writeStream) { + this.writeStream.end(); + this.writeStream = null; + } + } +} diff --git a/dexto/packages/core/src/logger/v2/transports/silent-transport.ts b/dexto/packages/core/src/logger/v2/transports/silent-transport.ts new file mode 100644 index 00000000..9dc71ea4 --- /dev/null +++ b/dexto/packages/core/src/logger/v2/transports/silent-transport.ts @@ -0,0 +1,21 @@ +/** + * Silent Transport + * + * A no-op transport that discards all log entries. + * Used when logging needs to be completely suppressed (e.g., sub-agents). + */ + +import type { ILoggerTransport, LogEntry } from '../types.js'; + +/** + * SilentTransport - Discards all log entries + */ +export class SilentTransport implements ILoggerTransport { + write(_entry: LogEntry): void { + // Intentionally do nothing - discard all logs + } + + destroy(): void { + // Nothing to clean up + } +} diff --git a/dexto/packages/core/src/logger/v2/types.ts b/dexto/packages/core/src/logger/v2/types.ts new file mode 100644 index 00000000..20dbbe3d --- /dev/null +++ b/dexto/packages/core/src/logger/v2/types.ts @@ -0,0 +1,157 @@ +/** + * Logger Types and Interfaces + * + * Defines the core abstractions for the multi-transport logger architecture. + */ + +/** + * Log levels in order of severity + * Following Winston convention: error < warn < info < debug < silly + */ +export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'silly'; + +/** + * Component identifiers for structured logging + * Mirrors ErrorScope for consistency, with additional execution context components + */ +export enum DextoLogComponent { + // Core functional domains (matches ErrorScope) + AGENT = 'agent', + LLM = 'llm', + CONFIG = 'config', + CONTEXT = 'context', + SESSION = 'session', + MCP = 'mcp', + TOOLS = 'tools', + STORAGE = 'storage', + SYSTEM_PROMPT = 'system_prompt', + RESOURCE = 'resource', + PROMPT = 'prompt', + MEMORY = 'memory', + PLUGIN = 'plugin', + FILESYSTEM = 'filesystem', + PROCESS = 'process', + APPROVAL = 'approval', + + // Additional execution context components + API = 'api', + CLI = 'cli', + TELEMETRY = 'telemetry', + EXECUTOR = 'executor', +} + +/** + * Structured log entry + * All logs are converted to this format before being sent to transports + */ +export interface LogEntry { + /** Log level */ + level: LogLevel; + /** Primary log message */ + message: string; + /** ISO timestamp */ + timestamp: string; + /** Component that generated the log */ + component: DextoLogComponent; + /** Agent ID for multi-agent isolation */ + agentId: string; + /** Optional structured context data */ + context?: Record | undefined; +} + +/** + * Logger interface + * All logger implementations must implement this interface + */ +export interface IDextoLogger { + /** + * Log debug message + * @param message Log message + * @param context Optional structured context + */ + debug(message: string, context?: Record): void; + + /** + * Log silly message (most verbose, for detailed debugging like full JSON dumps) + * @param message Log message + * @param context Optional structured context + */ + silly(message: string, context?: Record): void; + + /** + * Log info message + * @param message Log message + * @param context Optional structured context + */ + info(message: string, context?: Record): void; + + /** + * Log warning message + * @param message Log message + * @param context Optional structured context + */ + warn(message: string, context?: Record): void; + + /** + * Log error message + * @param message Log message + * @param context Optional structured context + */ + error(message: string, context?: Record): void; + + /** + * Track exception with stack trace + * @param error Error object + * @param context Optional additional context + */ + trackException(error: Error, context?: Record): void; + + /** + * Create a child logger with a different component + * Shares the same transports, agentId, and level but uses a different component identifier + * @param component Component identifier for the child logger + * @returns New logger instance with specified component + */ + createChild(component: DextoLogComponent): IDextoLogger; + + /** + * Set the log level dynamically + * Affects this logger and all child loggers created from it (shared level reference) + * @param level New log level + */ + setLevel(level: LogLevel): void; + + /** + * Get the current log level + * @returns Current log level + */ + getLevel(): LogLevel; + + /** + * Get the log file path if file logging is enabled + * @returns Log file path or null if file logging is not configured + */ + getLogFilePath(): string | null; + + /** + * Cleanup resources and close transports + */ + destroy(): Promise; +} + +/** + * Base transport interface + * All transport implementations must implement this interface + */ +export interface ILoggerTransport { + /** + * Write a log entry to the transport + * @param entry Structured log entry + */ + write(entry: LogEntry): void | Promise; + + /** + * Cleanup resources when logger is destroyed + */ + destroy?(): void | Promise; +} diff --git a/dexto/packages/core/src/mcp/error-codes.ts b/dexto/packages/core/src/mcp/error-codes.ts new file mode 100644 index 00000000..90e9bc9f --- /dev/null +++ b/dexto/packages/core/src/mcp/error-codes.ts @@ -0,0 +1,23 @@ +/** + * MCP-specific error codes + * Includes server configuration, connection, and protocol errors + */ +export enum MCPErrorCode { + // Configuration validation (used in schemas/resolver) + SCHEMA_VALIDATION = 'mcp_schema_validation', + COMMAND_MISSING = 'mcp_command_missing', + DUPLICATE_NAME = 'mcp_duplicate_name', + + // Connection and lifecycle + CONNECTION_FAILED = 'mcp_connection_failed', + DISCONNECTION_FAILED = 'mcp_disconnection_failed', + + // Protocol errors + PROTOCOL_ERROR = 'mcp_protocol_error', + + // Operations + SERVER_NOT_FOUND = 'mcp_server_not_found', + TOOL_NOT_FOUND = 'mcp_tool_not_found', + PROMPT_NOT_FOUND = 'mcp_prompt_not_found', + RESOURCE_NOT_FOUND = 'mcp_resource_not_found', +} diff --git a/dexto/packages/core/src/mcp/errors.ts b/dexto/packages/core/src/mcp/errors.ts new file mode 100644 index 00000000..289a4888 --- /dev/null +++ b/dexto/packages/core/src/mcp/errors.ts @@ -0,0 +1,143 @@ +import { DextoRuntimeError } from '@core/errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { MCPErrorCode } from './error-codes.js'; + +/** + * MCP-specific error factory + * Creates properly typed errors for MCP operations + */ +export class MCPError { + /** + * MCP server connection failed + */ + static connectionFailed(serverName: string, reason: string) { + return new DextoRuntimeError( + MCPErrorCode.CONNECTION_FAILED, + ErrorScope.MCP, + ErrorType.THIRD_PARTY, + `Failed to connect to MCP server '${serverName}': ${reason}`, + { serverName, reason }, + 'Check that the MCP server is running and accessible' + ); + } + + /** + * MCP server disconnection failed + */ + static disconnectionFailed(serverName: string, reason: string) { + return new DextoRuntimeError( + MCPErrorCode.DISCONNECTION_FAILED, + ErrorScope.MCP, + ErrorType.SYSTEM, + `Failed to disconnect MCP server '${serverName}': ${reason}`, + { serverName, reason }, + 'Try restarting the application if the server remains in an inconsistent state' + ); + } + + /** + * MCP protocol error + */ + static protocolError(message: string, details?: unknown) { + return new DextoRuntimeError( + MCPErrorCode.PROTOCOL_ERROR, + ErrorScope.MCP, + ErrorType.THIRD_PARTY, + `MCP protocol error: ${message}`, + details, + 'Check MCP server compatibility and protocol version' + ); + } + + /** + * MCP duplicate server name + */ + static duplicateName(name: string, existingName: string) { + return new DextoRuntimeError( + MCPErrorCode.DUPLICATE_NAME, + ErrorScope.MCP, + ErrorType.USER, + `Server name '${name}' conflicts with existing '${existingName}'`, + { name, existingName }, + 'Use a unique name for each MCP server' + ); + } + + /** + * MCP server not found + */ + static serverNotFound(serverName: string, reason?: string) { + return new DextoRuntimeError( + MCPErrorCode.SERVER_NOT_FOUND, + ErrorScope.MCP, + ErrorType.NOT_FOUND, + `MCP server '${serverName}' not found${reason ? `: ${reason}` : ''}`, + { serverName, reason } + ); + } + + /** + * MCP tool not found + */ + static toolNotFound(toolName: string) { + return new DextoRuntimeError( + MCPErrorCode.TOOL_NOT_FOUND, + ErrorScope.MCP, + ErrorType.NOT_FOUND, + `No MCP tool found: ${toolName}`, + { toolName } + ); + } + + /** + * MCP prompt not found + */ + static promptNotFound(promptName: string) { + return new DextoRuntimeError( + MCPErrorCode.PROMPT_NOT_FOUND, + ErrorScope.MCP, + ErrorType.NOT_FOUND, + `No client found for prompt: ${promptName}`, + { promptName } + ); + } + + /** + * MCP resource not found + */ + static resourceNotFound(resourceUri: string) { + return new DextoRuntimeError( + MCPErrorCode.RESOURCE_NOT_FOUND, + ErrorScope.MCP, + ErrorType.NOT_FOUND, + `No client found for resource: ${resourceUri}`, + { resourceUri } + ); + } + + /** + * MCP client not connected + */ + static clientNotConnected(context?: string) { + return new DextoRuntimeError( + MCPErrorCode.CONNECTION_FAILED, + ErrorScope.MCP, + ErrorType.SYSTEM, + `MCP client is not connected${context ? `: ${context}` : ''}`, + { context } + ); + } + + /** + * Invalid tool schema + */ + static invalidToolSchema(toolName: string, reason: string) { + return new DextoRuntimeError( + MCPErrorCode.PROTOCOL_ERROR, + ErrorScope.MCP, + ErrorType.THIRD_PARTY, + `Tool '${toolName}' has invalid schema: ${reason}`, + { toolName, reason } + ); + } +} diff --git a/dexto/packages/core/src/mcp/index.ts b/dexto/packages/core/src/mcp/index.ts new file mode 100644 index 00000000..d5041616 --- /dev/null +++ b/dexto/packages/core/src/mcp/index.ts @@ -0,0 +1,7 @@ +export * from './manager.js'; +export * from './mcp-client.js'; +export * from './types.js'; +export * from './schemas.js'; +export * from './errors.js'; +export * from './error-codes.js'; +export * from './resolver.js'; diff --git a/dexto/packages/core/src/mcp/manager.integration.test.ts b/dexto/packages/core/src/mcp/manager.integration.test.ts new file mode 100644 index 00000000..4e173f2b --- /dev/null +++ b/dexto/packages/core/src/mcp/manager.integration.test.ts @@ -0,0 +1,386 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { MCPManager } from './manager.js'; +import { MCPClient } from './mcp-client.js'; +import { McpServerConfigSchema } from './schemas.js'; +import type { MCPResolvedResource } from './types.js'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; +import { createMockLogger } from '../logger/v2/test-utils.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Path to resources-demo-server relative to this test file +const RESOURCES_DEMO_PATH = resolve( + __dirname, + '../../../../examples/resources-demo-server/server.js' +); + +/** + * Integration tests for MCPManager with real MCP servers + * + * These tests connect to actual MCP servers to verify: + * - Real connection and caching behavior + * - Tool/prompt/resource discovery + * - Multi-server coordination + * - Cache performance + */ + +describe('MCPManager Integration Tests', () => { + let manager: MCPManager; + const mockLogger = createMockLogger(); + + beforeEach(() => { + manager = new MCPManager(mockLogger); + }); + + afterEach(async () => { + await manager.disconnectAll(); + }); + + describe('Resources Demo Server (Comprehensive)', () => { + it('should connect to resources-demo server and cache resources', async () => { + const config = McpServerConfigSchema.parse({ + command: 'node', + args: [RESOURCES_DEMO_PATH], + type: 'stdio', + env: {}, + }); + + const client = new MCPClient(mockLogger); + await client.connect(config, 'resources-demo'); + + manager.registerClient('resources-demo', client); + await manager.refresh(); + + // Verify resources are cached + const resources = await manager.listAllResources(); + expect(resources.length).toBeGreaterThan(0); + + console.log( + 'Resources Demo resources:', + resources.map((r: MCPResolvedResource) => r.summary.uri) + ); + expect( + resources.some( + (r: MCPResolvedResource) => r.summary.uri?.includes('mcp-demo://') ?? false + ) + ).toBe(true); + + // Verify second call uses cache + const resourcesCached = await manager.listAllResources(); + expect(resourcesCached).toEqual(resources); + }, 15000); + + it('should read resource content from resources-demo server', async () => { + const config = McpServerConfigSchema.parse({ + command: 'node', + args: [RESOURCES_DEMO_PATH], + type: 'stdio', + env: {}, + }); + + const client = new MCPClient(mockLogger); + await client.connect(config, 'resources-demo'); + + manager.registerClient('resources-demo', client); + await manager.refresh(); + + // Read a resource + const content = await manager.readResource( + 'mcp:resources-demo:mcp-demo://product-metrics' + ); + expect(content).toBeDefined(); + expect(content.contents).toBeDefined(); + expect(content.contents.length).toBeGreaterThan(0); + + const firstContent = content.contents[0]; + expect(firstContent).toBeDefined(); + + console.log('Resource content type:', firstContent!.mimeType); + expect(firstContent!.mimeType).toBe('application/json'); + }, 15000); + + it('should cache and retrieve prompts from resources-demo server', async () => { + const config = McpServerConfigSchema.parse({ + command: 'node', + args: [RESOURCES_DEMO_PATH], + type: 'stdio', + env: {}, + }); + + const client = new MCPClient(mockLogger); + await client.connect(config, 'resources-demo'); + + manager.registerClient('resources-demo', client); + await manager.refresh(); + + // Verify prompts are cached + const prompts = manager.getAllPromptMetadata(); + expect(prompts.length).toBeGreaterThan(0); + + console.log( + 'Resources Demo prompts:', + prompts.map((p) => p.promptName) + ); + expect(prompts.some((p) => p.promptName === 'analyze-metrics')).toBe(true); + expect(prompts.some((p) => p.promptName === 'generate-report')).toBe(true); + + // Verify second call uses cache + const promptsCached = manager.getAllPromptMetadata(); + expect(promptsCached).toEqual(prompts); + }, 15000); + + it('should retrieve and execute prompts with arguments', async () => { + const config = McpServerConfigSchema.parse({ + command: 'node', + args: [RESOURCES_DEMO_PATH], + type: 'stdio', + env: {}, + }); + + const client = new MCPClient(mockLogger); + await client.connect(config, 'resources-demo'); + + manager.registerClient('resources-demo', client); + await manager.refresh(); + + // Get a prompt with arguments + const promptResult = await manager.getPrompt('analyze-metrics', { + metric_type: 'revenue', + time_period: 'Q1 2025', + }); + + expect(promptResult).toBeDefined(); + expect(promptResult.messages).toBeDefined(); + expect(promptResult.messages.length).toBeGreaterThan(0); + + const firstMessage = promptResult.messages[0]; + expect(firstMessage).toBeDefined(); + + console.log('Prompt message role:', firstMessage!.role); + expect(firstMessage!.role).toBe('user'); + }, 15000); + + it('should cache and discover tools from resources-demo server', async () => { + const config = McpServerConfigSchema.parse({ + command: 'node', + args: [RESOURCES_DEMO_PATH], + type: 'stdio', + env: {}, + }); + + const client = new MCPClient(mockLogger); + await client.connect(config, 'resources-demo'); + + manager.registerClient('resources-demo', client); + await manager.refresh(); + + // Verify tools are cached + const tools = await manager.getAllTools(); + const toolNames = Object.keys(tools); + + console.log('Resources Demo tools:', toolNames); + expect(toolNames.length).toBeGreaterThan(0); + expect(toolNames.some((name) => name.includes('calculate-growth-rate'))).toBe(true); + expect(toolNames.some((name) => name.includes('format-metric'))).toBe(true); + + // Verify tool schema - tools have 'parameters' field (converted from inputSchema) + const growthRateTool = tools['calculate-growth-rate']; + expect( + growthRateTool, + `Tool calculate-growth-rate should exist.\nAvailable tools: ${JSON.stringify(tools, null, 2)}` + ).toBeDefined(); + + console.log('Growth rate tool has parameters:', !!growthRateTool!.parameters); + expect(growthRateTool!.parameters).toBeDefined(); + expect(growthRateTool!.parameters.type).toBe('object'); + expect(growthRateTool!.parameters.properties).toBeDefined(); + }, 15000); + + it('should have all three capabilities: resources, prompts, and tools', async () => { + const config = McpServerConfigSchema.parse({ + command: 'node', + args: [RESOURCES_DEMO_PATH], + type: 'stdio', + env: {}, + }); + + const client = new MCPClient(mockLogger); + await client.connect(config, 'resources-demo'); + + manager.registerClient('resources-demo', client); + await manager.refresh(); + + // Check all three capabilities + const resources = await manager.listAllResources(); + const prompts = manager.getAllPromptMetadata(); + const tools = await manager.getAllTools(); + + console.log( + 'Summary: Resources:', + resources.length, + 'Prompts:', + prompts.length, + 'Tools:', + Object.keys(tools).length + ); + + expect(resources.length).toBeGreaterThan(0); + expect(prompts.length).toBeGreaterThan(0); + expect(Object.keys(tools).length).toBeGreaterThan(0); + }, 15000); + }); + + describe('Memory MCP Server', () => { + it('should connect to memory server and cache tools', async () => { + const config = McpServerConfigSchema.parse({ + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-memory'], + type: 'stdio', + env: {}, + }); + + const client = new MCPClient(mockLogger); + await client.connect(config, 'memory'); + + manager.registerClient('memory', client); + await manager.refresh(); + + // Verify tools are cached + const tools = await manager.getAllTools(); + expect(Object.keys(tools).length).toBeGreaterThan(0); + + // Memory server should have memory-related tools + const toolNames = Object.keys(tools); + console.log('Memory server tools:', toolNames); + expect( + toolNames.some((name) => name.includes('memory') || name.includes('entities')) + ).toBe(true); + }, 15000); + }); + + describe('Multi-Server Integration', () => { + it('should handle two different servers with separate caches', async () => { + // Connect resources-demo server + const resourcesConfig = McpServerConfigSchema.parse({ + command: 'node', + args: [RESOURCES_DEMO_PATH], + type: 'stdio', + env: {}, + }); + + const resourcesClient = new MCPClient(mockLogger); + await resourcesClient.connect(resourcesConfig, 'resources-demo'); + + // Connect memory server + const memoryConfig = McpServerConfigSchema.parse({ + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-memory'], + type: 'stdio', + env: {}, + }); + + const memoryClient = new MCPClient(mockLogger); + await memoryClient.connect(memoryConfig, 'memory'); + + // Register both + manager.registerClient('resources-demo', resourcesClient); + manager.registerClient('memory', memoryClient); + + await manager.refresh(); + + // Verify both servers' resources/tools are available + const resources = await manager.listAllResources(); + const tools = await manager.getAllTools(); + + console.log('Resources from both servers:', resources.length); + console.log('Tools from both servers:', Object.keys(tools).length); + + // Should have resources from resources-demo + expect(resources.length).toBeGreaterThan(0); + // Should have tools from memory server + expect(Object.keys(tools).length).toBeGreaterThan(0); + }, 20000); + + it('should cleanup one server without affecting the other', async () => { + // Connect both servers + const resourcesConfig = McpServerConfigSchema.parse({ + command: 'node', + args: [RESOURCES_DEMO_PATH], + type: 'stdio', + env: {}, + }); + + const resourcesClient = new MCPClient(mockLogger); + await resourcesClient.connect(resourcesConfig, 'resources-demo'); + + const memoryConfig = McpServerConfigSchema.parse({ + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-memory'], + type: 'stdio', + env: {}, + }); + + const memoryClient = new MCPClient(mockLogger); + await memoryClient.connect(memoryConfig, 'memory'); + + manager.registerClient('resources-demo', resourcesClient); + manager.registerClient('memory', memoryClient); + + await manager.refresh(); + + const toolsBefore = Object.keys(await manager.getAllTools()).length; + + // Remove resources-demo + await manager.removeClient('resources-demo'); + + const resourcesAfter = (await manager.listAllResources()).length; + const toolsAfter = Object.keys(await manager.getAllTools()).length; + + // Should have no resources now (resources-demo was removed) + expect(resourcesAfter).toBe(0); + + // Should still have memory server tools + expect(toolsAfter).toBeGreaterThan(0); + // Tool count decreased because resources-demo had tools that were removed + expect(toolsAfter).toBeLessThan(toolsBefore); + }, 20000); + }); + + describe('Cache Performance', () => { + it('should demonstrate caching eliminates network calls', async () => { + const config = McpServerConfigSchema.parse({ + command: 'npx', + args: ['-y', '@modelcontextprotocol/server-memory'], + type: 'stdio', + env: {}, + }); + + const client = new MCPClient(mockLogger); + await client.connect(config, 'memory'); + + manager.registerClient('memory', client); + + // Time first call (populates cache) + const start1 = Date.now(); + await manager.refresh(); + const firstCall = await manager.getAllTools(); + const time1 = Date.now() - start1; + + // Time second call (from cache) + const start2 = Date.now(); + const secondCall = await manager.getAllTools(); + const time2 = Date.now() - start2; + + // Verify same tools returned + expect(secondCall).toEqual(firstCall); + + // Cache should be faster (relaxed from time1/2 to avoid flakes under load) + console.log( + `First call (with cache): ${time1}ms, Second call (from cache): ${time2}ms` + ); + expect(time2).toBeLessThan(time1); + }, 15000); + }); +}); diff --git a/dexto/packages/core/src/mcp/manager.test.ts b/dexto/packages/core/src/mcp/manager.test.ts new file mode 100644 index 00000000..274a6948 --- /dev/null +++ b/dexto/packages/core/src/mcp/manager.test.ts @@ -0,0 +1,1014 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { EventEmitter } from 'events'; +import { MCPManager } from './manager.js'; +import { IMCPClient, MCPResourceSummary } from './types.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { MCPErrorCode } from './error-codes.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import { eventBus } from '../events/index.js'; +import type { JSONSchema7 } from 'json-schema'; +import type { Prompt } from '@modelcontextprotocol/sdk/types.js'; + +// Mock client for testing +class MockMCPClient extends EventEmitter implements IMCPClient { + private tools: Record< + string, + { name?: string; description?: string; parameters: JSONSchema7 } + > = {}; + private prompts: string[] = []; + private resources: MCPResourceSummary[] = []; + + constructor( + tools: Record< + string, + { name?: string; description?: string; parameters: JSONSchema7 } + > = {}, + prompts: string[] = [], + resources: MCPResourceSummary[] = [] + ) { + super(); + this.tools = tools; + this.prompts = prompts; + this.resources = resources; + } + + async connect(): Promise { + return {} as any; // Mock client + } + async disconnect(): Promise {} + + async getConnectedClient(): Promise { + return {} as any; // Mock client + } + + async getTools(): Promise< + Record + > { + return this.tools; + } + + async callTool(name: string, args: any): Promise { + if (!this.tools[name]) { + throw new Error(`Tool ${name} not found`); + } + return { result: `Called ${name} with ${JSON.stringify(args)}` }; + } + + async listPrompts(): Promise { + return this.prompts.map((name) => ({ + name, + description: `Prompt ${name}`, + })); + } + + async getPrompt(name: string, _args?: any): Promise { + if (!this.prompts.includes(name)) { + throw new Error(`Prompt ${name} not found`); + } + return { + description: `Prompt ${name}`, + messages: [{ role: 'user', content: { type: 'text', text: `Content for ${name}` } }], + }; + } + + async listResources(): Promise { + return this.resources; + } + + async readResource(uri: string): Promise { + if (!this.resources.find((r) => r.uri === uri)) { + throw new Error(`Resource ${uri} not found`); + } + return { + contents: [{ uri, mimeType: 'text/plain', text: `Resource content for ${uri}` }], + }; + } + + // Public setters for test manipulation + setTools( + tools: Record + ): void { + this.tools = tools; + } + + setPrompts(prompts: string[]): void { + this.prompts = prompts; + } + + setResources(resources: MCPResourceSummary[]): void { + this.resources = resources; + } +} + +describe('MCPManager Tool Conflict Resolution', () => { + let manager: MCPManager; + let client1: MockMCPClient; + let client2: MockMCPClient; + let client3: MockMCPClient; + let mockLogger: any; + + beforeEach(() => { + mockLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + silly: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), + } as any; + manager = new MCPManager(mockLogger); + + // Create clients with overlapping and unique tools + client1 = new MockMCPClient({ + unique_tool_1: { + description: 'Tool unique to server 1', + parameters: { type: 'object', properties: {} }, + }, + shared_tool: { + description: 'Tool shared between servers', + parameters: { type: 'object', properties: {} }, + }, + tool__with__underscores: { + description: 'Tool with underscores in name', + parameters: { type: 'object', properties: {} }, + }, + }); + + client2 = new MockMCPClient({ + unique_tool_2: { + description: 'Tool unique to server 2', + parameters: { type: 'object', properties: {} }, + }, + shared_tool: { + description: 'Different implementation of shared tool', + parameters: { type: 'object', properties: {} }, + }, + another_shared: { + description: 'Another shared tool', + parameters: { type: 'object', properties: {} }, + }, + }); + + client3 = new MockMCPClient({ + unique_tool_3: { + description: 'Tool unique to server 3', + parameters: { type: 'object', properties: {} }, + }, + another_shared: { + description: 'Third implementation of another_shared', + parameters: { type: 'object', properties: {} }, + }, + }); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe('Basic Tool Registration and Conflict Detection', () => { + it('should register tools from single client without conflicts', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + const tools = await manager.getAllTools(); + + expect(tools).toHaveProperty('unique_tool_1'); + expect(tools).toHaveProperty('shared_tool'); + expect(tools).toHaveProperty('tool__with__underscores'); + expect(Object.keys(tools)).toHaveLength(3); + }); + + it('should detect conflicts and use qualified names', async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + const tools = await manager.getAllTools(); + + // Unique tools should be available directly + expect(tools).toHaveProperty('unique_tool_1'); + expect(tools).toHaveProperty('unique_tool_2'); + + // Conflicted tools should be qualified + expect(tools).toHaveProperty('server1--shared_tool'); + expect(tools).toHaveProperty('server2--shared_tool'); + expect(tools).not.toHaveProperty('shared_tool'); // Unqualified should not exist + + // Verify descriptions are augmented (qualified tools always have descriptions) + expect(tools['server1--shared_tool']!.description!).toContain('(via server1)'); + expect(tools['server2--shared_tool']!.description!).toContain('(via server2)'); + }); + + it('should handle three-way conflicts correctly', async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + manager.registerClient('server3', client3); + + await Promise.all([ + manager['updateClientCache']('server1', client1), + manager['updateClientCache']('server2', client2), + manager['updateClientCache']('server3', client3), + ]); + + const tools = await manager.getAllTools(); + + // Check that 'another_shared' appears as qualified from server2 and server3 + expect(tools).toHaveProperty('server2--another_shared'); + expect(tools).toHaveProperty('server3--another_shared'); + expect(tools).not.toHaveProperty('another_shared'); + + // Unique tools should still be available + expect(tools).toHaveProperty('unique_tool_1'); + expect(tools).toHaveProperty('unique_tool_2'); + expect(tools).toHaveProperty('unique_tool_3'); + }); + }); + + describe('Conflict Resolution and Tool Restoration', () => { + it('should restore tools to fast lookup when conflicts disappear', async () => { + // Register two servers with conflicting tools + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + let tools = await manager.getAllTools(); + expect(tools).toHaveProperty('server1--shared_tool'); + expect(tools).toHaveProperty('server2--shared_tool'); + expect(tools).not.toHaveProperty('shared_tool'); + + // Remove one server to resolve conflict + await manager.removeClient('server2'); + + tools = await manager.getAllTools(); + + // Now shared_tool should be available directly (conflict resolved) + expect(tools).toHaveProperty('shared_tool'); + expect(tools).not.toHaveProperty('server1--shared_tool'); + expect(tools).not.toHaveProperty('server2--shared_tool'); + + // Verify it can be resolved via getToolClient + const client = manager.getToolClient('shared_tool'); + expect(client).toBe(client1); + }); + + it('should handle complex conflict resolution scenarios', async () => { + // Register all three servers + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + manager.registerClient('server3', client3); + + await Promise.all([ + manager['updateClientCache']('server1', client1), + manager['updateClientCache']('server2', client2), + manager['updateClientCache']('server3', client3), + ]); + + // Remove server2, 'another_shared' should still be conflicted between server3 + await manager.removeClient('server2'); + + const tools = await manager.getAllTools(); + + // 'shared_tool' should be resolved since only server1 has it now + expect(tools).toHaveProperty('shared_tool'); + expect(tools).not.toHaveProperty('server1--shared_tool'); + + // 'another_shared' should still not exist as direct tool since server3 still has it + // Actually, with only one server having it, it should be restored + expect(tools).toHaveProperty('another_shared'); + expect(tools).not.toHaveProperty('server3--another_shared'); + }); + }); + + describe('Server Name Sanitization and Collision Prevention', () => { + it('should sanitize server names correctly', () => { + const sanitize = manager['sanitizeServerName'].bind(manager); + + expect(sanitize('my-server')).toBe('my-server'); + expect(sanitize('my_server')).toBe('my_server'); + expect(sanitize('my@server')).toBe('my_server'); + expect(sanitize('my.server')).toBe('my_server'); + expect(sanitize('my server')).toBe('my_server'); + expect(sanitize('my/server\\path')).toBe('my_server_path'); + }); + + it('should prevent sanitized name collisions', () => { + manager.registerClient('my_server', client1); + + const error = (() => { + try { + manager.registerClient('my@server', client2); // Both sanitize to 'my_server' + return null; + } catch (e) { + return e; + } + })() as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(MCPErrorCode.DUPLICATE_NAME); + expect(error.scope).toBe(ErrorScope.MCP); + expect(error.type).toBe(ErrorType.USER); + }); + + it('should allow re-registering the same server name', () => { + manager.registerClient('server1', client1); + + // Should not throw when re-registering the same name + expect(() => { + manager.registerClient('server1', client2); + }).not.toThrow(); + }); + + it('should clean up sanitized mappings on client removal', async () => { + manager.registerClient('my@server', client1); + await manager.removeClient('my@server'); + + // Should now be able to register a server that sanitizes to the same name + expect(() => { + manager.registerClient('my_server', client2); + }).not.toThrow(); + }); + }); + + describe('Qualified Tool Name Parsing', () => { + beforeEach(async () => { + manager.registerClient('server__with__underscores', client1); + manager.registerClient('server@@with@@delimiters', client2); + await manager['updateClientCache']('server__with__underscores', client1); + await manager['updateClientCache']('server@@with@@delimiters', client2); + }); + + it('should parse qualified names correctly using last delimiter', async () => { + const parse = manager['parseQualifiedToolName'].bind(manager); + + // Wait for cache to populate + await manager['updateClientCache']('server__with__underscores', client1); + await manager['updateClientCache']('server@@with@@delimiters', client2); + + // shared_tool is the only conflicted tool - both servers have it, so it's qualified + const result1 = parse('server__with__underscores--shared_tool'); + expect(result1).toEqual({ + serverName: 'server__with__underscores', + toolName: 'shared_tool', + }); + + // Server name with delimiters gets sanitized, so we need to use the sanitized version + // 'server@@with@@delimiters' becomes 'server__with__delimiters' when sanitized + const result2 = parse('server__with__delimiters--shared_tool'); + expect(result2).toEqual({ + serverName: 'server@@with@@delimiters', + toolName: 'shared_tool', + }); + }); + + it('should return null for non-qualified names', () => { + const parse = manager['parseQualifiedToolName'].bind(manager); + + expect(parse('simple_tool')).toBeNull(); + expect(parse('tool__with__underscores')).toBeNull(); + expect(parse('')).toBeNull(); + }); + + it('should return null for invalid qualified names', () => { + const parse = manager['parseQualifiedToolName'].bind(manager); + + // Non-existent server + expect(parse('nonexistent--tool')).toBeNull(); + + // Non-existent tool on valid server + expect(parse('server__with__underscores--nonexistent_tool')).toBeNull(); + }); + }); + + describe('Tool Client Resolution', () => { + beforeEach(async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + }); + + it('should resolve non-conflicted tools directly', () => { + const client = manager.getToolClient('unique_tool_1'); + expect(client).toBe(client1); + + const client2Instance = manager.getToolClient('unique_tool_2'); + expect(client2Instance).toBe(client2); + }); + + it('should resolve qualified conflicted tools', () => { + const client1Instance = manager.getToolClient('server1--shared_tool'); + expect(client1Instance).toBe(client1); + + const client2Instance = manager.getToolClient('server2--shared_tool'); + expect(client2Instance).toBe(client2); + }); + + it('should return undefined for non-existent tools', () => { + expect(manager.getToolClient('nonexistent_tool')).toBeUndefined(); + expect(manager.getToolClient('server1--nonexistent_tool')).toBeUndefined(); + expect(manager.getToolClient('nonexistent_server--tool')).toBeUndefined(); + }); + + it('should not resolve conflicted tools without qualification', () => { + // 'shared_tool' exists on both servers, so unqualified lookup should fail + expect(manager.getToolClient('shared_tool')).toBeUndefined(); + }); + }); + + describe('Performance Optimizations and Caching', () => { + it('should use cache for getAllTools (no network calls)', async () => { + const getToolsSpy1 = vi.spyOn(client1, 'getTools'); + const getToolsSpy2 = vi.spyOn(client2, 'getTools'); + + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + // Reset spy counts (updateClientCache calls getTools during initialization) + getToolsSpy1.mockClear(); + getToolsSpy2.mockClear(); + + // Call getAllTools multiple times - should use cache, NO network calls + await manager.getAllTools(); + await manager.getAllTools(); + await manager.getAllTools(); + + // ZERO calls to getTools - uses toolCache + expect(getToolsSpy1).toHaveBeenCalledTimes(0); + expect(getToolsSpy2).toHaveBeenCalledTimes(0); + }); + + it('should use O(1) lookup for qualified name parsing', async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + // The parseQualifiedToolName method should use the sanitizedNameToServerMap + // for O(1) lookup instead of iterating through all servers + const result = manager['parseQualifiedToolName']('server1--shared_tool'); + expect(result).toEqual({ + serverName: 'server1', + toolName: 'shared_tool', + }); + + // Verify the sanitized map contains the expected mappings + const sanitizedMap = manager['sanitizedNameToServerMap']; + expect(sanitizedMap.get('server1')).toBe('server1'); + expect(sanitizedMap.get('server2')).toBe('server2'); + }); + }); + + describe('Tool Execution with Qualified Names', () => { + beforeEach(async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + }); + + it('should execute non-conflicted tools directly', async () => { + const result = await manager.executeTool('unique_tool_1', { param: 'value' }); + expect(result.result).toBe('Called unique_tool_1 with {"param":"value"}'); + }); + + it('should execute qualified conflicted tools', async () => { + const result1 = await manager.executeTool('server1--shared_tool', { param: 'test' }); + expect(result1.result).toBe('Called shared_tool with {"param":"test"}'); + + const result2 = await manager.executeTool('server2--shared_tool', { param: 'test' }); + expect(result2.result).toBe('Called shared_tool with {"param":"test"}'); + }); + + it('should throw error for non-existent tools', async () => { + await expect(manager.executeTool('nonexistent_tool', {})).rejects.toThrow( + 'No MCP tool found: nonexistent_tool' + ); + }); + + it('should throw error for unqualified conflicted tools', async () => { + await expect(manager.executeTool('shared_tool', {})).rejects.toThrow( + 'No MCP tool found: shared_tool' + ); + }); + }); + + describe('Edge Cases and Error Handling', () => { + it('should handle tools with @@ in their names', async () => { + const clientWithWeirdTool = new MockMCPClient({ + 'tool@@with@@delimiters': { + description: 'Tool with @@ in name', + parameters: { type: 'object', properties: {} }, + }, + }); + + manager.registerClient('normalserver', clientWithWeirdTool); + await manager['updateClientCache']('normalserver', clientWithWeirdTool); + + const tools = await manager.getAllTools(); + expect(tools).toHaveProperty('tool@@with@@delimiters'); + + // Should be able to execute it + const result = await manager.executeTool('tool@@with@@delimiters', {}); + expect(result.result).toBe('Called tool@@with@@delimiters with {}'); + }); + + it('should handle empty tool lists', async () => { + const emptyClient = new MockMCPClient({}); + manager.registerClient('empty_server', emptyClient); + await manager['updateClientCache']('empty_server', emptyClient); + + const tools = await manager.getAllTools(); + // Should not crash and should not add any tools + expect(Object.keys(tools)).toHaveLength(0); + }); + + it('should handle server disconnection gracefully', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + let tools = await manager.getAllTools(); + expect(Object.keys(tools)).toHaveLength(3); + + await manager.removeClient('server1'); + + // After removing the client, getAllTools should still call getTools() on disconnected clients + // but since we removed the server from serverToolsMap and toolToClientMap, it should return empty + tools = await manager.getAllTools(); + expect(Object.keys(tools)).toHaveLength(0); + }); + }); + + describe('Complete Cleanup', () => { + it('should clear all caches on disconnectAll', async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + // Verify caches are populated + expect(manager['sanitizedNameToServerMap'].size).toBe(2); + expect(manager['toolCache'].size).toBeGreaterThan(0); + expect(manager['toolConflicts'].size).toBeGreaterThan(0); + + await manager.disconnectAll(); + + // Verify all caches are cleared + expect(manager['sanitizedNameToServerMap'].size).toBe(0); + expect(manager['toolCache'].size).toBe(0); + expect(manager['toolConflicts'].size).toBe(0); + expect(manager['promptCache'].size).toBe(0); + expect(manager['resourceCache'].size).toBe(0); + }); + }); +}); + +describe('MCPManager Prompt Caching', () => { + let manager: MCPManager; + let client1: MockMCPClient; + let client2: MockMCPClient; + let mockLogger: any; + + beforeEach(() => { + mockLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + silly: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), + } as any; + manager = new MCPManager(mockLogger); + + client1 = new MockMCPClient({}, ['prompt1', 'prompt2', 'shared_prompt'], []); + + client2 = new MockMCPClient({}, ['prompt3', 'shared_prompt'], []); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('should cache prompts during updateClientCache', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + // Verify prompt cache is populated + expect(manager['promptCache'].size).toBe(3); + expect(manager['promptCache'].has('prompt1')).toBe(true); + expect(manager['promptCache'].has('prompt2')).toBe(true); + expect(manager['promptCache'].has('shared_prompt')).toBe(true); + }); + + it('should cache prompt metadata without calling getPrompt (performance optimization)', async () => { + const listPromptsSpy = vi.spyOn(client1, 'listPrompts'); + const getPromptSpy = vi.spyOn(client1, 'getPrompt'); + + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + // Verify listPrompts was called once during cache update + expect(listPromptsSpy).toHaveBeenCalledTimes(1); + + // Critical: getPrompt should NEVER be called - metadata comes from listPrompts + expect(getPromptSpy).toHaveBeenCalledTimes(0); + + // Verify metadata was cached correctly from listPrompts response + const metadata = manager.getPromptMetadata('prompt1'); + expect(metadata).toBeDefined(); + expect(metadata?.name).toBe('prompt1'); + expect(metadata?.description).toBe('Prompt prompt1'); + + // Still no getPrompt calls when retrieving metadata + expect(getPromptSpy).toHaveBeenCalledTimes(0); + }); + + it('should use cache for listAllPrompts (no network calls)', async () => { + const listPromptsSpy = vi.spyOn(client1, 'listPrompts'); + + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + listPromptsSpy.mockClear(); + + // Multiple calls should use cache + const prompts1 = await manager.listAllPrompts(); + const prompts2 = await manager.listAllPrompts(); + + expect(prompts1).toHaveLength(3); + expect(prompts2).toHaveLength(3); + expect(listPromptsSpy).toHaveBeenCalledTimes(0); // No network calls + }); + + it('should get prompt metadata from cache', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + const metadata = manager.getPromptMetadata('prompt1'); + expect(metadata).toBeDefined(); + expect(metadata?.name).toBe('prompt1'); + expect(metadata?.description).toBe('Prompt prompt1'); + }); + + it('should return all prompt metadata from cache', async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + const allMetadata = manager.getAllPromptMetadata(); + + // Should have 4 unique prompts (shared_prompt from server2 overwrites server1) + // Unlike tools, prompts don't have conflict detection - last writer wins + expect(allMetadata.length).toBe(4); + + const promptNames = allMetadata.map((m) => m.promptName); + expect(promptNames).toContain('prompt1'); + expect(promptNames).toContain('prompt2'); + expect(promptNames).toContain('prompt3'); + expect(promptNames).toContain('shared_prompt'); + + // Check server attribution + const prompt1Meta = allMetadata.find((m) => m.promptName === 'prompt1'); + expect(prompt1Meta?.serverName).toBe('server1'); + + // shared_prompt should be from server2 (last writer wins) + const sharedPromptMeta = allMetadata.find((m) => m.promptName === 'shared_prompt'); + expect(sharedPromptMeta?.serverName).toBe('server2'); + }); + + it('should clear prompt cache on client removal', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + expect(manager['promptCache'].size).toBe(3); + + await manager.removeClient('server1'); + + expect(manager['promptCache'].size).toBe(0); + }); +}); + +describe('MCPManager Resource Caching', () => { + let manager: MCPManager; + let client1: MockMCPClient; + let client2: MockMCPClient; + let mockLogger: any; + + beforeEach(() => { + mockLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + silly: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), + } as any; + manager = new MCPManager(mockLogger); + + client1 = new MockMCPClient( + {}, + [], + [ + { uri: 'file:///test1.txt', name: 'Test 1', mimeType: 'text/plain' }, + { uri: 'file:///test2.txt', name: 'Test 2', mimeType: 'text/plain' }, + ] + ); + + client2 = new MockMCPClient( + {}, + [], + [{ uri: 'file:///test3.txt', name: 'Test 3', mimeType: 'text/plain' }] + ); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('should cache resources during updateClientCache', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + // Verify resource cache is populated with qualified keys + expect(manager['resourceCache'].size).toBe(2); + expect(manager.hasResource('mcp:server1:file:///test1.txt')).toBe(true); + expect(manager.hasResource('mcp:server1:file:///test2.txt')).toBe(true); + }); + + it('should use cache for listAllResources (no network calls)', async () => { + const listResourcesSpy = vi.spyOn(client1, 'listResources'); + + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + listResourcesSpy.mockClear(); + + // Multiple calls should use cache + const resources1 = await manager.listAllResources(); + const resources2 = await manager.listAllResources(); + + expect(resources1).toHaveLength(2); + expect(resources2).toHaveLength(2); + expect(listResourcesSpy).toHaveBeenCalledTimes(0); // No network calls + }); + + it('should get resource metadata from cache', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + const resource = manager.getResource('mcp:server1:file:///test1.txt'); + expect(resource).toBeDefined(); + expect(resource?.summary.uri).toBe('file:///test1.txt'); + expect(resource?.summary.name).toBe('Test 1'); + expect(resource?.serverName).toBe('server1'); + }); + + it('should handle resources from multiple servers', async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + const allResources = await manager.listAllResources(); + + expect(allResources).toHaveLength(3); + + const serverNames = allResources.map((r) => r.serverName); + expect(serverNames).toContain('server1'); + expect(serverNames).toContain('server2'); + }); + + it('should clear resource cache on client removal', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + expect(manager['resourceCache'].size).toBe(2); + + await manager.removeClient('server1'); + + expect(manager['resourceCache'].size).toBe(0); + expect(manager.hasResource('mcp:server1:file:///test1.txt')).toBe(false); + }); + + it('should only clear resources for removed client', async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + expect(manager['resourceCache'].size).toBe(3); + + await manager.removeClient('server1'); + + // server2 resources should remain + expect(manager['resourceCache'].size).toBe(1); + expect(manager.hasResource('mcp:server2:file:///test3.txt')).toBe(true); + expect(manager.hasResource('mcp:server1:file:///test1.txt')).toBe(false); + }); +}); + +describe('Tool notification handling', () => { + let manager: MCPManager; + let client1: MockMCPClient; + let client2: MockMCPClient; + let mockLogger: any; + + beforeEach(() => { + mockLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + silly: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), + } as any; + manager = new MCPManager(mockLogger); + client1 = new MockMCPClient( + { + tool1: { name: 'tool1', description: 'Tool 1', parameters: {} }, + tool2: { name: 'tool2', description: 'Tool 2', parameters: {} }, + }, + [], + [] + ); + + client2 = new MockMCPClient( + { + tool3: { name: 'tool3', description: 'Tool 3', parameters: {} }, + }, + [], + [] + ); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('should refresh tool cache when toolsListChanged notification received', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + // Verify initial cache + expect(manager['toolCache'].size).toBe(2); + expect(manager['toolCache'].has('tool1')).toBe(true); + expect(manager['toolCache'].has('tool2')).toBe(true); + + // Update tools on the mock client + client1.setTools({ + tool1: { name: 'tool1', description: 'Tool 1 Updated', parameters: {} }, + tool3: { name: 'tool3', description: 'Tool 3', parameters: {} }, + }); + + // Trigger handleToolsListChanged + await manager['handleToolsListChanged']('server1', client1); + + // Verify cache was refreshed + expect(manager['toolCache'].size).toBe(2); + expect(manager['toolCache'].has('tool1')).toBe(true); + expect(manager['toolCache'].has('tool3')).toBe(true); + expect(manager['toolCache'].has('tool2')).toBe(false); // tool2 removed + }); + + it('should emit mcp:tools-list-changed event with correct payload', async () => { + const eventSpy = vi.fn(); + eventBus.on('mcp:tools-list-changed', eventSpy); + + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + // Update tools + client1.setTools({ + tool1: { name: 'tool1', description: 'Tool 1 Updated', parameters: {} }, + }); + + // Trigger notification handler + await manager['handleToolsListChanged']('server1', client1); + + expect(eventSpy).toHaveBeenCalledWith({ + serverName: 'server1', + tools: ['tool1'], + }); + + eventBus.off('mcp:tools-list-changed', eventSpy); + }); + + it('should detect conflicts when notification adds conflicting tool', async () => { + // Setup: server1 has tool1 + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + // Setup: server2 has tool3 + manager.registerClient('server2', client2); + await manager['updateClientCache']('server2', client2); + + expect(manager['toolConflicts'].has('tool1')).toBe(false); + + // Update server2 to also provide tool1 (conflict!) + client2.setTools({ + tool1: { name: 'tool1', description: 'Tool 1 from server2', parameters: {} }, + tool3: { name: 'tool3', description: 'Tool 3', parameters: {} }, + }); + + // Trigger notification handler + await manager['handleToolsListChanged']('server2', client2); + + // Should detect conflict and use qualified names + expect(manager['toolConflicts'].has('tool1')).toBe(true); + expect(manager['toolCache'].has('tool1')).toBe(false); // Simple name removed + expect(manager['toolCache'].has('server1--tool1')).toBe(true); + expect(manager['toolCache'].has('server2--tool1')).toBe(true); + }); + + it('should resolve conflicts when notification removes conflicting tool', async () => { + // Setup: Both servers provide tool1 (conflict exists) + client1.setTools({ + tool1: { name: 'tool1', description: 'Tool 1 from server1', parameters: {} }, + }); + client2.setTools({ + tool1: { name: 'tool1', description: 'Tool 1 from server2', parameters: {} }, + }); + + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + // Verify conflict detected + expect(manager['toolConflicts'].has('tool1')).toBe(true); + expect(manager['toolCache'].has('server1--tool1')).toBe(true); + expect(manager['toolCache'].has('server2--tool1')).toBe(true); + + // Update server2 to no longer provide tool1 + client2.setTools({ + tool3: { name: 'tool3', description: 'Tool 3', parameters: {} }, + }); + + // Trigger notification handler + await manager['handleToolsListChanged']('server2', client2); + + // Conflict should be resolved, tool1 restored to simple name + expect(manager['toolConflicts'].has('tool1')).toBe(false); + expect(manager['toolCache'].has('tool1')).toBe(true); + expect(manager['toolCache'].has('server1--tool1')).toBe(false); + expect(manager['toolCache'].has('server2--tool1')).toBe(false); + + // Verify it points to server1 + const entry = manager['toolCache'].get('tool1'); + expect(entry?.serverName).toBe('server1'); + }); + + it('should handle empty tool list notification', async () => { + manager.registerClient('server1', client1); + await manager['updateClientCache']('server1', client1); + + expect(manager['toolCache'].size).toBe(2); + + // Update to empty tools + client1.setTools({}); + + await manager['handleToolsListChanged']('server1', client1); + + // All server1 tools should be removed + const remainingTools = Array.from(manager['toolCache'].values()); + expect(remainingTools.every((entry) => entry.serverName !== 'server1')).toBe(true); + }); + + it('should not affect other servers tools during notification', async () => { + manager.registerClient('server1', client1); + manager.registerClient('server2', client2); + await manager['updateClientCache']('server1', client1); + await manager['updateClientCache']('server2', client2); + + expect(manager['toolCache'].size).toBe(3); + + // Update server1 tools + client1.setTools({ tool1: { name: 'tool1', description: 'Tool 1 Only', parameters: {} } }); + + await manager['handleToolsListChanged']('server1', client1); + + // server2's tool3 should still be there + expect(manager['toolCache'].has('tool3')).toBe(true); + const tool3Entry = manager['toolCache'].get('tool3'); + expect(tool3Entry?.serverName).toBe('server2'); + }); +}); diff --git a/dexto/packages/core/src/mcp/manager.ts b/dexto/packages/core/src/mcp/manager.ts new file mode 100644 index 00000000..0261e874 --- /dev/null +++ b/dexto/packages/core/src/mcp/manager.ts @@ -0,0 +1,1154 @@ +import { MCPClient } from './mcp-client.js'; +import { ValidatedServerConfigs, ValidatedMcpServerConfig } from './schemas.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import { GetPromptResult, ReadResourceResult, Prompt } from '@modelcontextprotocol/sdk/types.js'; +import { IMCPClient, MCPResolvedResource, MCPResourceSummary } from './types.js'; +import { ToolSet } from '../tools/types.js'; +import { MCPError } from './errors.js'; +import { eventBus } from '../events/index.js'; +import type { PromptDefinition } from '../prompts/types.js'; +import type { JSONSchema7 } from 'json-schema'; +import type { ApprovalManager } from '../approval/manager.js'; + +/** + * Centralized manager for Multiple Model Context Protocol (MCP) servers. + * + * The MCPManager serves as a focused interface for managing connections to MCP servers + * and providing access to their capabilities (tools, prompts, resources). + * + * Key responsibilities: + * - **Client Management**: Register, connect, disconnect, and remove MCP clients + * - **Resource Discovery**: Cache and provide access to tools, prompts, and resources from connected clients + * - **MCP Tool Execution**: Execute MCP tools with built-in confirmation mechanisms + * - **Connection Handling**: Support both strict and lenient connection modes with error tracking + * - **Caching**: Maintain efficient lookup maps for fast access to client capabilities + * + * The manager supports dynamic client connections, allowing servers to be added or removed at runtime. + * It includes robust error handling and maintains connection state for debugging purposes. + * + * Note: This class focuses only on MCP server management. For unified tool management + * across multiple sources (MCP + custom tools), use the ToolManager class. + * + * @example + * ```typescript + * const manager = new MCPManager(); + * await manager.initializeFromConfig(serverConfigs); + * + * // Execute an MCP tool + * const result = await manager.executeTool('my_tool', { param: 'value' }); + * + * // Get all available MCP tools + * const tools = await manager.getAllTools(); + * ``` + */ +type ResourceCacheEntry = { + serverName: string; + client: IMCPClient; + summary: MCPResourceSummary; +}; + +type PromptCacheEntry = { + serverName: string; + client: IMCPClient; + definition: PromptDefinition; +}; + +type ToolCacheEntry = { + serverName: string; + client: IMCPClient; + definition: { + name?: string; + description?: string; + parameters: JSONSchema7; + }; +}; + +export class MCPManager { + private clients: Map = new Map(); + private connectionErrors: { [key: string]: string } = {}; + private configCache: Map = new Map(); // Store original configs for restart + private toolCache: Map = new Map(); + private toolConflicts: Set = new Set(); // Track which tool names have conflicts + private promptCache: Map = new Map(); + private resourceCache: Map = new Map(); + private sanitizedNameToServerMap: Map = new Map(); + private approvalManager: ApprovalManager | null = null; // Will be set by service initializer + private logger: IDextoLogger; + + // Use a distinctive delimiter that won't appear in normal server/tool names + // Using double hyphen as it's allowed in LLM tool name patterns (^[a-zA-Z0-9_-]+$) + private static readonly SERVER_DELIMITER = '--'; + + constructor(logger: IDextoLogger) { + this.logger = logger.createChild(DextoLogComponent.MCP); + } + + /** + * Set the approval manager for handling elicitation requests from MCP servers + * + * TODO: Consider making ApprovalManager a required constructor parameter instead of using a setter. + * This would make the dependency explicit and remove the need for defensive `if (!approvalManager)` checks. + * Current setter pattern is useful if we want to expose MCPManager as a standalone service to end-users + * without requiring them to know about ApprovalManager. + */ + setApprovalManager(approvalManager: ApprovalManager): void { + this.approvalManager = approvalManager; + // Update all existing clients with the approval manager + for (const [_name, client] of this.clients.entries()) { + if (client instanceof MCPClient) { + client.setApprovalManager(approvalManager); + } + } + } + + private buildQualifiedResourceKey(serverName: string, resourceUri: string): string { + return `mcp:${serverName}:${resourceUri}`; + } + + private parseQualifiedResourceKey(key: string): { serverName: string; resourceUri: string } { + if (!key.startsWith('mcp:')) { + throw MCPError.resourceNotFound(key); + } + const [, serverName, ...rest] = key.split(':'); + if (!serverName || rest.length === 0) { + throw MCPError.resourceNotFound(key); + } + return { serverName, resourceUri: rest.join(':') }; + } + + private removeServerResources(serverName: string): void { + for (const [key, entry] of Array.from(this.resourceCache.entries())) { + if (entry.serverName === serverName) { + this.resourceCache.delete(key); + } + } + } + + private getResourceCacheEntry(resourceKey: string): ResourceCacheEntry | undefined { + if (this.resourceCache.has(resourceKey)) { + return this.resourceCache.get(resourceKey); + } + + try { + const { serverName, resourceUri } = this.parseQualifiedResourceKey(resourceKey); + const canonicalKey = this.buildQualifiedResourceKey(serverName, resourceUri); + return this.resourceCache.get(canonicalKey); + } catch { + return undefined; + } + } + + /** + * Register a client that provides tools (and potentially more) + * @param name Unique name for the client + * @param client The client instance, expected to be IMCPClient + */ + registerClient(name: string, client: IMCPClient): void { + if (this.clients.has(name)) { + this.logger.warn(`Client '${name}' already registered. Overwriting.`); + } + + // Clear cache first (which removes old mappings) + this.clearClientCache(name); + + // Validate sanitized name uniqueness to prevent collisions + const sanitizedName = this.sanitizeServerName(name); + const existingServerWithSameSanitizedName = + this.sanitizedNameToServerMap.get(sanitizedName); + if (existingServerWithSameSanitizedName && existingServerWithSameSanitizedName !== name) { + throw MCPError.duplicateName(name, existingServerWithSameSanitizedName); + } + + this.clients.set(name, client); + this.sanitizedNameToServerMap.set(sanitizedName, name); + this.setupClientNotifications(name, client); + + this.logger.info(`Registered client: ${name}`); + delete this.connectionErrors[name]; + } + + /** + * Clears all cached data for a disconnected MCP client + * + * Performs comprehensive cleanup of tool, prompt, and resource caches. + * Uses two-pass algorithm to detect and resolve tool name conflicts: + * if a conflicted tool now has only one provider, restores simple name. + * + * @param clientName - The name/identifier of the MCP server being removed + * @private + */ + private clearClientCache(clientName: string): void { + const client = this.clients.get(clientName); + if (!client) return; + + // Remove from sanitized name mapping + const sanitizedName = this.sanitizeServerName(clientName); + if (this.sanitizedNameToServerMap.get(sanitizedName) === clientName) { + this.sanitizedNameToServerMap.delete(sanitizedName); + } + + // Clear tool cache for this server and restore simple names when conflicts resolve + const removedToolBaseNames = new Set(); + + // First pass: collect base names and remove all tools from this server + for (const [toolKey, entry] of Array.from(this.toolCache.entries())) { + if (entry.serverName === clientName) { + // Extract base name from qualified key (handle both simple and qualified names) + const delimiterIndex = toolKey.lastIndexOf(MCPManager.SERVER_DELIMITER); + const baseName = + delimiterIndex === -1 + ? toolKey + : toolKey.substring(delimiterIndex + MCPManager.SERVER_DELIMITER.length); + + removedToolBaseNames.add(baseName); + this.toolCache.delete(toolKey); + } + } + + // Second pass: check for resolved conflicts and restore simple names + for (const baseName of removedToolBaseNames) { + // Find all remaining tools with this base name + const remainingTools = Array.from(this.toolCache.entries()).filter(([key, _]) => { + const delimiterIndex = key.lastIndexOf(MCPManager.SERVER_DELIMITER); + const bn = + delimiterIndex === -1 + ? key + : key.substring(delimiterIndex + MCPManager.SERVER_DELIMITER.length); + return bn === baseName; + }); + + if (remainingTools.length === 0) { + // No tools with this name remain + this.toolConflicts.delete(baseName); + } else if (remainingTools.length === 1 && this.toolConflicts.has(baseName)) { + // Exactly one tool remains - restore to simple name + const singleTool = remainingTools[0]; + if (singleTool) { + const [qualifiedKey, entry] = singleTool; + this.toolCache.delete(qualifiedKey); + this.toolCache.set(baseName, entry); + this.toolConflicts.delete(baseName); + this.logger.debug( + `Restored tool '${baseName}' to simple name (conflict resolved)` + ); + } + } + // If remainingTools.length > 1, conflict still exists, keep qualified names + } + + // Clear prompt metadata cache for this server + for (const [promptName, entry] of Array.from(this.promptCache.entries())) { + if (entry.serverName === clientName) { + this.promptCache.delete(promptName); + } + } + + // Clear resource cache for this server + for (const [key, entry] of Array.from(this.resourceCache.entries())) { + if (entry.client === client || entry.serverName === clientName) { + this.resourceCache.delete(key); + } + } + + this.logger.debug(`Cleared cache for client: ${clientName}`); + } + + /** + * Sanitize server name for use in tool prefixing. + * Ensures the name is safe for LLM provider tool naming constraints. + */ + private sanitizeServerName(serverName: string): string { + return serverName.replace(/[^a-zA-Z0-9_-]/g, '_'); + } + + /** + * Updates internal caches for a connected MCP client + * + * This method performs initial cache population after a client connects. + * It fetches and caches tools, prompts, and resources from the MCP server, + * implementing conflict detection and resolution for tool names. + * + * @param clientName - The name/identifier of the MCP server + * @param client - The connected MCP client instance + * + * @remarks + * **Tool Caching:** + * - Fetches all tools and caches them with full definitions + * - Detects naming conflicts when multiple servers provide same tool name + * - On conflict: uses qualified names (`server--toolname`) for all conflicting tools + * - Updates toolConflicts set to track which base names have conflicts + * + * **Prompt Caching:** + * - Fetches all prompts and their metadata (description, arguments) + * - Stores full prompt definitions in promptCache for efficient access + * - Falls back to minimal metadata if full definition fetch fails + * + * **Resource Caching:** + * - Fetches all resource summaries (uri, name, mimeType) + * - Stores resource metadata in resourceCache for quick lookups + * + * **Error Handling:** + * - Tool fetch errors abort caching entirely (early return) + * - Prompt/resource errors log warnings but don't block other caching + * - Individual prompt metadata errors are caught and logged + * + * @private + */ + private async updateClientCache(clientName: string, client: IMCPClient): Promise { + // Cache tools with full definitions + try { + const tools = await client.getTools(); + this.logger.debug( + `🔧 Discovered ${Object.keys(tools).length} tools from server '${clientName}': [${Object.keys(tools).join(', ')}]` + ); + + for (const toolName in tools) { + const toolDef = tools[toolName]; + if (!toolDef) continue; // Skip undefined tool definitions + + // Check if this tool name already exists from a different server + const existingEntry = this.toolCache.get(toolName); + if (existingEntry && existingEntry.serverName !== clientName) { + // Conflict detected! Move existing to qualified name + this.toolConflicts.add(toolName); + this.toolCache.delete(toolName); + + const existingSanitized = this.sanitizeServerName(existingEntry.serverName); + const existingQualified = `${existingSanitized}${MCPManager.SERVER_DELIMITER}${toolName}`; + this.toolCache.set(existingQualified, existingEntry); + + // Add new tool with qualified name + const newSanitized = this.sanitizeServerName(clientName); + const newQualified = `${newSanitized}${MCPManager.SERVER_DELIMITER}${toolName}`; + this.toolCache.set(newQualified, { + serverName: clientName, + client, + definition: toolDef, + }); + + this.logger.warn( + `⚠️ Tool conflict detected for '${toolName}' - using server prefixes: ${existingQualified}, ${newQualified}` + ); + } else if (this.toolConflicts.has(toolName)) { + // This tool name is already known to be conflicted + const sanitizedName = this.sanitizeServerName(clientName); + const qualifiedName = `${sanitizedName}${MCPManager.SERVER_DELIMITER}${toolName}`; + this.toolCache.set(qualifiedName, { + serverName: clientName, + client, + definition: toolDef, + }); + this.logger.debug(`✅ Tool '${qualifiedName}' cached (known conflict)`); + } else { + // No conflict, cache with simple name + this.toolCache.set(toolName, { + serverName: clientName, + client, + definition: toolDef, + }); + this.logger.debug(`✅ Tool '${toolName}' mapped to ${clientName}`); + } + } + this.logger.debug( + `✅ Successfully cached ${Object.keys(tools).length} tools for client: ${clientName}` + ); + } catch (error) { + this.logger.error( + `❌ Error retrieving tools for client ${clientName}: ${error instanceof Error ? error.message : String(error)}` + ); + return; // Early return on error, no caching + } + + // Cache prompts with metadata from listPrompts() (no additional network calls needed) + try { + const prompts: Prompt[] = await client.listPrompts(); + + for (const prompt of prompts) { + // Convert MCP SDK Prompt to our PromptDefinition + const definition: PromptDefinition = { + name: prompt.name, + ...(prompt.title && { title: prompt.title }), + ...(prompt.description && { description: prompt.description }), + ...(prompt.arguments && { arguments: prompt.arguments }), + }; + + this.promptCache.set(prompt.name, { + serverName: clientName, + client, + definition, + }); + } + + this.logger.debug(`Cached ${prompts.length} prompts for client: ${clientName}`); + } catch (error) { + this.logger.debug(`Skipping prompts for client ${clientName}: ${error}`); + } + + // Cache resources, if supported + // TODO: HF SERVER HAS 100000+ RESOURCES - need to think of a way to make resources/caching optional or better. + try { + this.removeServerResources(clientName); + const resources = await client.listResources(); + resources.forEach((summary) => { + const key = this.buildQualifiedResourceKey(clientName, summary.uri); + this.resourceCache.set(key, { + serverName: clientName, + client, + summary, + }); + }); + this.logger.debug(`Cached resources for client: ${clientName}`); + } catch (error) { + this.logger.debug(`Skipping resources for client ${clientName}: ${error}`); + } + } + + /** + * Get all available MCP tools from cache (no network calls). + * Conflicted tools are already stored with qualified names. + * @returns Promise resolving to a ToolSet mapping tool names to Tool definitions + */ + async getAllTools(): Promise { + const allTools: ToolSet = {}; + + // Build tool set from cache + for (const [toolKey, entry] of this.toolCache.entries()) { + const toolDef = entry.definition; + + // For qualified names (conflicts), enhance description with server name + if (toolKey.includes(MCPManager.SERVER_DELIMITER)) { + allTools[toolKey] = { + ...toolDef, + description: toolDef.description + ? `${toolDef.description} (via ${entry.serverName})` + : `Tool from ${entry.serverName}`, + }; + } else { + // Simple name, use as-is + allTools[toolKey] = toolDef; + } + } + + const serverNames = Array.from( + new Set(Array.from(this.toolCache.values()).map((e) => e.serverName)) + ); + + this.logger.debug( + `🔧 MCP tools from cache: ${Object.keys(allTools).length} total tools, ${this.toolConflicts.size} conflicts, connected servers: ${serverNames.join(', ')}` + ); + + Object.keys(allTools).forEach((toolName) => { + if (toolName.includes(MCPManager.SERVER_DELIMITER)) { + this.logger.debug(` - ${toolName} (qualified)`); + } else { + this.logger.debug(` - ${toolName}`); + } + }); + + this.logger.silly(`MCP tools: ${JSON.stringify(allTools, null, 2)}`); + return allTools; + } + + /** + * Get all MCP tools with their server metadata. + * This returns the internal tool cache entries which include server names. + * @returns Map of tool names to their cache entries (includes serverName, client, and definition) + */ + getAllToolsWithServerInfo(): Map { + return new Map(this.toolCache); + } + + /** + * Parse a qualified tool name to extract server name and actual tool name. + * Uses distinctive delimiter to avoid ambiguity and splits on last occurrence. + */ + private parseQualifiedToolName( + toolName: string + ): { serverName: string; toolName: string } | null { + const delimiterIndex = toolName.lastIndexOf(MCPManager.SERVER_DELIMITER); + if (delimiterIndex === -1) { + return null; // Not a qualified tool name + } + + const serverPrefix = toolName.substring(0, delimiterIndex); + const actualToolName = toolName.substring( + delimiterIndex + MCPManager.SERVER_DELIMITER.length + ); + + // O(1) lookup using pre-computed sanitized name map + const originalServerName = this.sanitizedNameToServerMap.get(serverPrefix); + + // Verify this qualified name exists in cache + if (originalServerName && this.toolCache.has(toolName)) { + return { serverName: originalServerName, toolName: actualToolName }; + } + + return null; + } + + /** + * Get client that provides a specific tool from the cache. + * Handles both simple tool names and server-prefixed tool names. + * @param toolName Name of the tool (may include server prefix) + * @returns The client that provides the tool, or undefined if not found + */ + getToolClient(toolName: string): IMCPClient | undefined { + // Try to get directly from cache (handles both simple and qualified names) + return this.toolCache.get(toolName)?.client; + } + + /** + * Execute a specific MCP tool with the given arguments. + * @param toolName Name of the MCP tool to execute (may include server prefix) + * @param args Arguments to pass to the tool + * @param sessionId Optional session ID + * @returns Promise resolving to the tool execution result + */ + async executeTool(toolName: string, args: any, _sessionId?: string): Promise { + const client = this.getToolClient(toolName); + if (!client) { + this.logger.error(`❌ No MCP tool found: ${toolName}`); + this.logger.debug( + `Available MCP tools: ${Array.from(this.toolCache.keys()).join(', ')}` + ); + this.logger.debug(`Conflicted tools: ${Array.from(this.toolConflicts).join(', ')}`); + throw MCPError.toolNotFound(toolName); + } + + // Extract actual tool name (remove server prefix if present) + const parsed = this.parseQualifiedToolName(toolName); + const actualToolName = parsed ? parsed.toolName : toolName; + const serverName = parsed ? parsed.serverName : 'direct'; + + this.logger.debug( + `▶️ Executing MCP tool '${actualToolName}' on server '${serverName}'...` + ); + + try { + const result = await client.callTool(actualToolName, args); + return result; + } catch (error) { + this.logger.error( + `❌ MCP tool execution failed: '${actualToolName}' - ${error instanceof Error ? error.message : String(error)}` + ); + throw error; + } + } + + /** + * Get all available prompt names from all connected clients, updating the cache. + * @returns Promise resolving to an array of unique prompt names. + */ + async listAllPrompts(): Promise { + return Array.from(this.promptCache.keys()); + } + + /** + * Get the client that provides a specific prompt from the cache. + * @param promptName Name of the prompt. + * @returns The client instance or undefined. + */ + getPromptClient(promptName: string): IMCPClient | undefined { + return this.promptCache.get(promptName)?.client; + } + + /** + * Get a specific prompt definition by name. + * @param name Name of the prompt. + * @param args Arguments for the prompt (optional). + * @returns Promise resolving to the prompt definition. + */ + async getPrompt(name: string, args?: Record): Promise { + const client = this.getPromptClient(name); + if (!client) { + throw MCPError.promptNotFound(name); + } + return await client.getPrompt(name, args); + } + + /** + * Get cached prompt metadata (no network calls). + * @param promptName Name of the prompt. + * @returns Cached prompt definition or undefined if not cached. + */ + getPromptMetadata(promptName: string): PromptDefinition | undefined { + const entry = this.promptCache.get(promptName); + return entry?.definition; + } + + /** + * Get all cached prompt metadata (no network calls). + * @returns Array of all cached prompt definitions with server info. + */ + getAllPromptMetadata(): Array<{ + promptName: string; + serverName: string; + definition: PromptDefinition; + }> { + return Array.from(this.promptCache.entries()).map(([promptName, entry]) => ({ + promptName, + serverName: entry.serverName, + definition: entry.definition, + })); + } + + /** + * Get all cached MCP resources (no network calls). + */ + async listAllResources(): Promise { + return Array.from(this.resourceCache.entries()).map(([key, { serverName, summary }]) => ({ + key, + serverName, + summary, + })); + } + + /** + * Determine if a qualified MCP resource is cached. + */ + hasResource(resourceKey: string): boolean { + return this.getResourceCacheEntry(resourceKey) !== undefined; + } + + /** + * Get cached resource metadata by qualified key. + */ + getResource(resourceKey: string): MCPResolvedResource | undefined { + const entry = this.getResourceCacheEntry(resourceKey); + if (!entry) return undefined; + return { + key: resourceKey, + serverName: entry.serverName, + summary: entry.summary, + }; + } + + /** + * Read a specific resource by qualified URI. + * @param resourceKey Qualified resource key in the form mcp:server:uri. + * @returns Promise resolving to the resource content. + */ + async readResource(resourceKey: string): Promise { + const entry = this.getResourceCacheEntry(resourceKey); + if (!entry) { + throw MCPError.resourceNotFound(resourceKey); + } + return await entry.client.readResource(entry.summary.uri); + } + + /** + * Initialize clients from server configurations + * @param serverConfigs Server configurations with individual connection modes + * @returns Promise resolving when initialization is complete + */ + async initializeFromConfig(serverConfigs: ValidatedServerConfigs): Promise { + // Handle empty server configurations gracefully + if (Object.keys(serverConfigs).length === 0) { + this.logger.info('No MCP servers configured - running without external tools'); + return; + } + + const successfulConnections: string[] = []; + const connectionPromises: Promise[] = []; + const strictServers: string[] = []; + const lenientServers: string[] = []; + + // Categorize servers by their connection mode + for (const [name, config] of Object.entries(serverConfigs)) { + // Skip disabled servers + if (config.enabled === false) { + this.logger.info(`Skipping disabled server '${name}'`); + continue; + } + + const effectiveMode = config.connectionMode || 'lenient'; + if (effectiveMode === 'strict') { + strictServers.push(name); + } else { + lenientServers.push(name); + } + + const connectPromise = this.connectServer(name, config) + .then(() => { + successfulConnections.push(name); + }) + .catch((error) => { + this.logger.debug( + `Handled connection error for '${name}' during initialization: ${error.message}` + ); + }); + connectionPromises.push(connectPromise); + } + + await Promise.all(connectionPromises); + + // Check strict servers - all must succeed + const failedStrictServers = strictServers.filter( + (name) => !successfulConnections.includes(name) + ); + if (failedStrictServers.length > 0) { + const strictErrors = failedStrictServers + .map((name) => `${name}: ${this.connectionErrors[name] || 'Unknown error'}`) + .join('; '); + throw MCPError.connectionFailed('strict servers', strictErrors); + } + + // Lenient servers are allowed to fail without throwing errors + // No additional validation needed for lenient servers + } + + /** + * Dynamically connect to a new MCP server. + * @param name The unique name for the new server connection. + * @param config The configuration for the server. + * @returns Promise resolving when the connection attempt is complete. + * @throws Error if the connection fails. + */ + async connectServer(name: string, config: ValidatedMcpServerConfig): Promise { + if (this.clients.has(name)) { + this.logger.warn(`Client '${name}' is already connected or registered.`); + return; + } + + const client = new MCPClient(this.logger); + try { + this.logger.info(`Attempting to connect to new server '${name}'...`); + await client.connect(config, name); + + // Set approval manager if available + if (this.approvalManager) { + client.setApprovalManager(this.approvalManager); + } + + this.registerClient(name, client); + await this.updateClientCache(name, client); + + // Store config for potential restart + this.configCache.set(name, config); + + this.logger.info(`Successfully connected and cached new server '${name}'`); + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + this.connectionErrors[name] = errorMsg; + this.logger.error(`Failed to connect to new server '${name}': ${errorMsg}`); + this.clients.delete(name); + throw MCPError.connectionFailed(name, errorMsg); + } + } + + /** + * Get all registered clients + * @returns Map of client names to client instances + */ + getClients(): Map { + return this.clients; + } + + /** + * Get the errors from failed connections + * @returns Map of server names to error messages + */ + getFailedConnections(): { [key: string]: string } { + return this.connectionErrors; + } + + /** + * Refresh all client caches by re-fetching capabilities from servers + * Useful when you want to force a full refresh of tools, prompts, and resources + * In normal operation, caches are automatically kept fresh via server notifications + */ + async refresh(): Promise { + this.logger.debug('Refreshing all MCPManager caches...'); + const refreshPromises: Promise[] = []; + + for (const [clientName, client] of this.clients.entries()) { + refreshPromises.push(this.updateClientCache(clientName, client)); + } + + await Promise.all(refreshPromises); + this.logger.debug( + `✅ MCPManager cache refresh complete for ${this.clients.size} client(s)` + ); + } + + /** + * Disconnect and remove a specific client by name. + * @param name The name of the client to remove. + */ + async removeClient(name: string): Promise { + const client = this.clients.get(name); + if (client) { + try { + await client.disconnect(); + this.logger.info(`Successfully disconnected client: ${name}`); + } catch (error) { + this.logger.error( + `Error disconnecting client '${name}': ${error instanceof Error ? error.message : String(error)}` + ); + // Continue with removal even if disconnection fails + } + // Clear cache BEFORE removing from clients map + this.clearClientCache(name); + this.clients.delete(name); + // Remove stored config + this.configCache.delete(name); + this.logger.info(`Removed client from manager: ${name}`); + } + // Also remove from failed connections if it was registered there before successful connection or if it failed. + if (this.connectionErrors[name]) { + delete this.connectionErrors[name]; + this.logger.info(`Cleared connection error for removed client: ${name}`); + } + } + + /** + * Restart a specific MCP server by disconnecting and reconnecting with original config. + * @param name The name of the server to restart. + * @throws Error if server doesn't exist or config is not cached. + */ + async restartServer(name: string): Promise { + // Get stored config first (this is the critical check) + const config = this.configCache.get(name); + if (!config) { + throw MCPError.serverNotFound( + name, + 'Server config not found - cannot restart dynamically added servers without stored config' + ); + } + + // Allow restart even if client is not currently registered (enables retries after failed restart) + const client = this.clients.get(name); + + this.logger.info(`Restarting MCP server '${name}'...`); + + // Disconnect existing client if one exists + if (client) { + try { + await client.disconnect(); + this.logger.info(`Disconnected server '${name}' for restart`); + } catch (error) { + this.logger.warn( + `Error disconnecting server '${name}' during restart (continuing): ${error instanceof Error ? error.message : String(error)}` + ); + } + } else { + this.logger.info( + `No active client found for '${name}' during restart; attempting fresh connection` + ); + } + + // Clear caches but keep config + this.clearClientCache(name); + this.clients.delete(name); + delete this.connectionErrors[name]; + + // Reconnect with original config + try { + const newClient = new MCPClient(this.logger); + await newClient.connect(config, name); + + // Set approval manager if available + if (this.approvalManager) { + newClient.setApprovalManager(this.approvalManager); + } + + this.registerClient(name, newClient); + await this.updateClientCache(name, newClient); + + // Config is still in cache from original connection + this.logger.info(`Successfully restarted server '${name}'`); + + // Emit event for restart + eventBus.emit('mcp:server-restarted', { serverName: name }); + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + this.connectionErrors[name] = errorMsg; + this.logger.error(`Failed to restart server '${name}': ${errorMsg}`); + // Note: Config remains in cache for potential retry + throw MCPError.connectionFailed(name, errorMsg); + } + } + + /** + * Disconnect all clients and clear caches + */ + async disconnectAll(): Promise { + const disconnectPromises: Promise[] = []; + for (const [name, client] of Array.from(this.clients.entries())) { + disconnectPromises.push( + client + .disconnect() + .then(() => this.logger.info(`Disconnected client: ${name}`)) + .catch((error) => + this.logger.error(`Failed to disconnect client '${name}': ${error}`) + ) + ); + } + await Promise.all(disconnectPromises); + + this.clients.clear(); + this.connectionErrors = {}; + this.configCache.clear(); + this.toolCache.clear(); + this.toolConflicts.clear(); + this.promptCache.clear(); + this.resourceCache.clear(); + this.sanitizedNameToServerMap.clear(); + this.logger.info('Disconnected all clients and cleared caches.'); + } + + /** + * Set up notification listeners for a specific client + */ + private setupClientNotifications(clientName: string, client: IMCPClient): void { + try { + // Listen for resource updates + client.on('resourceUpdated', async (params: { uri: string }) => { + this.logger.debug( + `Received resource update notification from ${clientName}: ${params.uri}` + ); + await this.handleResourceUpdated(clientName, params); + }); + + // Listen for prompt list changes + client.on('promptsListChanged', async () => { + this.logger.debug(`Received prompts list change notification from ${clientName}`); + await this.handlePromptsListChanged(clientName, client); + }); + + // Listen for tool list changes + client.on('toolsListChanged', async () => { + this.logger.debug(`Received tools list change notification from ${clientName}`); + await this.handleToolsListChanged(clientName, client); + }); + + this.logger.debug(`Set up notification listeners for client: ${clientName}`); + } catch (error) { + this.logger.warn(`Failed to set up notification listeners for ${clientName}: ${error}`); + } + } + + /** + * Handle resource updated notification + */ + private async handleResourceUpdated( + serverName: string, + params: { uri: string } + ): Promise { + try { + // Update the resource cache for this specific resource + const client = this.clients.get(serverName); + if (client) { + const key = this.buildQualifiedResourceKey(serverName, params.uri); + + // Try to get updated resource info + try { + const resources = await client.listResources(); + const updatedResource = resources.find((r) => r.uri === params.uri); + + if (updatedResource) { + // Update cache with new resource info + this.resourceCache.set(key, { + serverName, + client, + summary: updatedResource, + }); + this.logger.debug(`Updated resource cache for: ${params.uri}`); + } + } catch (error) { + this.logger.warn(`Failed to refresh resource ${params.uri}: ${error}`); + } + } + + // Emit event to notify other parts of the system + eventBus.emit('mcp:resource-updated', { + serverName, + resourceUri: params.uri, + }); + } catch (error) { + this.logger.error(`Error handling resource update: ${error}`); + } + } + + /** + * Handle prompts list changed notification + */ + private async handlePromptsListChanged(serverName: string, client: IMCPClient): Promise { + try { + // Refresh the prompts for this client + const existingPrompts = Array.from(this.promptCache.entries()) + .filter(([_, entry]) => entry.client === client) + .map(([promptName]) => promptName); + + // Remove old prompts from cache + existingPrompts.forEach((promptName) => { + this.promptCache.delete(promptName); + }); + + // Add new prompts with metadata from listPrompts() (no additional network calls needed) + try { + const newPrompts: Prompt[] = await client.listPrompts(); + + for (const prompt of newPrompts) { + // Convert MCP SDK Prompt to our PromptDefinition + const definition: PromptDefinition = { + name: prompt.name, + ...(prompt.title && { title: prompt.title }), + ...(prompt.description && { description: prompt.description }), + ...(prompt.arguments && { arguments: prompt.arguments }), + }; + + this.promptCache.set(prompt.name, { + serverName, + client, + definition, + }); + } + + const promptNames = newPrompts.map((p) => p.name); + this.logger.debug( + `Updated prompts cache for ${serverName}: [${promptNames.join(', ')}]` + ); + + // Emit event to notify other parts of the system + eventBus.emit('mcp:prompts-list-changed', { + serverName, + prompts: promptNames, + }); + } catch (error) { + this.logger.warn(`Failed to refresh prompts for ${serverName}: ${error}`); + } + } catch (error) { + this.logger.error(`Error handling prompts list change: ${error}`); + } + } + + /** + * Handle tools list changed notification + */ + private async handleToolsListChanged(serverName: string, client: IMCPClient): Promise { + try { + // Remove old tools for this client + const removedToolBaseNames = new Set(); + for (const [toolKey, entry] of Array.from(this.toolCache.entries())) { + if (entry.serverName === serverName) { + const delimiterIndex = toolKey.lastIndexOf(MCPManager.SERVER_DELIMITER); + const baseName = + delimiterIndex === -1 + ? toolKey + : toolKey.substring( + delimiterIndex + MCPManager.SERVER_DELIMITER.length + ); + removedToolBaseNames.add(baseName); + this.toolCache.delete(toolKey); + } + } + + // Fetch and cache new tools + try { + const tools = await client.getTools(); + const toolNames = Object.keys(tools); + + this.logger.debug( + `🔧 Refreshing tools from server '${serverName}': [${toolNames.join(', ')}]` + ); + + // Re-run conflict detection logic for each tool + for (const toolName in tools) { + const toolDef = tools[toolName]; + if (!toolDef) continue; + + // Check if this tool name already exists from a different server + const existingEntry = this.toolCache.get(toolName); + if (existingEntry && existingEntry.serverName !== serverName) { + // Conflict detected! Move existing to qualified name + this.toolConflicts.add(toolName); + this.toolCache.delete(toolName); + + const existingSanitized = this.sanitizeServerName(existingEntry.serverName); + const existingQualified = `${existingSanitized}${MCPManager.SERVER_DELIMITER}${toolName}`; + this.toolCache.set(existingQualified, existingEntry); + + // Add new tool with qualified name + const newSanitized = this.sanitizeServerName(serverName); + const newQualified = `${newSanitized}${MCPManager.SERVER_DELIMITER}${toolName}`; + this.toolCache.set(newQualified, { + serverName, + client, + definition: toolDef, + }); + + this.logger.warn( + `⚠️ Tool conflict detected for '${toolName}' - using server prefixes: ${existingQualified}, ${newQualified}` + ); + } else if (this.toolConflicts.has(toolName)) { + // This tool name is already known to be conflicted + const sanitizedName = this.sanitizeServerName(serverName); + const qualifiedName = `${sanitizedName}${MCPManager.SERVER_DELIMITER}${toolName}`; + this.toolCache.set(qualifiedName, { + serverName, + client, + definition: toolDef, + }); + this.logger.debug(`✅ Tool '${qualifiedName}' cached (known conflict)`); + } else { + // No conflict, cache with simple name + this.toolCache.set(toolName, { + serverName, + client, + definition: toolDef, + }); + this.logger.debug(`✅ Tool '${toolName}' mapped to ${serverName}`); + } + } + + // Check for resolved conflicts from removed tools + for (const baseName of removedToolBaseNames) { + const remainingTools = Array.from(this.toolCache.entries()).filter( + ([key, _]) => { + const delimiterIndex = key.lastIndexOf(MCPManager.SERVER_DELIMITER); + const bn = + delimiterIndex === -1 + ? key + : key.substring( + delimiterIndex + MCPManager.SERVER_DELIMITER.length + ); + return bn === baseName; + } + ); + + if (remainingTools.length === 0) { + this.toolConflicts.delete(baseName); + } else if (remainingTools.length === 1 && this.toolConflicts.has(baseName)) { + // Restore to simple name + const singleTool = remainingTools[0]; + if (singleTool) { + const [qualifiedKey, entry] = singleTool; + this.toolCache.delete(qualifiedKey); + this.toolCache.set(baseName, entry); + this.toolConflicts.delete(baseName); + this.logger.debug( + `Restored tool '${baseName}' to simple name (conflict resolved)` + ); + } + } + } + + this.logger.debug( + `Updated tools cache for ${serverName}: [${toolNames.join(', ')}]` + ); + + // Emit event to notify other parts of the system + eventBus.emit('mcp:tools-list-changed', { + serverName, + tools: toolNames, + }); + } catch (error) { + this.logger.warn(`Failed to refresh tools for ${serverName}: ${error}`); + } + } catch (error) { + this.logger.error(`Error handling tools list change: ${error}`); + } + } +} diff --git a/dexto/packages/core/src/mcp/mcp-client.ts b/dexto/packages/core/src/mcp/mcp-client.ts new file mode 100644 index 00000000..e43e2b15 --- /dev/null +++ b/dexto/packages/core/src/mcp/mcp-client.ts @@ -0,0 +1,743 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { EventEmitter } from 'events'; +import { z } from 'zod'; + +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import type { ApprovalManager } from '../approval/manager.js'; +import { ApprovalStatus } from '../approval/types.js'; +import type { + ValidatedMcpServerConfig, + ValidatedStdioServerConfig, + ValidatedSseServerConfig, + ValidatedHttpServerConfig, +} from './schemas.js'; +import { ToolSet } from '../tools/types.js'; +import { IMCPClient, MCPResourceSummary } from './types.js'; +import { MCPError } from './errors.js'; +import type { + GetPromptResult, + ReadResourceResult, + Resource, + ResourceUpdatedNotification, + Prompt, +} from '@modelcontextprotocol/sdk/types.js'; +import { + ResourceUpdatedNotificationSchema, + PromptListChangedNotificationSchema, + ToolListChangedNotificationSchema, +} from '@modelcontextprotocol/sdk/types.js'; +import { trace, context, SpanStatusCode, SpanKind } from '@opentelemetry/api'; +import { hasActiveTelemetry, addBaggageAttributesToSpan } from '../telemetry/utils.js'; +import { safeStringify } from '../utils/safe-stringify.js'; + +// const DEFAULT_TIMEOUT = 60000; // Commented out or remove if not used elsewhere +/** + * Wrapper on top of Client class provided in model context protocol SDK, to add additional metadata about the server + */ +export class MCPClient extends EventEmitter implements IMCPClient { + private client: Client | null = null; + private transport: any = null; + private isConnected = false; + private serverCommand: string | null = null; + private originalArgs: string[] | null = null; + private resolvedArgs: string[] | null = null; + private serverEnv: Record | null = null; + private serverSpawned = false; + private serverPid: number | null = null; + private serverAlias: string | null = null; + private timeout: number = 60000; // Default timeout value + private approvalManager: ApprovalManager | null = null; // Will be set by MCPManager + private logger: IDextoLogger; + + constructor(logger: IDextoLogger) { + super(); + this.logger = logger.createChild(DextoLogComponent.MCP); + } + + async connect(config: ValidatedMcpServerConfig, serverName: string): Promise { + this.timeout = config.timeout ?? 30000; // Use config timeout or Zod schema default + if (config.type === 'stdio') { + const stdioConfig: ValidatedStdioServerConfig = config; + + // Auto-resolve npx path on Windows + let command = stdioConfig.command; + if (process.platform === 'win32' && command === 'npx') { + command = 'C:\\Program Files\\nodejs\\npx.cmd'; + } + + return this.connectViaStdio(command, stdioConfig.args, stdioConfig.env, serverName); + } else if (config.type === 'sse') { + const sseConfig: ValidatedSseServerConfig = config; + return this.connectViaSSE(sseConfig.url, sseConfig.headers, serverName); + } else if (config.type === 'http') { + const httpConfig: ValidatedHttpServerConfig = config; + return this.connectViaHttp(httpConfig.url, httpConfig.headers || {}, serverName); + } else { + // TypeScript exhaustiveness check - should never reach here + const _exhaustive: never = config; + throw MCPError.protocolError(`Unsupported server type: ${JSON.stringify(_exhaustive)}`); + } + } + + /** + * Connect to an MCP server via stdio + * @param command Command to run + * @param args Arguments for the command + * @param env Environment variables + * @param serverAlias Optional server alias/name to show in logs + */ + async connectViaStdio( + command: string, + args: string[] = [], + env?: Record, + serverAlias?: string + ): Promise { + // Store server details + this.serverCommand = command; + this.originalArgs = [...args]; + this.resolvedArgs = [...this.originalArgs]; + this.serverEnv = env || null; + this.serverAlias = serverAlias || null; + + this.logger.info('======================================='); + this.logger.info(`MCP SERVER: ${command} ${this.resolvedArgs.join(' ')}`); + if (env) { + this.logger.info('Environment:'); + Object.entries(env).forEach(([key, _]) => { + this.logger.info(` ${key}= [value hidden]`); + }); + } + this.logger.info('=======================================\n'); + + const serverName = this.serverAlias + ? `"${this.serverAlias}" (${command} ${this.resolvedArgs.join(' ')})` + : `${command} ${this.resolvedArgs.join(' ')}`; + this.logger.info(`Connecting to MCP server: ${serverName}`); + + // Create a properly expanded environment by combining process.env with the provided env + const expandedEnv = { + ...process.env, + ...(env || {}), + }; + + // Create transport for stdio connection with expanded environment + this.transport = new StdioClientTransport({ + command: command, + args: this.resolvedArgs, + env: expandedEnv as Record, + }); + + this.client = new Client( + { + name: 'Dexto-stdio-mcp-client', + version: '1.0.0', + }, + { + capabilities: { + elicitation: {}, // Enable elicitation capability + }, + } + ); + + try { + this.logger.info('Establishing connection...'); + await this.client.connect(this.transport); + + // If connection is successful, we know the server was spawned + this.serverSpawned = true; + this.logger.info(`✅ Stdio SERVER ${serverName} SPAWNED`); + this.logger.info('Connection established!\n\n'); + this.isConnected = true; + this.setupNotificationHandlers(); + // Set up elicitation handler now that client is connected + this.setupElicitationHandler(); + + return this.client; + } catch (error: any) { + this.logger.error( + `Failed to connect to MCP server ${serverName}: ${JSON.stringify(error.message, null, 2)}` + ); + throw error; + } + } + + async connectViaSSE( + url: string, + headers: Record = {}, + serverName: string + ): Promise { + this.logger.debug(`Connecting to SSE MCP server at url: ${url}`); + + this.transport = new SSEClientTransport(new URL(url), { + // For regular HTTP requests + requestInit: { + headers: headers, + }, + // Need to implement eventSourceInit for SSE events. + }); + + // Avoid logging full transport to prevent leaking headers/tokens + this.logger.debug('[connectViaSSE] SSE transport initialized'); + this.client = new Client( + { + name: 'Dexto-sse-mcp-client', + version: '1.0.0', + }, + { + capabilities: { + elicitation: {}, // Enable elicitation capability + }, + } + ); + + try { + this.logger.info('Establishing connection...'); + await this.client.connect(this.transport); + // If connection is successful, we know the server was spawned + this.serverSpawned = true; + this.logger.info(`✅ ${serverName} SSE SERVER SPAWNED`); + this.logger.info('Connection established!\n\n'); + this.isConnected = true; + this.setupNotificationHandlers(); + // Set up elicitation handler now that client is connected + this.setupElicitationHandler(); + + return this.client; + } catch (error: any) { + this.logger.error( + `Failed to connect to SSE MCP server ${url}: ${JSON.stringify(error.message, null, 2)}` + ); + throw error; + } + } + + /** + * Connect to an MCP server via Streamable HTTP transport + */ + private async connectViaHttp( + url: string, + headers: Record = {}, + serverAlias?: string + ): Promise { + this.logger.info(`Connecting to HTTP MCP server at ${url}`); + // Ensure required Accept headers are set for Streamable HTTP transport + const defaultHeaders = { + Accept: 'application/json, text/event-stream', + }; + const mergedHeaders = { ...defaultHeaders, ...headers }; + this.transport = new StreamableHTTPClientTransport(new URL(url), { + requestInit: { headers: mergedHeaders }, + }); + this.client = new Client( + { name: 'Dexto-http-mcp-client', version: '1.0.0' }, + { + capabilities: { + elicitation: {}, // Enable elicitation capability + }, + } + ); + try { + this.logger.info('Establishing HTTP connection...'); + await this.client.connect(this.transport); + this.isConnected = true; + this.logger.info(`✅ HTTP SERVER ${serverAlias ?? url} CONNECTED`); + this.setupNotificationHandlers(); + // Set up elicitation handler now that client is connected + this.setupElicitationHandler(); + return this.client; + } catch (error: any) { + this.logger.error( + `Failed to connect to HTTP MCP server ${url}: ${JSON.stringify(error.message, null, 2)}` + ); + throw error; + } + } + + /** + * Disconnect from the server + */ + async disconnect(): Promise { + if (this.transport && typeof this.transport.close === 'function') { + try { + await this.transport.close(); + this.isConnected = false; + this.serverSpawned = false; + this.logger.info('Disconnected from MCP server'); + } catch (error: any) { + this.logger.error( + `Error disconnecting from MCP server: ${JSON.stringify(error.message, null, 2)}` + ); + } + } + } + + /** + * Call a tool with given name and arguments + * @param name Tool name + * @param args Tool arguments + * @returns Result of the tool execution + */ + async callTool(name: string, args: any): Promise { + this.ensureConnected(); + + // Only create telemetry span if telemetry is active + const shouldTrace = hasActiveTelemetry(); + const tracer = shouldTrace ? trace.getTracer('dexto') : null; + const span = tracer?.startSpan(`mcp.tool.${name}`, { + kind: SpanKind.CLIENT, + }); + + try { + // Add telemetry attributes + if (span) { + const ctx = trace.setSpan(context.active(), span); + addBaggageAttributesToSpan(span, ctx, this.logger); + span.setAttribute('tool.name', name); + span.setAttribute('tool.server', this.serverAlias || 'unknown'); + span.setAttribute('tool.timeout', this.timeout); + // Sanitize and truncate arguments for telemetry + span.setAttribute('tool.arguments', safeStringify(args, 4096)); + } + + this.logger.debug(`Calling tool '${name}' with args: ${JSON.stringify(args, null, 2)}`); + + // Parse args if it's a string (handle JSON strings) + let toolArgs = args; + if (typeof args === 'string') { + try { + toolArgs = JSON.parse(args); + } catch { + // If it's not valid JSON, keep as string + toolArgs = { input: args }; + } + } + + // Call the tool with properly formatted arguments + this.logger.debug(`Using timeout: ${this.timeout}`); + + const result = await this.client!.callTool( + { name, arguments: toolArgs }, + undefined, // resultSchema (optional) + { timeout: this.timeout } // Use server-specific timeout, default 1 minute + ); + + // Log result with base64 truncation for readability + const logResult = JSON.stringify( + result, + (key, value) => { + if (key === 'data' && typeof value === 'string' && value.length > 100) { + return `[Base64 data: ${value.length} chars]`; + } + return value; + }, + 2 + ); + this.logger.debug(`Tool '${name}' result: ${logResult}`); + + // Add result to telemetry span (sanitized and truncated) + if (span) { + span.setAttribute('tool.result', safeStringify(result, 4096)); + span.setStatus({ code: SpanStatusCode.OK }); + } + + // Check for null or undefined result + if (result === null || result === undefined) { + return 'Tool executed successfully with no result data.'; + } + return result; + } catch (error) { + this.logger.error(`Tool call '${name}' failed: ${JSON.stringify(error, null, 2)}`); + + // Record error in telemetry span + if (span) { + span.recordException(error as Error); + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : String(error), + }); + } + + return `Error executing tool '${name}': ${error instanceof Error ? error.message : String(error)}`; + } finally { + // End telemetry span + span?.end(); + } + } + + /** + * Get the list of tools provided by this client + * @returns Array of available tools + */ + async getTools(): Promise { + this.ensureConnected(); + const tools: ToolSet = {}; + try { + // Call listTools with parameters only + const listToolResult = await this.client!.listTools({}); + this.logger.silly(`listTools result: ${JSON.stringify(listToolResult, null, 2)}`); + + // Populate tools + if (listToolResult && listToolResult.tools) { + listToolResult.tools.forEach((tool: any) => { + if (!tool.description) { + this.logger.warn(`Tool '${tool.name}' is missing a description`); + } + if (!tool.inputSchema) { + throw MCPError.invalidToolSchema(tool.name, 'missing input schema'); + } + tools[tool.name] = { + description: tool.description ?? '', + parameters: tool.inputSchema, + }; + }); + } else { + throw MCPError.protocolError( + 'listTools did not return the expected structure: missing tools' + ); + } + } catch (error) { + this.logger.warn( + `Failed to get tools from MCP server, proceeding with zero tools: ${JSON.stringify(error, null, 2)}` + ); + return tools; + } + return tools; + } + + /** + * Get the list of prompts provided by this client with full metadata + * @returns Array of Prompt objects from MCP SDK with name, title, description, and arguments + */ + async listPrompts(): Promise { + this.ensureConnected(); + try { + const response = await this.client!.listPrompts(); + this.logger.debug(`listPrompts response: ${JSON.stringify(response, null, 2)}`); + return response.prompts; + } catch (error) { + this.logger.debug( + `Failed to list prompts from MCP server (optional feature), skipping: ${JSON.stringify(error, null, 2)}` + ); + return []; + } + } + + /** + * Get a specific prompt definition + * @param name Name of the prompt + * @param args Arguments for the prompt (optional) + * @returns Prompt definition (structure depends on SDK) + * TODO: Turn exception logs back into error and only call this based on capabilities of the server + */ + async getPrompt(name: string, args?: any): Promise { + this.ensureConnected(); + try { + this.logger.debug( + `Getting prompt '${name}' with args: ${JSON.stringify(args, null, 2)}` + ); + // Pass params first, then options + const response = await this.client!.getPrompt( + { name, arguments: args }, + { timeout: this.timeout } + ); + this.logger.debug(`getPrompt '${name}' response: ${JSON.stringify(response, null, 2)}`); + return response; // Return the full response object + } catch (error: any) { + this.logger.debug( + `Failed to get prompt '${name}' from MCP server: ${JSON.stringify(error, null, 2)}` + ); + throw MCPError.protocolError( + `Error getting prompt '${name}': ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + /** + * Get the list of resources provided by this client + * @returns Array of available resource URIs + * TODO: Turn exception logs back into error and only call this based on capabilities of the server + */ + async listResources(): Promise { + this.ensureConnected(); + try { + const response = await this.client!.listResources(); + this.logger.debug(`listResources response: ${JSON.stringify(response, null, 2)}`); + return response.resources.map( + (r: Resource): MCPResourceSummary => ({ + uri: r.uri, + name: r.name, + ...(r.description !== undefined && { description: r.description }), + ...(r.mimeType !== undefined && { mimeType: r.mimeType }), + }) + ); + } catch (error) { + this.logger.debug( + `Failed to list resources from MCP server (optional feature), skipping: ${JSON.stringify(error, null, 2)}` + ); + return []; + } + } + + /** + * Read the content of a specific resource + * @param uri URI of the resource + * @returns Content of the resource (structure depends on SDK) + */ + async readResource(uri: string): Promise { + this.ensureConnected(); + try { + this.logger.debug(`Reading resource '${uri}'`); + // Pass params first, then options + const response = await this.client!.readResource({ uri }, { timeout: this.timeout }); + this.logger.debug( + `readResource '${uri}' response: ${JSON.stringify(response, null, 2)}` + ); + return response; // Return the full response object + } catch (error: any) { + this.logger.debug( + `Failed to read resource '${uri}' from MCP server: ${JSON.stringify(error, null, 2)}` + ); + throw MCPError.protocolError( + `Error reading resource '${uri}': ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + /** + * Check if the client is connected + */ + getConnectionStatus(): boolean { + return this.isConnected; + } + + /** + * Get the connected client + */ + getClient(): Client | null { + return this.client; + } + + /** + * Get server status information + */ + getServerInfo(): { + spawned: boolean; + pid: number | null; + command: string | null; + originalArgs: string[] | null; + resolvedArgs: string[] | null; + env: Record | null; + alias: string | null; + } { + return { + spawned: this.serverSpawned, + pid: this.serverPid, + command: this.serverCommand, + originalArgs: this.originalArgs, + resolvedArgs: this.resolvedArgs, + env: this.serverEnv, + alias: this.serverAlias, + }; + } + + /** + * Get the client instance once connected + * @returns Promise with the MCP client + */ + async getConnectedClient(): Promise { + if (!this.client || !this.isConnected) { + throw MCPError.clientNotConnected(); + } + return this.client; + } + + private ensureConnected(): void { + if (!this.isConnected || !this.client) { + throw MCPError.clientNotConnected('Please call connect() first'); + } + } + + /** + * Set up notification handlers for MCP server notifications + */ + private setupNotificationHandlers(): void { + if (!this.client) return; + + try { + // Resource updated + this.client.setNotificationHandler( + ResourceUpdatedNotificationSchema, + (notification: ResourceUpdatedNotification) => { + // SDK notification.params has type { uri: string; _meta?: {...} } with passthrough + // Access uri directly - it's the only guaranteed field per SDK spec + this.handleResourceUpdated({ + uri: notification.params.uri, + }); + } + ); + } catch (error) { + this.logger.warn(`Could not set resources/updated notification handler: ${error}`); + } + try { + // Prompts list changed + this.client.setNotificationHandler(PromptListChangedNotificationSchema, () => { + this.handlePromptsListChanged(); + }); + } catch (error) { + this.logger.warn(`Could not set prompts/list_changed notification handler: ${error}`); + } + try { + // Tools list changed + this.client.setNotificationHandler(ToolListChangedNotificationSchema, () => { + this.handleToolsListChanged(); + }); + } catch (error) { + this.logger.warn(`Could not set tools/list_changed notification handler: ${error}`); + } + + this.logger.debug('MCP notification handlers registered (resources, prompts, tools)'); + } + + /** + * Handle resource updated notification + */ + private handleResourceUpdated(params: { uri: string }): void { + this.logger.debug(`Resource updated: ${params.uri}`); + this.emit('resourceUpdated', params); + } + + /** + * Handle prompts list changed notification + */ + private handlePromptsListChanged(): void { + this.logger.debug('Prompts list changed'); + this.emit('promptsListChanged'); + } + + /** + * Handle tools list changed notification + */ + private handleToolsListChanged(): void { + this.logger.debug('Tools list changed'); + this.emit('toolsListChanged'); + } + + /** + * Set the approval manager for handling elicitation requests + */ + setApprovalManager(approvalManager: ApprovalManager): void { + this.approvalManager = approvalManager; + // Set up handler if client is already connected + if (this.client) { + this.setupElicitationHandler(); + } + } + + /** + * Set up handler for elicitation requests from MCP server + */ + private setupElicitationHandler(): void { + if (!this.client) { + this.logger.warn('Cannot setup elicitation handler: client not initialized'); + return; + } + + if (!this.approvalManager) { + this.logger.warn('Cannot setup elicitation handler: approval manager not set'); + return; + } + + // Create the request schema for elicitation/create + const ElicitationCreateRequestSchema = z + .object({ + method: z.literal('elicitation/create'), + params: z + .object({ + message: z.string(), + requestedSchema: z.unknown(), + }) + .passthrough(), + }) + .passthrough(); + + // Set up request handler for elicitation/create + this.client.setRequestHandler(ElicitationCreateRequestSchema, async (request) => { + const params = request.params; + this.logger.info( + `Elicitation request from MCP server '${this.serverAlias}': ${params.message}` + ); + + try { + // Request elicitation through ApprovalManager + if (!this.approvalManager) { + this.logger.error('Approval manager not available for elicitation request'); + return { action: 'decline' }; + } + + // Note: MCP elicitation requests do not include sessionId + // MCP servers are shared across sessions and the MCP protocol doesn't include + // session context. Elicitations are typically for server-level data (credentials, + // config) rather than session-specific data. + + // Validate requestedSchema is an object before casting + if ( + typeof params.requestedSchema !== 'object' || + params.requestedSchema === null || + Array.isArray(params.requestedSchema) + ) { + this.logger.error( + `Invalid elicitation schema from '${this.serverAlias}': expected object, got ${typeof params.requestedSchema}` + ); + return { action: 'decline' }; + } + + const response = await this.approvalManager.requestElicitation({ + schema: params.requestedSchema as Record, + prompt: params.message, + serverName: this.serverAlias || 'unknown', + }); + + if (response.status === ApprovalStatus.APPROVED && response.data) { + // User accepted and provided data + const formData = + response.data && + typeof response.data === 'object' && + 'formData' in response.data + ? (response.data as { formData: unknown }).formData + : {}; + this.logger.info( + `Elicitation approved for '${this.serverAlias}', returning data` + ); + return { + action: 'accept', + content: formData, + }; + } else if (response.status === ApprovalStatus.DENIED) { + // User declined + this.logger.info(`Elicitation declined for '${this.serverAlias}'`); + return { + action: 'decline', + }; + } else { + // User cancelled + this.logger.info(`Elicitation cancelled for '${this.serverAlias}'`); + return { + action: 'cancel', + }; + } + } catch (error) { + this.logger.error(`Elicitation error for '${this.serverAlias}': ${error}`); + // On error, return decline + return { + action: 'decline', + }; + } + }); + + this.logger.debug(`Elicitation handler registered for MCP server '${this.serverAlias}'`); + } +} diff --git a/dexto/packages/core/src/mcp/resolver.ts b/dexto/packages/core/src/mcp/resolver.ts new file mode 100644 index 00000000..76b9630c --- /dev/null +++ b/dexto/packages/core/src/mcp/resolver.ts @@ -0,0 +1,79 @@ +// src/config/mcp/resolver.ts +import { ok, fail, type Result, hasErrors, zodToIssues } from '../utils/result.js'; +import { type Issue, ErrorScope, ErrorType } from '@core/errors/types.js'; +import { MCPErrorCode } from './error-codes.js'; + +import { + McpServerConfigSchema, + type McpServerConfig, + type ValidatedMcpServerConfig, +} from './schemas.js'; + +export type McpServerContext = { serverName?: string }; + +export function resolveAndValidateMcpServerConfig( + serverName: string, + serverConfig: McpServerConfig, + existingServerNames: string[] = [] +): Result { + const { candidate, warnings } = resolveMcpServerConfig( + serverName, + serverConfig, + existingServerNames + ); + if (hasErrors(warnings)) { + return fail(warnings); + } + return validateMcpServerConfig(candidate, warnings); +} + +function resolveMcpServerConfig( + serverName: string, + candidate: McpServerConfig, + existingServerNames: string[] +): { candidate: McpServerConfig; warnings: Issue[] } { + const warnings: Issue[] = []; + + // name sanity: keep it as a hard error if it's clearly unusable + if (typeof serverName !== 'string' || serverName.trim() === '') { + warnings.push({ + code: MCPErrorCode.SCHEMA_VALIDATION, + message: 'Server name must be a non-empty string', + severity: 'error', + scope: ErrorScope.MCP, + type: ErrorType.USER, + context: { serverName }, + }); + } + + // duplicate (case-insensitive) → warning + const dup = existingServerNames.find( + (n) => n.toLowerCase() === serverName.toLowerCase() && n !== serverName + ); + if (dup) { + warnings.push({ + code: MCPErrorCode.DUPLICATE_NAME, + message: `Server name '${serverName}' is similar to existing '${dup}' (case differs)`, + severity: 'warning', + scope: ErrorScope.MCP, + type: ErrorType.USER, + context: { serverName }, + }); + } + + return { candidate, warnings }; +} + +/** Validates MCP server by prasing it through zod schema*/ +function validateMcpServerConfig( + candidate: McpServerConfig, + warnings: Issue[] +): Result { + const parsed = McpServerConfigSchema.safeParse(candidate); + if (!parsed.success) { + return fail( + zodToIssues(parsed.error, 'error') + ); + } + return ok(parsed.data, warnings); +} diff --git a/dexto/packages/core/src/mcp/schemas.test.ts b/dexto/packages/core/src/mcp/schemas.test.ts new file mode 100644 index 00000000..90e3965f --- /dev/null +++ b/dexto/packages/core/src/mcp/schemas.test.ts @@ -0,0 +1,687 @@ +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { + StdioServerConfigSchema, + SseServerConfigSchema, + HttpServerConfigSchema, + McpServerConfigSchema, + ServerConfigsSchema, + type StdioServerConfig, + type SseServerConfig, + type HttpServerConfig, + type McpServerConfig, + type ServerConfigs, +} from './schemas.js'; +import { MCPErrorCode } from './error-codes.js'; + +describe('MCP Schemas', () => { + describe('StdioServerConfigSchema', () => { + describe('Basic Validation', () => { + it('should accept valid minimal config', () => { + const config: StdioServerConfig = { + type: 'stdio', + command: 'node', + args: ['server.js'], + }; + + const result = StdioServerConfigSchema.parse(config); + expect(result.type).toBe('stdio'); + expect((result as any).command).toBe('node'); + expect(result.args).toEqual(['server.js']); + }); + + it('should apply default values', () => { + const config: StdioServerConfig = { + type: 'stdio', + command: 'node', + }; + + const result = StdioServerConfigSchema.parse(config); + expect(result.args).toEqual([]); // default + expect(result.env).toEqual({}); // default + expect(result.timeout).toBe(30000); // default + expect(result.connectionMode).toBe('lenient'); // default + }); + + it('should accept all optional fields', () => { + const config: StdioServerConfig = { + type: 'stdio', + command: 'python', + args: ['-m', 'my_server'], + env: { PYTHONPATH: '/custom/path', DEBUG: '1' }, + timeout: 45000, + connectionMode: 'strict', + }; + + const result = StdioServerConfigSchema.parse(config); + expect(result.args).toEqual(['-m', 'my_server']); + expect(result.env).toEqual({ PYTHONPATH: '/custom/path', DEBUG: '1' }); + expect(result.timeout).toBe(45000); + expect(result.connectionMode).toBe('strict'); + }); + }); + + describe('Field Validation', () => { + it('should require command field', () => { + const config = { + type: 'stdio', + args: ['server.js'], + }; + const result = StdioServerConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['command']); + }); + + it('should reject empty command', () => { + const config = { + type: 'stdio', + command: '', + }; + const result = StdioServerConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.custom); + expect((result.error?.issues[0] as any)?.params?.code).toBe( + MCPErrorCode.COMMAND_MISSING + ); + }); + + it('should validate connectionMode values', () => { + const validModes = ['strict', 'lenient']; + + for (const connectionMode of validModes) { + const config = { + type: 'stdio', + command: 'node', + connectionMode, + }; + const result = StdioServerConfigSchema.safeParse(config); + expect(result.success).toBe(true); + } + + const invalidConfig = { + type: 'stdio', + command: 'node', + connectionMode: 'invalid', + }; + const result = StdioServerConfigSchema.safeParse(invalidConfig); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_enum_value); + expect(result.error?.issues[0]?.path).toEqual(['connectionMode']); + }); + + it('should validate field types', () => { + // Command must be string + const invalidCommand = { type: 'stdio', command: 123 }; + let result = StdioServerConfigSchema.safeParse(invalidCommand); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['command']); + + // Args must be array + const invalidArgs = { type: 'stdio', command: 'node', args: 'not-array' }; + result = StdioServerConfigSchema.safeParse(invalidArgs); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['args']); + + // Env must be object + const invalidEnv = { type: 'stdio', command: 'node', env: ['not-object'] }; + result = StdioServerConfigSchema.safeParse(invalidEnv); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['env']); + + // Timeout must be positive number + const invalidTimeout = { type: 'stdio', command: 'node', timeout: 'fast' }; + result = StdioServerConfigSchema.safeParse(invalidTimeout); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['timeout']); + }); + }); + + describe('Strict Validation', () => { + it('should reject unknown fields', () => { + const config = { + type: 'stdio', + command: 'node', + unknownField: 'should-fail', + }; + const result = StdioServerConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + }); + + describe('SseServerConfigSchema', () => { + describe('Basic Validation', () => { + it('should accept valid minimal config', () => { + const config: SseServerConfig = { + type: 'sse', + url: 'http://localhost:8080/events', + }; + + const result = SseServerConfigSchema.parse(config); + expect(result.type).toBe('sse'); + expect(result.url).toBe('http://localhost:8080/events'); + }); + + it('should apply default values', () => { + const config: SseServerConfig = { + type: 'sse', + url: 'https://api.example.com/sse', + }; + + const result = SseServerConfigSchema.parse(config); + expect(result.headers).toEqual({}); // default + expect(result.timeout).toBe(30000); // default + expect(result.connectionMode).toBe('lenient'); // default + }); + + it('should accept all optional fields', () => { + const config: SseServerConfig = { + type: 'sse', + url: 'https://api.example.com/events', + headers: { + Authorization: 'Bearer token123', + 'X-API-Key': 'key456', + }, + timeout: 60000, + connectionMode: 'strict', + }; + + const result = SseServerConfigSchema.parse(config); + expect(result.headers).toEqual({ + Authorization: 'Bearer token123', + 'X-API-Key': 'key456', + }); + expect(result.timeout).toBe(60000); + expect(result.connectionMode).toBe('strict'); + }); + }); + + describe('Field Validation', () => { + it('should require url field', () => { + const config = { + type: 'sse', + headers: {}, + }; + const result = SseServerConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['url']); + }); + + it('should validate URL format', () => { + const validUrls = [ + 'http://localhost:8080/events', + 'https://api.example.com/sse', + 'http://127.0.0.1:3000/stream', + ]; + + for (const url of validUrls) { + const config = { type: 'sse', url }; + const result = SseServerConfigSchema.safeParse(config); + expect(result.success).toBe(true); + } + + const invalidConfig = { + type: 'sse', + url: 'not-a-valid-url', + }; + const result = SseServerConfigSchema.safeParse(invalidConfig); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.custom); + expect(result.error?.issues[0]?.path).toEqual(['url']); + }); + + it('should validate connectionMode values', () => { + const validModes = ['strict', 'lenient']; + + for (const connectionMode of validModes) { + const config = { + type: 'sse', + url: 'http://localhost:8080/events', + connectionMode, + }; + const result = SseServerConfigSchema.safeParse(config); + expect(result.success).toBe(true); + } + + const invalidConfig = { + type: 'sse', + url: 'http://localhost:8080/events', + connectionMode: 'invalid', + }; + const result = SseServerConfigSchema.safeParse(invalidConfig); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_enum_value); + expect(result.error?.issues[0]?.path).toEqual(['connectionMode']); + }); + + it('should validate field types', () => { + // URL must be string + const invalidUrl = { type: 'sse', url: 12345 }; + let result = SseServerConfigSchema.safeParse(invalidUrl); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['url']); + + // Headers must be object + const invalidHeaders = { + type: 'sse', + url: 'http://localhost', + headers: 'not-object', + }; + result = SseServerConfigSchema.safeParse(invalidHeaders); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['headers']); + + // Timeout must be positive number + const invalidTimeout = { type: 'sse', url: 'http://localhost', timeout: 'slow' }; + result = SseServerConfigSchema.safeParse(invalidTimeout); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['timeout']); + }); + }); + + describe('Strict Validation', () => { + it('should reject unknown fields', () => { + const config = { + type: 'sse', + url: 'http://localhost:8080/events', + unknownField: 'should-fail', + }; + const result = SseServerConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + }); + + describe('HttpServerConfigSchema', () => { + describe('Basic Validation', () => { + it('should accept valid minimal config', () => { + const config: HttpServerConfig = { + type: 'http', + url: 'http://localhost:9000/api', + }; + + const result = HttpServerConfigSchema.parse(config); + expect(result.type).toBe('http'); + expect(result.url).toBe('http://localhost:9000/api'); + }); + + it('should apply default values', () => { + const config: HttpServerConfig = { + type: 'http', + url: 'https://api.example.com/mcp', + }; + + const result = HttpServerConfigSchema.parse(config); + expect(result.connectionMode).toBe('lenient'); // default + }); + + it('should accept all optional fields', () => { + const config: HttpServerConfig = { + type: 'http', + url: 'https://api.example.com/mcp', + headers: { + Authorization: 'Bearer token789', + 'Content-Type': 'application/json', + }, + timeout: 25000, + connectionMode: 'strict', + }; + + const result = HttpServerConfigSchema.parse(config); + expect(result.headers).toEqual({ + Authorization: 'Bearer token789', + 'Content-Type': 'application/json', + }); + expect(result.timeout).toBe(25000); + expect(result.connectionMode).toBe('strict'); + }); + }); + + describe('Field Validation', () => { + it('should require url field', () => { + const config = { + type: 'http', + headers: {}, + }; + const result = HttpServerConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['url']); + }); + + it('should validate URL format', () => { + const validUrls = [ + 'http://localhost:9000/api', + 'https://api.example.com/mcp', + 'http://127.0.0.1:4000/webhook', + ]; + + for (const url of validUrls) { + const config = { type: 'http', url }; + const result = HttpServerConfigSchema.safeParse(config); + expect(result.success).toBe(true); + } + + const invalidConfig = { + type: 'http', + url: 'invalid-url-format', + }; + const result = HttpServerConfigSchema.safeParse(invalidConfig); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.custom); + expect(result.error?.issues[0]?.path).toEqual(['url']); + }); + + it('should validate connectionMode values', () => { + const validModes = ['strict', 'lenient']; + + for (const connectionMode of validModes) { + const config = { + type: 'http', + url: 'http://localhost:9000/api', + connectionMode, + }; + const result = HttpServerConfigSchema.safeParse(config); + expect(result.success).toBe(true); + } + + const invalidConfig = { + type: 'http', + url: 'http://localhost:9000/api', + connectionMode: 'invalid', + }; + const result = HttpServerConfigSchema.safeParse(invalidConfig); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_enum_value); + expect(result.error?.issues[0]?.path).toEqual(['connectionMode']); + }); + + it('should validate field types', () => { + // URL must be valid URL format + const invalidUrl = { type: 'http', url: 'not-a-url' }; + let result = HttpServerConfigSchema.safeParse(invalidUrl); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.custom); + expect(result.error?.issues[0]?.path).toEqual(['url']); + + // Headers must be object + const invalidHeaders = { + type: 'http', + url: 'http://localhost', + headers: 'not-object', + }; + result = HttpServerConfigSchema.safeParse(invalidHeaders); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['headers']); + + // Timeout must be positive number + const invalidTimeout = { type: 'http', url: 'http://localhost', timeout: false }; + result = HttpServerConfigSchema.safeParse(invalidTimeout); + expect(result.success).toBe(false); + // weird behaviour, but this is how zod works for coerce + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['timeout']); + }); + }); + + describe('Strict Validation', () => { + it('should reject unknown fields', () => { + const config = { + type: 'http', + url: 'http://localhost:9000/api', + unknownField: 'should-fail', + }; + const result = HttpServerConfigSchema.safeParse(config); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + }); + + describe('McpServerConfigSchema (Discriminated Union)', () => { + describe('Type Discrimination', () => { + it('should accept valid stdio server config', () => { + const config: McpServerConfig = { + type: 'stdio', + command: 'node', + args: ['mcp-server.js'], + }; + + const result = McpServerConfigSchema.parse(config); + expect(result.type).toBe('stdio'); + expect((result as StdioServerConfig).command).toBe('node'); + }); + + it('should accept valid sse server config', () => { + const config: McpServerConfig = { + type: 'sse', + url: 'http://localhost:8080/sse', + }; + + const result = McpServerConfigSchema.parse(config); + expect(result.type).toBe('sse'); + expect((result as SseServerConfig).url).toBe('http://localhost:8080/sse'); + }); + + it('should accept valid http server config', () => { + const config: McpServerConfig = { + type: 'http', + url: 'http://localhost:9000/mcp', + }; + + const result = McpServerConfigSchema.parse(config); + expect(result.type).toBe('http'); + expect((result as HttpServerConfig).url).toBe('http://localhost:9000/mcp'); + }); + }); + + describe('Error Handling', () => { + it('should reject config without type field', () => { + const config = { + command: 'node', + args: ['server.js'], + }; + expect(() => McpServerConfigSchema.parse(config)).toThrow(); + }); + + it('should reject config with invalid type', () => { + const config = { + type: 'invalid-type', // Invalid type + url: 'ws://localhost:8080', + }; + expect(() => McpServerConfigSchema.parse(config)).toThrow(); + }); + + it('should reject config with valid type but wrong fields', () => { + const config = { + type: 'stdio', + url: 'http://localhost', // Wrong field for stdio + }; + expect(() => McpServerConfigSchema.parse(config)).toThrow(); + }); + }); + }); + + describe('ServerConfigsSchema', () => { + describe('Basic Validation', () => { + it('should accept empty server configs', () => { + const configs: ServerConfigs = {}; + const result = ServerConfigsSchema.parse(configs); + expect(result).toEqual({}); + }); + + it('should accept single server config', () => { + const configs: ServerConfigs = { + myServer: { + type: 'stdio', + command: 'node', + args: ['server.js'], + }, + }; + + const result = ServerConfigsSchema.parse(configs); + expect(result.myServer!.type).toBe('stdio'); + expect((result.myServer! as any).command).toBe('node'); + }); + + it('should accept multiple mixed server configs', () => { + const configs: ServerConfigs = { + stdioServer: { + type: 'stdio', + command: 'python', + args: ['-m', 'my_server'], + env: { DEBUG: '1' }, + }, + sseServer: { + type: 'sse', + url: 'http://localhost:8080/events', + headers: { Authorization: 'Bearer token' }, + }, + httpServer: { + type: 'http', + url: 'https://api.example.com/mcp', + timeout: 20000, + }, + }; + + const result = ServerConfigsSchema.parse(configs); + expect(Object.keys(result)).toHaveLength(3); + expect(result.stdioServer!.type).toBe('stdio'); + expect(result.sseServer!.type).toBe('sse'); + expect(result.httpServer!.type).toBe('http'); + }); + }); + + describe('Validation Propagation', () => { + it('should validate each server config individually', () => { + const configs = { + validServer: { + type: 'stdio', + command: 'node', + args: ['server.js'], + }, + invalidServer: { + type: 'stdio', + // Missing required command field + args: ['server.js'], + }, + }; + + expect(() => ServerConfigsSchema.parse(configs)).toThrow(); + }); + + it('should reject configs with invalid server types', () => { + const configs = { + server1: { + type: 'stdio', + command: 'node', + }, + server2: { + type: 'invalid-type', // Invalid type + command: 'python', + }, + }; + + expect(() => ServerConfigsSchema.parse(configs)).toThrow(); + }); + }); + + describe('Real-world Scenarios', () => { + it('should handle typical development setup', () => { + const devConfig = { + fileSystem: { + type: 'stdio', + command: 'npx', + args: ['@modelcontextprotocol/server-filesystem', '/path/to/project'], + env: { NODE_ENV: 'development' }, + timeout: 10000, + }, + database: { + type: 'sse', + url: 'http://localhost:8080/db-events', + headers: { 'X-API-Key': 'dev-key-123' }, + connectionMode: 'lenient', + }, + }; + + const result = ServerConfigsSchema.parse(devConfig); + expect(result.fileSystem!.type).toBe('stdio'); + expect(result.database!.type).toBe('sse'); + }); + + it('should handle production setup', () => { + const prodConfig = { + analytics: { + type: 'http', + url: 'https://analytics.company.com/mcp/endpoint', + headers: { + Authorization: 'Bearer prod-token-xyz', + 'X-Service': 'dexto-agent', + }, + timeout: 30000, + connectionMode: 'strict', + }, + monitoring: { + type: 'sse', + url: 'https://monitoring.company.com/stream', + headers: { 'X-Monitor-Key': 'monitor-key-789' }, + connectionMode: 'strict', + }, + }; + + const result = ServerConfigsSchema.parse(prodConfig); + expect(result.analytics!.connectionMode).toBe('strict'); + expect(result.monitoring!.connectionMode).toBe('strict'); + }); + }); + }); + + describe('Type Safety', () => { + it('should maintain proper type inference for stdio config', () => { + const config = { + type: 'stdio' as const, + command: 'node', + args: ['server.js'], + }; + + const result = StdioServerConfigSchema.parse(config); + + // TypeScript should infer correct types + expect(typeof result.type).toBe('string'); + expect(typeof result.command).toBe('string'); + expect(Array.isArray(result.args)).toBe(true); + expect(typeof result.env).toBe('object'); + }); + + it('should maintain proper type inference for server configs', () => { + const configs = { + server1: { + type: 'stdio' as const, + command: 'node', + }, + server2: { + type: 'sse' as const, + url: 'http://localhost:8080/events', + }, + }; + + const result = ServerConfigsSchema.parse(configs); + + // Should preserve discriminated union types + expect(result.server1!.type).toBe('stdio'); + expect(result.server2!.type).toBe('sse'); + }); + }); +}); diff --git a/dexto/packages/core/src/mcp/schemas.ts b/dexto/packages/core/src/mcp/schemas.ts new file mode 100644 index 00000000..eb72e78b --- /dev/null +++ b/dexto/packages/core/src/mcp/schemas.ts @@ -0,0 +1,125 @@ +import { MCPErrorCode } from './error-codes.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { EnvExpandedString, RequiredEnvURL } from '@core/utils/result.js'; +import { z } from 'zod'; + +export const MCP_SERVER_TYPES = ['stdio', 'sse', 'http'] as const; +export type McpServerType = (typeof MCP_SERVER_TYPES)[number]; + +export const MCP_CONNECTION_MODES = ['strict', 'lenient'] as const; +export type McpConnectionMode = (typeof MCP_CONNECTION_MODES)[number]; + +export const MCP_CONNECTION_STATUSES = ['connected', 'disconnected', 'error'] as const; +export type McpConnectionStatus = (typeof MCP_CONNECTION_STATUSES)[number]; + +/** + * MCP server info with computed connection status. + * Returned by DextoAgent.getMcpServersWithStatus() + */ +export interface McpServerStatus { + name: string; + type: McpServerType; + enabled: boolean; + status: McpConnectionStatus; + error?: string; +} + +export const DEFAULT_MCP_CONNECTION_MODE: McpConnectionMode = 'lenient'; + +// ---- stdio ---- + +export const StdioServerConfigSchema = z + .object({ + type: z.literal('stdio'), + enabled: z + .boolean() + .default(true) + .describe('Whether this server is enabled (disabled servers are not connected)'), + // allow env in command & args if you want; remove EnvExpandedString if not desired + command: EnvExpandedString().superRefine((s, ctx) => { + if (s.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Stdio server requires a non-empty command', + params: { + code: MCPErrorCode.COMMAND_MISSING, + scope: ErrorScope.MCP, + type: ErrorType.USER, + }, + }); + } + }), + args: z + .array(EnvExpandedString()) + .default([]) + .describe("Array of arguments for the command (e.g., ['script.js'])"), + env: z + .record(EnvExpandedString()) + .default({}) + .describe('Optional environment variables for the server process'), + timeout: z.coerce.number().int().positive().default(30000), + connectionMode: z.enum(MCP_CONNECTION_MODES).default(DEFAULT_MCP_CONNECTION_MODE), + }) + .strict(); + +export type StdioServerConfig = z.input; +export type ValidatedStdioServerConfig = z.output; +// ---- sse ---- + +export const SseServerConfigSchema = z + .object({ + type: z.literal('sse'), + enabled: z + .boolean() + .default(true) + .describe('Whether this server is enabled (disabled servers are not connected)'), + url: RequiredEnvURL(process.env).describe('URL for the SSE server endpoint'), + headers: z.record(EnvExpandedString()).default({}), + timeout: z.coerce.number().int().positive().default(30000), + connectionMode: z.enum(MCP_CONNECTION_MODES).default(DEFAULT_MCP_CONNECTION_MODE), + }) + .strict(); + +export type SseServerConfig = z.input; +export type ValidatedSseServerConfig = z.output; +// ---- http ---- + +export const HttpServerConfigSchema = z + .object({ + type: z.literal('http'), + enabled: z + .boolean() + .default(true) + .describe('Whether this server is enabled (disabled servers are not connected)'), + url: RequiredEnvURL(process.env).describe('URL for the HTTP server'), + headers: z.record(EnvExpandedString()).default({}), + timeout: z.coerce.number().int().positive().default(30000), + connectionMode: z.enum(MCP_CONNECTION_MODES).default(DEFAULT_MCP_CONNECTION_MODE), + }) + .strict(); + +export type HttpServerConfig = z.input; +export type ValidatedHttpServerConfig = z.output; +// ---- discriminated union ---- + +export const McpServerConfigSchema = z + .discriminatedUnion('type', [ + StdioServerConfigSchema, + SseServerConfigSchema, + HttpServerConfigSchema, + ]) + .superRefine((_data, _ctx) => { + // cross-type business rules if you ever need them + }) + .brand<'ValidatedMcpServerConfig'>(); + +export type McpServerConfig = z.input; +export type ValidatedMcpServerConfig = z.output; + +export const ServerConfigsSchema = z + .record(McpServerConfigSchema) + .describe('A dictionary of server configurations, keyed by server name') + .brand<'ValidatedServerConfigs'>(); + +export type ServerConfigs = z.input; +export type ValidatedServerConfigs = z.output; diff --git a/dexto/packages/core/src/mcp/types.ts b/dexto/packages/core/src/mcp/types.ts new file mode 100644 index 00000000..28ab5572 --- /dev/null +++ b/dexto/packages/core/src/mcp/types.ts @@ -0,0 +1,38 @@ +import { ValidatedMcpServerConfig } from './schemas.js'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import type { ToolProvider } from '../tools/types.js'; +import { GetPromptResult, ReadResourceResult, Prompt } from '@modelcontextprotocol/sdk/types.js'; +import { EventEmitter } from 'events'; + +export interface MCPResourceSummary { + uri: string; + name?: string; + description?: string; + mimeType?: string; +} + +export interface MCPResolvedResource { + key: string; + serverName: string; + summary: MCPResourceSummary; +} + +/** + * Interface for MCP clients specifically, that can provide tools + */ +export interface IMCPClient extends ToolProvider, EventEmitter { + // Connection Management + connect(config: ValidatedMcpServerConfig, serverName: string): Promise; + disconnect(): Promise; + + // Prompt Management + listPrompts(): Promise; + getPrompt(name: string, args?: Record): Promise; + + // Resource Management + listResources(): Promise; + readResource(uri: string): Promise; + + // MCP Client Management + getConnectedClient(): Promise; +} diff --git a/dexto/packages/core/src/memory/error-codes.ts b/dexto/packages/core/src/memory/error-codes.ts new file mode 100644 index 00000000..c6119be1 --- /dev/null +++ b/dexto/packages/core/src/memory/error-codes.ts @@ -0,0 +1,19 @@ +/** + * Error codes for memory operations + */ +export enum MemoryErrorCode { + // General errors + MEMORY_NOT_FOUND = 'MEMORY_NOT_FOUND', + MEMORY_ALREADY_EXISTS = 'MEMORY_ALREADY_EXISTS', + + // Validation errors + MEMORY_CONTENT_REQUIRED = 'MEMORY_CONTENT_REQUIRED', + MEMORY_CONTENT_TOO_LONG = 'MEMORY_CONTENT_TOO_LONG', + MEMORY_INVALID_ID = 'MEMORY_INVALID_ID', + MEMORY_INVALID_TAGS = 'MEMORY_INVALID_TAGS', + + // Storage errors + MEMORY_STORAGE_ERROR = 'MEMORY_STORAGE_ERROR', + MEMORY_RETRIEVAL_ERROR = 'MEMORY_RETRIEVAL_ERROR', + MEMORY_DELETE_ERROR = 'MEMORY_DELETE_ERROR', +} diff --git a/dexto/packages/core/src/memory/errors.ts b/dexto/packages/core/src/memory/errors.ts new file mode 100644 index 00000000..14f78973 --- /dev/null +++ b/dexto/packages/core/src/memory/errors.ts @@ -0,0 +1,97 @@ +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import { MemoryErrorCode } from './error-codes.js'; + +/** + * Memory error factory following the error pattern from config/errors.ts + */ +export class MemoryError { + static notFound(id: string): DextoRuntimeError { + return new DextoRuntimeError( + MemoryErrorCode.MEMORY_NOT_FOUND, + ErrorScope.MEMORY, + ErrorType.NOT_FOUND, + `Memory not found: ${id}`, + { id } + ); + } + + static alreadyExists(id: string): DextoRuntimeError { + return new DextoRuntimeError( + MemoryErrorCode.MEMORY_ALREADY_EXISTS, + ErrorScope.MEMORY, + ErrorType.USER, + `Memory already exists: ${id}`, + { id } + ); + } + + static contentRequired(): DextoRuntimeError { + return new DextoRuntimeError( + MemoryErrorCode.MEMORY_CONTENT_REQUIRED, + ErrorScope.MEMORY, + ErrorType.USER, + 'Memory content is required' + ); + } + + static contentTooLong(length: number, maxLength: number): DextoRuntimeError { + return new DextoRuntimeError( + MemoryErrorCode.MEMORY_CONTENT_TOO_LONG, + ErrorScope.MEMORY, + ErrorType.USER, + `Memory content too long: ${length} characters (max: ${maxLength})`, + { length, maxLength } + ); + } + + static invalidId(id: string): DextoRuntimeError { + return new DextoRuntimeError( + MemoryErrorCode.MEMORY_INVALID_ID, + ErrorScope.MEMORY, + ErrorType.USER, + `Invalid memory ID: ${id}`, + { id } + ); + } + + static invalidTags(tags: unknown): DextoRuntimeError { + return new DextoRuntimeError( + MemoryErrorCode.MEMORY_INVALID_TAGS, + ErrorScope.MEMORY, + ErrorType.USER, + `Invalid tags format: ${JSON.stringify(tags)}`, + { tags } + ); + } + + static storageError(message: string, cause?: Error): DextoRuntimeError { + return new DextoRuntimeError( + MemoryErrorCode.MEMORY_STORAGE_ERROR, + ErrorScope.MEMORY, + ErrorType.SYSTEM, + `Memory storage error: ${message}`, + { cause } + ); + } + + static retrievalError(message: string, cause?: Error): DextoRuntimeError { + return new DextoRuntimeError( + MemoryErrorCode.MEMORY_RETRIEVAL_ERROR, + ErrorScope.MEMORY, + ErrorType.SYSTEM, + `Memory retrieval error: ${message}`, + { cause } + ); + } + + static deleteError(message: string, cause?: Error): DextoRuntimeError { + return new DextoRuntimeError( + MemoryErrorCode.MEMORY_DELETE_ERROR, + ErrorScope.MEMORY, + ErrorType.SYSTEM, + `Memory deletion error: ${message}`, + { cause } + ); + } +} diff --git a/dexto/packages/core/src/memory/index.ts b/dexto/packages/core/src/memory/index.ts new file mode 100644 index 00000000..9d7650d5 --- /dev/null +++ b/dexto/packages/core/src/memory/index.ts @@ -0,0 +1,23 @@ +export { MemoryManager } from './manager.js'; +export type { + Memory, + CreateMemoryInput, + UpdateMemoryInput, + ListMemoriesOptions, + MemorySource, +} from './types.js'; +export { + MemorySchema, + CreateMemoryInputSchema, + UpdateMemoryInputSchema, + ListMemoriesOptionsSchema, + MemoriesConfigSchema, + type ValidatedMemory, + type ValidatedCreateMemoryInput, + type ValidatedUpdateMemoryInput, + type ValidatedListMemoriesOptions, + type MemoriesConfig, + type ValidatedMemoriesConfig, +} from './schemas.js'; +export { MemoryError } from './errors.js'; +export { MemoryErrorCode } from './error-codes.js'; diff --git a/dexto/packages/core/src/memory/manager.integration.test.ts b/dexto/packages/core/src/memory/manager.integration.test.ts new file mode 100644 index 00000000..2c7d2231 --- /dev/null +++ b/dexto/packages/core/src/memory/manager.integration.test.ts @@ -0,0 +1,206 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { MemoryManager } from './manager.js'; +// Import from index to ensure providers are registered +import { createDatabase } from '../storage/database/index.js'; +import type { Database } from '../storage/database/types.js'; +import type { CreateMemoryInput } from './types.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; + +describe('MemoryManager Integration Tests', () => { + let memoryManager: MemoryManager; + let database: Database; + const mockLogger = createMockLogger(); + + beforeEach(async () => { + // Use in-memory database for integration tests + database = await createDatabase({ type: 'in-memory' }, mockLogger); + await database.connect(); + memoryManager = new MemoryManager(database, mockLogger); + }); + + afterEach(async () => { + await database.disconnect(); + }); + + it('should create, retrieve, update, and delete a memory', async () => { + // Create + const input: CreateMemoryInput = { + content: 'Integration test memory', + tags: ['test', 'integration'], + metadata: { source: 'user' }, + }; + + const created = await memoryManager.create(input); + expect(created.id).toBeDefined(); + expect(created.content).toBe('Integration test memory'); + + // Retrieve + const retrieved = await memoryManager.get(created.id); + expect(retrieved).toEqual(created); + + // Update + const updated = await memoryManager.update(created.id, { + content: 'Updated content', + tags: ['updated'], + }); + expect(updated.content).toBe('Updated content'); + expect(updated.tags).toEqual(['updated']); + expect(updated.updatedAt).toBeGreaterThanOrEqual(created.updatedAt); + + // Delete + await memoryManager.delete(created.id); + await expect(memoryManager.get(created.id)).rejects.toThrow('Memory not found'); + }); + + it('should list and filter memories correctly', async () => { + // Create multiple memories + const memories = await Promise.all([ + memoryManager.create({ + content: 'Work memory', + tags: ['work', 'important'], + }), + memoryManager.create({ + content: 'Personal memory', + tags: ['personal'], + }), + memoryManager.create({ + content: 'Another work memory', + tags: ['work'], + }), + ]); + + // List all + const allMemories = await memoryManager.list(); + expect(allMemories).toHaveLength(3); + + // Filter by tag + const workMemories = await memoryManager.list({ tags: ['work'] }); + expect(workMemories).toHaveLength(2); + + // Count + const count = await memoryManager.count({ tags: ['work'] }); + expect(count).toBe(2); + + // Limit + const limited = await memoryManager.list({ limit: 2 }); + expect(limited).toHaveLength(2); + + // Cleanup + for (const memory of memories) { + await memoryManager.delete(memory.id); + } + }); + + it('should handle metadata correctly across operations', async () => { + const input: CreateMemoryInput = { + content: 'Memory with metadata', + metadata: { + source: 'user', + customField: 'custom value', + }, + }; + + const created = await memoryManager.create(input); + expect(created.metadata).toMatchObject({ + source: 'user', + customField: 'custom value', + }); + + // Update with additional metadata + const updated = await memoryManager.update(created.id, { + metadata: { + pinned: true, + anotherField: 'another value', + }, + }); + + // Should merge with existing metadata + expect(updated.metadata).toMatchObject({ + source: 'user', + customField: 'custom value', + pinned: true, + anotherField: 'another value', + }); + + await memoryManager.delete(created.id); + }); + + it('should maintain sort order (most recently updated first)', async () => { + const mem1 = await memoryManager.create({ content: 'First' }); + // Small delay to ensure different timestamps + await new Promise((resolve) => setTimeout(resolve, 10)); + const mem2 = await memoryManager.create({ content: 'Second' }); + await new Promise((resolve) => setTimeout(resolve, 10)); + const mem3 = await memoryManager.create({ content: 'Third' }); + + const list = await memoryManager.list(); + + // Should be in reverse chronological order (newest first) + expect(list[0]!.id).toBe(mem3.id); + expect(list[1]!.id).toBe(mem2.id); + expect(list[2]!.id).toBe(mem1.id); + + // Update the first one + await new Promise((resolve) => setTimeout(resolve, 10)); + await memoryManager.update(mem1.id, { content: 'First Updated' }); + + const updatedList = await memoryManager.list(); + + // Now mem1 should be first (most recently updated) + expect(updatedList[0]!.id).toBe(mem1.id); + + // Cleanup + await memoryManager.delete(mem1.id); + await memoryManager.delete(mem2.id); + await memoryManager.delete(mem3.id); + }); + + it('should handle edge cases gracefully', async () => { + // Create memory with minimal data + const minimal = await memoryManager.create({ content: 'Minimal' }); + expect(minimal.tags).toBeUndefined(); + expect(minimal.metadata).toBeUndefined(); + + // Update with empty updates (should not throw) + const updated = await memoryManager.update(minimal.id, {}); + expect(updated.content).toBe('Minimal'); + + // Has method + expect(await memoryManager.has(minimal.id)).toBe(true); + expect(await memoryManager.has('non-existent-id')).toBe(false); + + // List with no results + await memoryManager.delete(minimal.id); + const empty = await memoryManager.list(); + expect(empty).toHaveLength(0); + }); + + it('should validate memory content length', async () => { + // Content too long (>10000 characters) + const longContent = 'a'.repeat(10001); + + await expect(memoryManager.create({ content: longContent })).rejects.toThrow(); + }); + + it('should validate tags', async () => { + // Too many tags (>10) + const manyTags = Array.from({ length: 11 }, (_, i) => `tag${i}`); + + await expect( + memoryManager.create({ + content: 'Test', + tags: manyTags, + }) + ).rejects.toThrow(); + + // Tag too long (>50 characters) + const longTag = 'a'.repeat(51); + + await expect( + memoryManager.create({ + content: 'Test', + tags: [longTag], + }) + ).rejects.toThrow(); + }); +}); diff --git a/dexto/packages/core/src/memory/manager.test.ts b/dexto/packages/core/src/memory/manager.test.ts new file mode 100644 index 00000000..dcb913e1 --- /dev/null +++ b/dexto/packages/core/src/memory/manager.test.ts @@ -0,0 +1,349 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { MemoryManager } from './manager.js'; +import type { Database } from '../storage/database/types.js'; +import type { CreateMemoryInput, UpdateMemoryInput } from './types.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; + +describe('MemoryManager', () => { + let memoryManager: MemoryManager; + let mockDatabase: Database; + const mockLogger = createMockLogger(); + + beforeEach(() => { + // Create a mock database + mockDatabase = { + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + list: vi.fn(), + append: vi.fn(), + getRange: vi.fn(), + connect: vi.fn(), + disconnect: vi.fn(), + isConnected: vi.fn(), + getStoreType: vi.fn(), + }; + + memoryManager = new MemoryManager(mockDatabase, mockLogger); + }); + + describe('create', () => { + it('should create a memory with valid input', async () => { + const input: CreateMemoryInput = { + content: 'Test memory content', + tags: ['work', 'important'], + metadata: { source: 'user' }, + }; + + vi.mocked(mockDatabase.set).mockResolvedValue(undefined); + + const memory = await memoryManager.create(input); + + expect(memory).toMatchObject({ + content: 'Test memory content', + tags: ['work', 'important'], + }); + expect(memory.id).toBeDefined(); + expect(memory.createdAt).toBeDefined(); + expect(memory.updatedAt).toBeDefined(); + expect(memory.metadata?.source).toBe('user'); + expect(mockDatabase.set).toHaveBeenCalledWith( + expect.stringContaining('memory:item:'), + expect.objectContaining({ content: 'Test memory content' }) + ); + }); + + it('should create a memory without optional fields', async () => { + const input: CreateMemoryInput = { + content: 'Simple memory', + }; + + vi.mocked(mockDatabase.set).mockResolvedValue(undefined); + + const memory = await memoryManager.create(input); + + expect(memory.content).toBe('Simple memory'); + expect(memory.tags).toBeUndefined(); + expect(memory.metadata).toBeUndefined(); + }); + + it('should throw validation error for empty content', async () => { + const input = { + content: '', + } as CreateMemoryInput; + + await expect(memoryManager.create(input)).rejects.toThrow(); + }); + + it('should throw storage error if database fails', async () => { + const input: CreateMemoryInput = { + content: 'Test memory', + }; + + vi.mocked(mockDatabase.set).mockRejectedValue(new Error('Database error')); + + await expect(memoryManager.create(input)).rejects.toThrow('Memory storage error'); + }); + }); + + describe('get', () => { + it('should retrieve an existing memory', async () => { + const mockMemory = { + id: 'test-id', + content: 'Test memory', + createdAt: Date.now(), + updatedAt: Date.now(), + }; + + vi.mocked(mockDatabase.get).mockResolvedValue(mockMemory); + + const memory = await memoryManager.get('test-id'); + + expect(memory).toEqual(mockMemory); + expect(mockDatabase.get).toHaveBeenCalledWith('memory:item:test-id'); + }); + + it('should throw not found error for non-existent memory', async () => { + vi.mocked(mockDatabase.get).mockResolvedValue(undefined); + + await expect(memoryManager.get('non-existent')).rejects.toThrow('Memory not found'); + }); + + it('should throw error for invalid ID', async () => { + await expect(memoryManager.get('')).rejects.toThrow('Invalid memory ID'); + }); + }); + + describe('update', () => { + it('should update memory content', async () => { + const existingMemory = { + id: 'test-id', + content: 'Original content', + createdAt: Date.now() - 1000, + updatedAt: Date.now() - 1000, + }; + + const updates: UpdateMemoryInput = { + content: 'Updated content', + }; + + vi.mocked(mockDatabase.get).mockResolvedValue(existingMemory); + vi.mocked(mockDatabase.set).mockResolvedValue(undefined); + + const updatedMemory = await memoryManager.update('test-id', updates); + + expect(updatedMemory.content).toBe('Updated content'); + expect(updatedMemory.updatedAt).toBeGreaterThan(existingMemory.updatedAt); + expect(updatedMemory.createdAt).toBe(existingMemory.createdAt); + }); + + it('should update memory tags', async () => { + const existingMemory = { + id: 'test-id', + content: 'Test content', + createdAt: Date.now(), + updatedAt: Date.now(), + tags: ['old-tag'], + }; + + const updates: UpdateMemoryInput = { + tags: ['new-tag', 'another-tag'], + }; + + vi.mocked(mockDatabase.get).mockResolvedValue(existingMemory); + vi.mocked(mockDatabase.set).mockResolvedValue(undefined); + + const updatedMemory = await memoryManager.update('test-id', updates); + + expect(updatedMemory.tags).toEqual(['new-tag', 'another-tag']); + }); + + it('should merge metadata on update', async () => { + const existingMemory = { + id: 'test-id', + content: 'Test content', + createdAt: Date.now(), + updatedAt: Date.now(), + metadata: { + source: 'user' as const, + customField: 'value', + }, + }; + + const updates: UpdateMemoryInput = { + metadata: { + pinned: true, + }, + }; + + vi.mocked(mockDatabase.get).mockResolvedValue(existingMemory); + vi.mocked(mockDatabase.set).mockResolvedValue(undefined); + + const updatedMemory = await memoryManager.update('test-id', updates); + + expect(updatedMemory.metadata).toMatchObject({ + source: 'user', + customField: 'value', + pinned: true, + }); + }); + }); + + describe('delete', () => { + it('should delete an existing memory', async () => { + const existingMemory = { + id: 'test-id', + content: 'Test content', + createdAt: Date.now(), + updatedAt: Date.now(), + }; + + vi.mocked(mockDatabase.get).mockResolvedValue(existingMemory); + vi.mocked(mockDatabase.delete).mockResolvedValue(undefined); + + await memoryManager.delete('test-id'); + + expect(mockDatabase.delete).toHaveBeenCalledWith('memory:item:test-id'); + }); + + it('should throw error if memory does not exist', async () => { + vi.mocked(mockDatabase.get).mockResolvedValue(undefined); + + await expect(memoryManager.delete('non-existent')).rejects.toThrow('Memory not found'); + }); + }); + + describe('list', () => { + it('should list all memories', async () => { + const mockMemories = [ + { + id: 'mem-1', + content: 'Memory 1', + createdAt: Date.now() - 2000, + updatedAt: Date.now() - 2000, + }, + { + id: 'mem-2', + content: 'Memory 2', + createdAt: Date.now() - 1000, + updatedAt: Date.now() - 1000, + }, + ]; + + vi.mocked(mockDatabase.list).mockResolvedValue([ + 'memory:item:mem-1', + 'memory:item:mem-2', + ]); + vi.mocked(mockDatabase.get) + .mockResolvedValueOnce(mockMemories[0]) + .mockResolvedValueOnce(mockMemories[1]); + + const memories = await memoryManager.list(); + + expect(memories).toHaveLength(2); + // Should be sorted by updatedAt descending (most recent first) + expect(memories[0]!.id).toBe('mem-2'); + expect(memories[1]!.id).toBe('mem-1'); + }); + + it('should filter memories by tags', async () => { + const mockMemories = [ + { + id: 'mem-1', + content: 'Memory 1', + createdAt: Date.now(), + updatedAt: Date.now(), + tags: ['work', 'important'], + }, + { + id: 'mem-2', + content: 'Memory 2', + createdAt: Date.now(), + updatedAt: Date.now(), + tags: ['personal'], + }, + ]; + + vi.mocked(mockDatabase.list).mockResolvedValue([ + 'memory:item:mem-1', + 'memory:item:mem-2', + ]); + vi.mocked(mockDatabase.get) + .mockResolvedValueOnce(mockMemories[0]) + .mockResolvedValueOnce(mockMemories[1]); + + const memories = await memoryManager.list({ tags: ['work'] }); + + expect(memories).toHaveLength(1); + expect(memories[0]!.id).toBe('mem-1'); + }); + + it('should apply limit and offset', async () => { + const mockMemories = Array.from({ length: 5 }, (_, i) => ({ + id: `mem-${i}`, + content: `Memory ${i}`, + createdAt: Date.now() - (5 - i) * 1000, + updatedAt: Date.now() - (5 - i) * 1000, + })); + + vi.mocked(mockDatabase.list).mockResolvedValue( + mockMemories.map((m) => `memory:item:${m.id}`) + ); + mockMemories.forEach((mem) => { + vi.mocked(mockDatabase.get).mockResolvedValueOnce(mem); + }); + + const memories = await memoryManager.list({ limit: 2, offset: 1 }); + + expect(memories).toHaveLength(2); + // After sorting by updatedAt desc and applying offset 1, limit 2 + expect(memories[0]!.id).toBe('mem-3'); + expect(memories[1]!.id).toBe('mem-2'); + }); + }); + + describe('has', () => { + it('should return true for existing memory', async () => { + vi.mocked(mockDatabase.get).mockResolvedValue({ + id: 'test-id', + content: 'Test', + createdAt: Date.now(), + updatedAt: Date.now(), + }); + + const exists = await memoryManager.has('test-id'); + + expect(exists).toBe(true); + }); + + it('should return false for non-existent memory', async () => { + vi.mocked(mockDatabase.get).mockResolvedValue(undefined); + + const exists = await memoryManager.has('non-existent'); + + expect(exists).toBe(false); + }); + }); + + describe('count', () => { + it('should return total count of memories', async () => { + const mockMemories = Array.from({ length: 3 }, (_, i) => ({ + id: `mem-${i}`, + content: `Memory ${i}`, + createdAt: Date.now(), + updatedAt: Date.now(), + })); + + vi.mocked(mockDatabase.list).mockResolvedValue( + mockMemories.map((m) => `memory:item:${m.id}`) + ); + mockMemories.forEach((mem) => { + vi.mocked(mockDatabase.get).mockResolvedValueOnce(mem); + }); + + const count = await memoryManager.count(); + + expect(count).toBe(3); + }); + }); +}); diff --git a/dexto/packages/core/src/memory/manager.ts b/dexto/packages/core/src/memory/manager.ts new file mode 100644 index 00000000..c7616413 --- /dev/null +++ b/dexto/packages/core/src/memory/manager.ts @@ -0,0 +1,275 @@ +import type { Database } from '../storage/database/types.js'; +import type { Memory, CreateMemoryInput, UpdateMemoryInput, ListMemoriesOptions } from './types.js'; +import { + MemorySchema, + CreateMemoryInputSchema, + UpdateMemoryInputSchema, + ListMemoriesOptionsSchema, +} from './schemas.js'; +import { MemoryError } from './errors.js'; +import { MemoryErrorCode } from './error-codes.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import { nanoid } from 'nanoid'; + +const MEMORY_KEY_PREFIX = 'memory:item:'; + +/** + * MemoryManager handles CRUD operations for user memories + * + * Responsibilities: + * - Store and retrieve memories from the database + * - Validate memory data + * - Generate unique IDs for memories + * - Filter and search memories + * + * Storage format: + * - Key: `memory:item:{id}` + * - Value: Memory object + * + * TODO: Expand to support multi-scope memories (user, agent, entity, session) + * with namespaced keys (e.g., `memory:user:{userId}:item:{id}`) and + * context-aware retrieval. + */ +export class MemoryManager { + private logger: IDextoLogger; + + constructor( + private database: Database, + logger: IDextoLogger + ) { + this.logger = logger.createChild(DextoLogComponent.MEMORY); + this.logger.debug('MemoryManager initialized'); + } + + /** + * Create a new memory + */ + async create(input: CreateMemoryInput): Promise { + // Validate input + const validatedInput = CreateMemoryInputSchema.parse(input); + + // Generate unique ID + const id = nanoid(12); + + const now = Date.now(); + const memory: Memory = { + id, + content: validatedInput.content, + createdAt: now, + updatedAt: now, + tags: validatedInput.tags, + metadata: validatedInput.metadata, + }; + + // Validate the complete memory object + const validatedMemory = MemorySchema.parse(memory); + + try { + // Store in database + await this.database.set(this.toKey(id), validatedMemory); + this.logger.info(`Created memory: ${id}`); + return validatedMemory; + } catch (error) { + throw MemoryError.storageError( + `Failed to store memory: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error : undefined + ); + } + } + + /** + * Get a memory by ID + */ + async get(id: string): Promise { + if (!id || typeof id !== 'string') { + throw MemoryError.invalidId(id); + } + + try { + const memory = await this.database.get(this.toKey(id)); + if (!memory) { + throw MemoryError.notFound(id); + } + return memory; + } catch (error) { + if ( + error instanceof DextoRuntimeError && + error.code === MemoryErrorCode.MEMORY_NOT_FOUND + ) { + throw error; + } + throw MemoryError.retrievalError( + `Failed to retrieve memory: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error : undefined + ); + } + } + + /** + * Update an existing memory + */ + async update(id: string, input: UpdateMemoryInput): Promise { + if (!id || typeof id !== 'string') { + throw MemoryError.invalidId(id); + } + + // Validate input + const validatedInput = UpdateMemoryInputSchema.parse(input); + + // Get existing memory + const existing = await this.get(id); + + // Merge updates + const updated: Memory = { + ...existing, + content: + validatedInput.content !== undefined ? validatedInput.content : existing.content, + tags: validatedInput.tags !== undefined ? validatedInput.tags : existing.tags, + updatedAt: Date.now(), + }; + + // Merge metadata if provided + if (validatedInput.metadata) { + updated.metadata = { + ...(existing.metadata || {}), + ...validatedInput.metadata, + }; + } + + // Validate the updated memory + const validatedMemory = MemorySchema.parse(updated); + + try { + await this.database.set(this.toKey(id), validatedMemory); + this.logger.info(`Updated memory: ${id}`); + return validatedMemory; + } catch (error) { + throw MemoryError.storageError( + `Failed to update memory: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error : undefined + ); + } + } + + /** + * Delete a memory by ID + */ + async delete(id: string): Promise { + if (!id || typeof id !== 'string') { + throw MemoryError.invalidId(id); + } + + // Verify memory exists before deleting + await this.get(id); + + try { + await this.database.delete(this.toKey(id)); + this.logger.info(`Deleted memory: ${id}`); + } catch (error) { + throw MemoryError.deleteError( + `Failed to delete memory: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error : undefined + ); + } + } + + /** + * List all memories with optional filtering + */ + async list(options: ListMemoriesOptions = {}): Promise { + // Validate and parse options + const validatedOptions = ListMemoriesOptionsSchema.parse(options); + + try { + // Get all memory keys + const keys = await this.database.list(MEMORY_KEY_PREFIX); + + // Retrieve all memories + const memories: Memory[] = []; + for (const key of keys) { + try { + const memory = await this.database.get(key); + if (memory) { + memories.push(memory); + } + } catch (error) { + this.logger.warn( + `Failed to retrieve memory from key ${key}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + // Apply filters + let filtered = memories; + + // Filter by tags + if (validatedOptions.tags && validatedOptions.tags.length > 0) { + filtered = filtered.filter((m) => + m.tags?.some((tag) => validatedOptions.tags!.includes(tag)) + ); + } + + // Filter by source + if (validatedOptions.source) { + filtered = filtered.filter((m) => m.metadata?.source === validatedOptions.source); + } + + // Filter by pinned status + if (validatedOptions.pinned !== undefined) { + filtered = filtered.filter((m) => m.metadata?.pinned === validatedOptions.pinned); + } + + // Sort by updatedAt descending (most recent first) + filtered.sort((a, b) => b.updatedAt - a.updatedAt); + + // Apply pagination + if (validatedOptions.offset !== undefined || validatedOptions.limit !== undefined) { + const start = validatedOptions.offset ?? 0; + const end = validatedOptions.limit ? start + validatedOptions.limit : undefined; + filtered = filtered.slice(start, end); + } + + return filtered; + } catch (error) { + throw MemoryError.retrievalError( + `Failed to list memories: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error : undefined + ); + } + } + + /** + * Check if a memory exists + */ + async has(id: string): Promise { + try { + await this.get(id); + return true; + } catch (error) { + if ( + error instanceof DextoRuntimeError && + error.code === MemoryErrorCode.MEMORY_NOT_FOUND + ) { + return false; + } + throw error; + } + } + + /** + * Get count of total memories + */ + async count(options: ListMemoriesOptions = {}): Promise { + const memories = await this.list(options); + return memories.length; + } + + /** + * Convert memory ID to database key + */ + private toKey(id: string): string { + return `${MEMORY_KEY_PREFIX}${id}`; + } +} diff --git a/dexto/packages/core/src/memory/schemas.ts b/dexto/packages/core/src/memory/schemas.ts new file mode 100644 index 00000000..dde086dc --- /dev/null +++ b/dexto/packages/core/src/memory/schemas.ts @@ -0,0 +1,139 @@ +import { z } from 'zod'; + +/** + * Memory schemas following the Zod best practices from CLAUDE.md + */ + +const MAX_CONTENT_LENGTH = 10000; // 10k characters max per memory +const MAX_TAG_LENGTH = 50; +const MAX_TAGS = 10; + +export const MemorySourceSchema = z.enum(['user', 'system']).describe('Source of the memory'); + +export const MemoryMetadataSchema = z + .object({ + source: MemorySourceSchema.optional().describe('Source of the memory'), + pinned: z.boolean().optional().describe('Whether this memory is pinned for auto-loading'), + }) + .passthrough() // Allow additional custom fields + .describe('Memory metadata'); + +export const MemorySchema = z + .object({ + id: z.string().min(1).describe('Unique identifier for the memory'), + content: z + .string() + .min(1, 'Memory content cannot be empty') + .max( + MAX_CONTENT_LENGTH, + `Memory content cannot exceed ${MAX_CONTENT_LENGTH} characters` + ) + .describe('The actual memory content'), + createdAt: z.number().int().positive().describe('Creation timestamp (Unix ms)'), + updatedAt: z.number().int().positive().describe('Last update timestamp (Unix ms)'), + tags: z + .array(z.string().min(1).max(MAX_TAG_LENGTH)) + .max(MAX_TAGS) + .optional() + .describe('Optional tags for categorization'), + metadata: MemoryMetadataSchema.optional().describe('Additional metadata'), + }) + .strict() + .describe('Memory item stored in the system'); + +export const CreateMemoryInputSchema = z + .object({ + content: z + .string() + .min(1, 'Memory content cannot be empty') + .max( + MAX_CONTENT_LENGTH, + `Memory content cannot exceed ${MAX_CONTENT_LENGTH} characters` + ) + .describe('The memory content'), + tags: z + .array(z.string().min(1).max(MAX_TAG_LENGTH)) + .max(MAX_TAGS) + .optional() + .describe('Optional tags'), + metadata: MemoryMetadataSchema.optional().describe('Optional metadata'), + }) + .strict() + .describe('Input for creating a new memory'); + +export const UpdateMemoryInputSchema = z + .object({ + content: z + .string() + .min(1, 'Memory content cannot be empty') + .max( + MAX_CONTENT_LENGTH, + `Memory content cannot exceed ${MAX_CONTENT_LENGTH} characters` + ) + .optional() + .describe('Updated content'), + tags: z + .array(z.string().min(1).max(MAX_TAG_LENGTH)) + .max(MAX_TAGS) + .optional() + .describe('Updated tags (replaces existing)'), + metadata: MemoryMetadataSchema.optional().describe( + 'Updated metadata (merges with existing)' + ), + }) + .strict() + .describe('Input for updating an existing memory'); + +export const ListMemoriesOptionsSchema = z + .object({ + tags: z.array(z.string()).optional().describe('Filter by tags'), + source: MemorySourceSchema.optional().describe('Filter by source'), + pinned: z.boolean().optional().describe('Filter by pinned status'), + limit: z.number().int().positive().optional().describe('Limit number of results'), + offset: z.number().int().nonnegative().optional().describe('Skip first N results'), + }) + .strict() + .describe('Options for listing memories'); + +/** + * Configuration schema for memory inclusion in system prompts. + * This is a top-level agent config field that controls how memories + * are injected into the system prompt. + */ +export const MemoriesConfigSchema = z + .object({ + enabled: z + .boolean() + .default(false) + .describe('Whether to include memories in system prompt (optional'), + priority: z + .number() + .int() + .nonnegative() + .default(40) + .describe('Priority in system prompt (lower = earlier)'), + includeTimestamps: z + .boolean() + .default(false) + .describe('Whether to include timestamps in memory display'), + includeTags: z + .boolean() + .default(true) + .describe('Whether to include tags in memory display'), + limit: z + .number() + .int() + .positive() + .optional() + .describe('Maximum number of memories to include'), + pinnedOnly: z.boolean().default(false).describe('Only include pinned memories'), + }) + .strict() + .describe('Memory configuration for system prompt inclusion'); + +export type ValidatedMemory = z.output; +export type ValidatedCreateMemoryInput = z.output; +export type ValidatedUpdateMemoryInput = z.output; +export type ValidatedListMemoriesOptions = z.output; +export type MemoriesConfig = z.input; +export type ValidatedMemoriesConfig = z.output; diff --git a/dexto/packages/core/src/memory/types.ts b/dexto/packages/core/src/memory/types.ts new file mode 100644 index 00000000..94fc4e78 --- /dev/null +++ b/dexto/packages/core/src/memory/types.ts @@ -0,0 +1,82 @@ +/** + * Core memory types and interfaces + */ + +/** + * Supported memory sources + */ +export type MemorySource = 'user' | 'system'; + +/** + * Memory item stored in the system + */ +export interface Memory { + /** Unique identifier for the memory */ + id: string; + /** The actual memory content */ + content: string; + /** When the memory was created (Unix timestamp in milliseconds) */ + createdAt: number; + /** When the memory was last updated (Unix timestamp in milliseconds) */ + updatedAt: number; + /** Optional tags for categorization */ + tags?: string[] | undefined; + /** Additional metadata */ + metadata?: + | { + /** Source of the memory */ + source?: MemorySource | undefined; + /** Whether this memory is pinned (for future hybrid approach) */ + pinned?: boolean | undefined; + /** Any additional custom metadata */ + [key: string]: unknown; + } + | undefined; +} + +/** + * Input for creating a new memory + */ +export interface CreateMemoryInput { + /** The memory content */ + content: string; + /** Optional tags */ + tags?: string[]; + /** Optional metadata */ + metadata?: { + source?: MemorySource; + [key: string]: unknown; + }; +} + +/** + * Input for updating an existing memory + */ +export interface UpdateMemoryInput { + /** Updated content (optional) */ + content?: string; + /** Updated tags (optional, replaces existing) */ + tags?: string[]; + /** Updated metadata (optional, merges with existing) */ + metadata?: { + source?: MemorySource; + pinned?: boolean; + [key: string]: unknown; + }; +} + +/** + * Options for listing memories + */ +export interface ListMemoriesOptions { + /** Filter by tags */ + tags?: string[]; + /** Filter by source */ + source?: MemorySource; + /** Filter by pinned status */ + pinned?: boolean; + /** Limit number of results */ + limit?: number; + /** Skip first N results */ + offset?: number; +} diff --git a/dexto/packages/core/src/plugins/builtins/content-policy.ts b/dexto/packages/core/src/plugins/builtins/content-policy.ts new file mode 100644 index 00000000..08ba55bf --- /dev/null +++ b/dexto/packages/core/src/plugins/builtins/content-policy.ts @@ -0,0 +1,134 @@ +import type { + DextoPlugin, + PluginResult, + PluginNotice, + BeforeLLMRequestPayload, + PluginExecutionContext, +} from '../types.js'; + +/** + * Configuration options for the ContentPolicy plugin + */ +export interface ContentPolicyConfig { + maxInputChars?: number; + redactEmails?: boolean; + redactApiKeys?: boolean; +} + +const DEFAULTS: Required = { + maxInputChars: 0, + redactEmails: false, + redactApiKeys: false, +}; + +const ABUSIVE_PATTERNS: RegExp[] = [ + /\b(?:fuck|shit|bitch|asshole|bastard|cunt|dick|fag|slut|whore|nigger|retard|motherfucker|cock|piss|twat|wank|prick|spastic|chink|gook|kike|spic|wetback|tranny|dyke|homo|queer|faggot|rape|rapist)\b/i, +]; + +function containsAbusiveLanguage(text: string): boolean { + return ABUSIVE_PATTERNS.some((pattern) => pattern.test(text)); +} + +/** + * ContentPolicy Plugin + * + * Enforces content policies on LLM requests including: + * - Abusive language detection (blocking) + * - Input length limits + * - Email address redaction + * - API key redaction + * + * Ported from feat/hooks content-policy hook implementation + */ +export class ContentPolicyPlugin implements DextoPlugin { + private config: Required = DEFAULTS; + + async initialize(config: Record): Promise { + this.config = { + maxInputChars: config.maxInputChars ?? DEFAULTS.maxInputChars, + redactEmails: config.redactEmails ?? DEFAULTS.redactEmails, + redactApiKeys: config.redactApiKeys ?? DEFAULTS.redactApiKeys, + }; + } + + async beforeLLMRequest( + payload: BeforeLLMRequestPayload, + _context: PluginExecutionContext + ): Promise { + const notices: PluginNotice[] = []; + const { text } = payload; + + // Check for abusive language (blocking) + if (containsAbusiveLanguage(text)) { + const abusiveNotice: PluginNotice = { + kind: 'block', + code: 'content_policy.abusive_language', + message: 'Input violates content policy due to abusive language.', + }; + notices.push(abusiveNotice); + return { + ok: false, + cancel: true, + message: abusiveNotice.message, + notices, + }; + } + + let modified = text; + + // Apply input length limit + if (this.config.maxInputChars > 0 && modified.length > this.config.maxInputChars) { + modified = modified.slice(0, this.config.maxInputChars); + notices.push({ + kind: 'warn', + code: 'content_policy.truncated', + message: `Input truncated to ${this.config.maxInputChars} characters to meet policy limits.`, + details: { originalLength: text.length }, + }); + } + + // Redact email addresses + if (this.config.redactEmails) { + const replaced = modified.replace( + /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, + '[redacted-email]' + ); + if (replaced !== modified) { + notices.push({ + kind: 'info', + code: 'content_policy.redact_email', + message: 'Email addresses were redacted from the input.', + }); + modified = replaced; + } + } + + // Redact API keys + if (this.config.redactApiKeys) { + const replaced = modified.replace( + /(api_key|apikey|bearer)\s*[:=]?\s*([A-Za-z0-9_-]{12,})/gi, + '$1 [redacted]' + ); + if (replaced !== modified) { + notices.push({ + kind: 'info', + code: 'content_policy.redact_api_key', + message: 'Potential API keys were redacted from the input.', + }); + modified = replaced; + } + } + + // Return result with modifications if any + if (modified !== text || notices.length > 0) { + return { + ok: true, + modify: { text: modified }, + notices, + }; + } + + // No changes needed + return { ok: true }; + } +} diff --git a/dexto/packages/core/src/plugins/builtins/response-sanitizer.ts b/dexto/packages/core/src/plugins/builtins/response-sanitizer.ts new file mode 100644 index 00000000..c2c06b64 --- /dev/null +++ b/dexto/packages/core/src/plugins/builtins/response-sanitizer.ts @@ -0,0 +1,120 @@ +import type { + DextoPlugin, + BeforeResponsePayload, + PluginExecutionContext, + PluginResult, + PluginNotice, +} from '../types.js'; + +export interface ResponseSanitizerConfig { + redactEmails?: boolean; + redactApiKeys?: boolean; + maxResponseLength?: number; +} + +const DEFAULTS: Required = { + redactEmails: false, + redactApiKeys: false, + maxResponseLength: 0, +}; + +/** + * Response sanitizer built-in plugin + * + * This plugin redacts sensitive information from LLM responses to prevent accidental leakage: + * - Email addresses + * - API keys and tokens + * - Optional: Truncates responses that exceed length limits + * + * This demonstrates how plugins can modify response content before it's sent to users, + * using the beforeResponse extension point. + */ +export class ResponseSanitizerPlugin implements DextoPlugin { + private redactEmails: boolean = DEFAULTS.redactEmails; + private redactApiKeys: boolean = DEFAULTS.redactApiKeys; + private maxResponseLength: number = DEFAULTS.maxResponseLength; + + async initialize(config: Record): Promise { + const sanitizerConfig = config as ResponseSanitizerConfig; + this.redactEmails = sanitizerConfig.redactEmails ?? DEFAULTS.redactEmails; + this.redactApiKeys = sanitizerConfig.redactApiKeys ?? DEFAULTS.redactApiKeys; + this.maxResponseLength = sanitizerConfig.maxResponseLength ?? DEFAULTS.maxResponseLength; + } + + async beforeResponse( + payload: BeforeResponsePayload, + _context: PluginExecutionContext + ): Promise { + const notices: PluginNotice[] = []; + let modified = payload.content; + + // Redact email addresses + if (this.redactEmails) { + const replaced = modified.replace( + /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, + '[redacted-email]' + ); + if (replaced !== modified) { + notices.push({ + kind: 'info', + code: 'response_sanitizer.redact_email', + message: 'Email addresses were redacted from the response.', + }); + modified = replaced; + } + } + + // Redact potential API keys and tokens + if (this.redactApiKeys) { + // Pattern matches common API key formats + const replaced = modified.replace( + /(api[_-]?key|apikey|token|bearer|secret)\s*[:=]?\s*['"]?([A-Za-z0-9_-]{12,})['"]?/gi, + '$1 [redacted]' + ); + if (replaced !== modified) { + notices.push({ + kind: 'info', + code: 'response_sanitizer.redact_api_key', + message: 'Potential API keys were redacted from the response.', + }); + modified = replaced; + } + } + + // Truncate responses that exceed max length + if (this.maxResponseLength > 0 && modified.length > this.maxResponseLength) { + const originalLength = modified.length; + const suffix = '... [truncated]'; + + if (this.maxResponseLength <= suffix.length) { + // If maxResponseLength is too small, use truncated suffix + modified = suffix.slice(0, this.maxResponseLength); + } else { + // Calculate adjusted length to account for suffix + const adjustedLength = this.maxResponseLength - suffix.length; + modified = modified.slice(0, adjustedLength) + suffix; + } + + notices.push({ + kind: 'warn', + code: 'response_sanitizer.truncated', + message: `Response truncated to ${this.maxResponseLength} characters.`, + details: { originalLength }, + }); + } + + // Return modifications if any were made + if (modified !== payload.content) { + const result: PluginResult = { + ok: true, + modify: { content: modified }, + }; + if (notices.length > 0) { + result.notices = notices; + } + return result; + } + + return { ok: true }; + } +} diff --git a/dexto/packages/core/src/plugins/error-codes.ts b/dexto/packages/core/src/plugins/error-codes.ts new file mode 100644 index 00000000..d74d1b6a --- /dev/null +++ b/dexto/packages/core/src/plugins/error-codes.ts @@ -0,0 +1,46 @@ +/** + * Plugin-specific error codes + * Used for plugin loading, validation, and execution errors + */ +export enum PluginErrorCode { + /** Plugin file not found or cannot be loaded */ + PLUGIN_LOAD_FAILED = 'PLUGIN_LOAD_FAILED', + + /** Plugin does not implement required interface */ + PLUGIN_INVALID_SHAPE = 'PLUGIN_INVALID_SHAPE', + + /** Plugin constructor threw an error */ + PLUGIN_INSTANTIATION_FAILED = 'PLUGIN_INSTANTIATION_FAILED', + + /** Plugin initialization failed */ + PLUGIN_INITIALIZATION_FAILED = 'PLUGIN_INITIALIZATION_FAILED', + + /** Plugin configuration is invalid */ + PLUGIN_CONFIGURATION_INVALID = 'PLUGIN_CONFIGURATION_INVALID', + + /** Plugin execution failed */ + PLUGIN_EXECUTION_FAILED = 'PLUGIN_EXECUTION_FAILED', + + /** Plugin execution timed out */ + PLUGIN_EXECUTION_TIMEOUT = 'PLUGIN_EXECUTION_TIMEOUT', + + /** Plugin blocked execution */ + PLUGIN_BLOCKED_EXECUTION = 'PLUGIN_BLOCKED_EXECUTION', + + /** Duplicate plugin priority */ + PLUGIN_DUPLICATE_PRIORITY = 'PLUGIN_DUPLICATE_PRIORITY', + + /** Required dependency not installed for plugin loading */ + PLUGIN_DEPENDENCY_NOT_INSTALLED = 'PLUGIN_DEPENDENCY_NOT_INSTALLED', + + /** Plugin provider already registered in registry */ + PLUGIN_PROVIDER_ALREADY_REGISTERED = 'PLUGIN_PROVIDER_ALREADY_REGISTERED', + + /** Plugin provider not found in registry */ + PLUGIN_PROVIDER_NOT_FOUND = 'PLUGIN_PROVIDER_NOT_FOUND', + + /** Plugin provider configuration validation failed */ + PLUGIN_PROVIDER_VALIDATION_FAILED = 'PLUGIN_PROVIDER_VALIDATION_FAILED', +} + +export type { PluginErrorCode as default }; diff --git a/dexto/packages/core/src/plugins/index.ts b/dexto/packages/core/src/plugins/index.ts new file mode 100644 index 00000000..019ce266 --- /dev/null +++ b/dexto/packages/core/src/plugins/index.ts @@ -0,0 +1,52 @@ +/** + * Plugin System + * + * Unified plugin architecture for extending agent behavior at key extension points. + * Replaces the hooks system from PR #385 with a more flexible plugin model. + */ + +// Core types for plugin development +export type { + DextoPlugin, + PluginConfig, + PluginExecutionContext, + PluginResult, + PluginNotice, + ExtensionPoint, + BeforeLLMRequestPayload, + BeforeToolCallPayload, + AfterToolResultPayload, + BeforeResponsePayload, +} from './types.js'; + +// Plugin manager for service integration +export { PluginManager } from './manager.js'; +export type { PluginManagerOptions, ExecutionContextOptions } from './manager.js'; + +// Plugin configuration schemas +export { + CustomPluginConfigSchema, + BuiltInPluginConfigSchema, + PluginsConfigSchema, + RegistryPluginConfigSchema, +} from './schemas.js'; +export type { PluginsConfig, ValidatedPluginsConfig, RegistryPluginConfig } from './schemas.js'; + +// Plugin registry for programmatic plugin registration +export { PluginRegistry, pluginRegistry } from './registry.js'; +export type { PluginProvider, PluginCreationContext } from './registry.js'; + +// Error codes +export { PluginErrorCode } from './error-codes.js'; + +// Plugin utilities for advanced use cases +export { loadPluginModule, resolvePluginPath, validatePluginShape } from './loader.js'; + +// Built-in plugin registry (for extending with custom built-ins) +export { registerBuiltInPlugins } from './registrations/builtins.js'; + +// Built-in plugins +export { ContentPolicyPlugin } from './builtins/content-policy.js'; +export type { ContentPolicyConfig } from './builtins/content-policy.js'; +export { ResponseSanitizerPlugin } from './builtins/response-sanitizer.js'; +export type { ResponseSanitizerConfig } from './builtins/response-sanitizer.js'; diff --git a/dexto/packages/core/src/plugins/loader.ts b/dexto/packages/core/src/plugins/loader.ts new file mode 100644 index 00000000..517505d2 --- /dev/null +++ b/dexto/packages/core/src/plugins/loader.ts @@ -0,0 +1,212 @@ +import { isAbsolute } from 'path'; +import { pathToFileURL } from 'url'; +import { DextoRuntimeError, ErrorScope, ErrorType } from '../errors/index.js'; +import { PluginErrorCode } from './error-codes.js'; + +/** + * Validate that a loaded plugin implements the DextoPlugin interface correctly + * Performs runtime checks that complement TypeScript's compile-time type checking + * + * @param PluginClass - The plugin class constructor + * @param pluginName - Name for error messages + * @throws {DextoRuntimeError} If validation fails + */ +export function validatePluginShape(PluginClass: any, pluginName: string): void { + // 1. Check it's a class/constructor function with a prototype + if (typeof PluginClass !== 'function' || !PluginClass.prototype) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_INVALID_SHAPE, + ErrorScope.PLUGIN, + ErrorType.USER, + `Plugin '${pluginName}' default export must be a class or constructor function` + ); + } + + // 2. Use prototype for shape validation (avoid constructor side effects) + const proto = PluginClass.prototype; + + // 3. Check it has at least one extension point method + const extensionPoints = [ + 'beforeLLMRequest', + 'beforeToolCall', + 'afterToolResult', + 'beforeResponse', + ]; + + const hasExtensionPoint = extensionPoints.some((point) => typeof proto[point] === 'function'); + + if (!hasExtensionPoint) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_INVALID_SHAPE, + ErrorScope.PLUGIN, + ErrorType.USER, + `Plugin '${pluginName}' must implement at least one extension point method`, + { availableExtensionPoints: extensionPoints } + ); + } + + // 4. Validate initialize if present + if ('initialize' in proto && typeof proto.initialize !== 'function') { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_INVALID_SHAPE, + ErrorScope.PLUGIN, + ErrorType.USER, + `Plugin '${pluginName}' initialize property must be a function (found ${typeof proto.initialize})` + ); + } + + // 5. Validate cleanup if present + if ('cleanup' in proto && typeof proto.cleanup !== 'function') { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_INVALID_SHAPE, + ErrorScope.PLUGIN, + ErrorType.USER, + `Plugin '${pluginName}' cleanup property must be a function (found ${typeof proto.cleanup})` + ); + } +} + +/** + * Resolve and validate plugin module path + * Ensures path is absolute after template variable expansion + * + * @param modulePath - Path from config (after template expansion) + * @param configDir - Directory containing agent config (for validation context) + * @returns Resolved absolute path + * @throws {DextoRuntimeError} If path is not absolute + */ +export function resolvePluginPath(modulePath: string, configDir: string): string { + // Path should already be absolute after template expansion in config loader + // We just validate it here + if (!isAbsolute(modulePath)) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_CONFIGURATION_INVALID, + ErrorScope.PLUGIN, + ErrorType.USER, + "Plugin module path must be absolute (got '" + + modulePath + + "'). Use ${{dexto.agent_dir}} template variable for agent-relative paths.", + { + modulePath, + configDir, + hint: 'Example: module: "${{dexto.agent_dir}}/plugins/my-plugin.ts"', + } + ); + } + + return modulePath; +} + +/** + * Load a plugin from a module path + * Supports both .ts (via tsx) and .js files + * + * @param modulePath - Absolute path to plugin module + * @param pluginName - Name for error messages + * @returns Plugin class constructor + * @throws {DextoRuntimeError} If loading or validation fails + */ +export async function loadPluginModule(modulePath: string, pluginName: string): Promise { + try { + // TODO: Replace tsx runtime loader with build-time bundling for production + // SHORT-TERM: tsx provides on-the-fly TypeScript loading for development + // LONG-TERM: Implement `dexto bundle` CLI command that: + // 1. Parses agent config to discover all plugins + // 2. Generates static imports: import tenantAuth from './plugins/tenant-auth.js' + // 3. Creates plugin registry: { 'tenant-auth': tenantAuth } + // 4. Bundles with esbuild/tsup into single artifact + // 5. Loads from registry in production (no runtime compilation) + // Benefits: Zero runtime overhead, works in serverless, smaller bundle size + // See: feature-plans/plugin-system.md lines 2082-2133 for full design + let pluginModule: any; + + if (modulePath.endsWith('.ts') || modulePath.endsWith('.tsx')) { + // Use tsx for TypeScript files (development mode) + // tsx is Node.js-only, so check environment first + if (typeof process === 'undefined' || !process.versions?.node) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_LOAD_FAILED, + ErrorScope.PLUGIN, + ErrorType.SYSTEM, + `Cannot load TypeScript plugin '${pluginName}' in browser environment. ` + + `Plugins with .ts extension require Node.js runtime.`, + { modulePath, pluginName } + ); + } + + // Use computed string + webpackIgnore to prevent webpack from analyzing/bundling tsx + // This tells webpack to skip this import during static analysis + const tsxPackage = 'tsx/esm/api'; + let tsx: any; + try { + tsx = await import(/* webpackIgnore: true */ tsxPackage); + } catch (importError: unknown) { + const err = importError as NodeJS.ErrnoException; + if (err.code === 'ERR_MODULE_NOT_FOUND') { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_DEPENDENCY_NOT_INSTALLED, + ErrorScope.PLUGIN, + ErrorType.USER, + `Cannot load TypeScript plugin '${pluginName}': tsx package is not installed.\n` + + `Install with: npm install tsx\n` + + `Or pre-compile your plugin to .js for production use.`, + { modulePath, pluginName, packageName: 'tsx' } + ); + } + throw importError; + } + // Convert absolute path to file:// URL for cross-platform ESM compatibility + const moduleUrl = pathToFileURL(modulePath).href; + pluginModule = await tsx.tsImport(moduleUrl, import.meta.url); + } else { + // Direct import for JavaScript files (production mode) + // Convert absolute path to file:// URL for cross-platform ESM compatibility + const moduleUrl = pathToFileURL(modulePath).href; + pluginModule = await import(/* webpackIgnore: true */ moduleUrl); + } + + // Check for default export + // Handle tsx ESM interop which may double-wrap the default export + // as { default: { default: [class] } } instead of { default: [class] } + let PluginClass = pluginModule.default; + if (PluginClass && typeof PluginClass === 'object' && 'default' in PluginClass) { + PluginClass = PluginClass.default; + } + + if (!PluginClass) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_INVALID_SHAPE, + ErrorScope.PLUGIN, + ErrorType.USER, + `Plugin '${pluginName}' at '${modulePath}' has no default export. ` + + `Ensure your plugin exports a class as default.`, + { modulePath, pluginName } + ); + } + + // Validate plugin shape + validatePluginShape(PluginClass, pluginName); + + return PluginClass; + } catch (error) { + // Re-throw our own errors + if (error instanceof DextoRuntimeError) { + throw error; + } + + // Wrap other errors (import failures, etc.) + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_LOAD_FAILED, + ErrorScope.PLUGIN, + ErrorType.SYSTEM, + `Failed to load plugin '${pluginName}' from '${modulePath}': ${ + error instanceof Error ? error.message : String(error) + }`, + { + modulePath, + pluginName, + originalError: error instanceof Error ? error.message : String(error), + } + ); + } +} diff --git a/dexto/packages/core/src/plugins/manager.ts b/dexto/packages/core/src/plugins/manager.ts new file mode 100644 index 00000000..35b7cfaf --- /dev/null +++ b/dexto/packages/core/src/plugins/manager.ts @@ -0,0 +1,612 @@ +import { DextoRuntimeError, ErrorScope, ErrorType } from '../errors/index.js'; +import { PluginErrorCode } from './error-codes.js'; +import { loadPluginModule, resolvePluginPath } from './loader.js'; +import { getContext } from '../utils/async-context.js'; +import { pluginRegistry, type PluginCreationContext } from './registry.js'; +import type { RegistryPluginConfig } from './schemas.js'; +import type { + ExtensionPoint, + PluginExecutionContext, + PluginConfig, + LoadedPlugin, + PluginResult, +} from './types.js'; +import type { AgentEventBus } from '../events/index.js'; +import type { StorageManager } from '../storage/index.js'; +import type { SessionManager } from '../session/index.js'; +import type { MCPManager } from '../mcp/manager.js'; +import type { ToolManager } from '../tools/tool-manager.js'; +import type { AgentStateManager } from '../agent/state-manager.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; + +/** + * Options for PluginManager construction + */ +export interface PluginManagerOptions { + agentEventBus: AgentEventBus; + storageManager: StorageManager; + configDir: string; +} + +/** + * Options for building ExecutionContext + * Used when calling executePlugins + */ +export interface ExecutionContextOptions { + sessionManager: SessionManager; + mcpManager: MCPManager; + toolManager: ToolManager; + stateManager: AgentStateManager; + sessionId?: string; + abortSignal?: AbortSignal; +} + +/** + * Plugin Manager - Orchestrates plugin loading and execution + * + * Responsibilities: + * - Load plugins from configuration (built-in + custom) + * - Validate plugin shape and priority uniqueness + * - Manage plugin lifecycle (initialize, execute, cleanup) + * - Execute plugins sequentially at extension points + * - Handle timeouts and errors with fail-fast policy + */ +export class PluginManager { + private plugins: Map = new Map(); + private pluginsByExtensionPoint: Map = new Map(); + private options: PluginManagerOptions; + private initialized: boolean = false; + private logger: IDextoLogger; + + /** Default timeout for plugin execution (milliseconds) */ + private static readonly DEFAULT_TIMEOUT = 5000; + + constructor(options: PluginManagerOptions, logger: IDextoLogger) { + this.options = options; + this.logger = logger.createChild(DextoLogComponent.PLUGIN); + this.logger.debug('PluginManager created'); + } + + /** + * Register a built-in plugin + * Called by the built-in plugin registry before initialize() + * + * @param name - Plugin name + * @param PluginClass - Plugin class constructor + * @param config - Plugin configuration + */ + registerBuiltin(name: string, PluginClass: any, config: Omit): void { + if (this.initialized) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_CONFIGURATION_INVALID, + ErrorScope.PLUGIN, + ErrorType.SYSTEM, + 'Cannot register built-in plugins after initialization' + ); + } + + // Create plugin instance + const plugin = new PluginClass(); + + // Store as loaded plugin with synthetic module path + const loadedPlugin: LoadedPlugin = { + plugin, + config: { + name, + module: ``, + enabled: config.enabled ?? true, + blocking: config.blocking, + priority: config.priority, + config: config.config ?? undefined, + }, + }; + + this.plugins.set(name, loadedPlugin); + this.logger.debug(`Built-in plugin registered: ${name}`); + } + + /** + * Initialize all plugins from configuration + * Loads custom plugins (file-based), registry plugins (programmatic), validates priorities, + * sorts by priority, and calls initialize() + * + * TODO: Consider adding an MCP server-like convention for plugin discovery. + * Instead of requiring explicit file paths, plugins could be connected as + * plugin servers to the PluginManager. + * + * @param customPlugins - Array of custom plugin configurations from YAML (file-based) + * @param registryPlugins - Array of registry plugin configurations from YAML (programmatic) + * @throws {DextoRuntimeError} If any plugin fails to load or initialize (fail-fast) + */ + async initialize( + customPlugins: PluginConfig[] = [], + registryPlugins: RegistryPluginConfig[] = [] + ): Promise { + if (this.initialized) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_CONFIGURATION_INVALID, + ErrorScope.PLUGIN, + ErrorType.SYSTEM, + 'PluginManager already initialized' + ); + } + + // 1. Validate priority uniqueness across all plugins (built-in + custom + registry) + const priorities = new Set(); + const allPluginConfigs = [ + ...Array.from(this.plugins.values()).map((p) => p.config), + ...customPlugins, + ...registryPlugins.map((r) => ({ + name: r.type, + module: ``, + enabled: r.enabled, + blocking: r.blocking, + priority: r.priority, + config: r.config, + })), + ]; + + for (const config of allPluginConfigs) { + if (!config.enabled) continue; + + if (priorities.has(config.priority)) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_DUPLICATE_PRIORITY, + ErrorScope.PLUGIN, + ErrorType.USER, + `Duplicate plugin priority: ${config.priority}. Each plugin must have a unique priority.`, + { + priority: config.priority, + hint: 'Ensure all enabled plugins (built-in, custom, and registry) have unique priority values.', + } + ); + } + priorities.add(config.priority); + } + + // 2. Load registry plugins first (they're programmatically registered) + for (const registryConfig of registryPlugins) { + if (!registryConfig.enabled) { + this.logger.debug(`Skipping disabled registry plugin: ${registryConfig.type}`); + continue; + } + + try { + // Get the provider from registry + const provider = pluginRegistry.get(registryConfig.type); + if (!provider) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_PROVIDER_NOT_FOUND, + ErrorScope.PLUGIN, + ErrorType.USER, + `Plugin provider '${registryConfig.type}' not found in registry`, + { + type: registryConfig.type, + available: pluginRegistry.getTypes(), + }, + `Available plugin providers: ${pluginRegistry.getTypes().join(', ') || 'none'}. Register the provider using pluginRegistry.register() before agent initialization.` + ); + } + + // Validate config against provider schema + const validatedConfig = provider.configSchema.safeParse({ + type: registryConfig.type, + ...registryConfig.config, + }); + + if (!validatedConfig.success) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_PROVIDER_VALIDATION_FAILED, + ErrorScope.PLUGIN, + ErrorType.USER, + `Invalid configuration for plugin provider '${registryConfig.type}'`, + { + type: registryConfig.type, + errors: validatedConfig.error.errors, + }, + 'Check the configuration schema for this plugin provider' + ); + } + + // Create plugin instance + const creationContext: PluginCreationContext = { + config: registryConfig.config || {}, + blocking: registryConfig.blocking, + priority: registryConfig.priority, + }; + + const plugin = provider.create(validatedConfig.data, creationContext); + + // Store as loaded plugin + const loadedPlugin: LoadedPlugin = { + plugin, + config: { + name: registryConfig.type, + module: ``, + enabled: registryConfig.enabled, + blocking: registryConfig.blocking, + priority: registryConfig.priority, + config: registryConfig.config, + }, + }; + this.plugins.set(registryConfig.type, loadedPlugin); + + this.logger.info(`Registry plugin loaded: ${registryConfig.type}`); + } catch (error) { + // Re-throw our own errors + if (error instanceof DextoRuntimeError) { + throw error; + } + + // Wrap other errors + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_INITIALIZATION_FAILED, + ErrorScope.PLUGIN, + ErrorType.SYSTEM, + `Failed to load registry plugin '${registryConfig.type}': ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + // 3. Load custom plugins from config (file-based) + for (const pluginConfig of customPlugins) { + if (!pluginConfig.enabled) { + this.logger.debug(`Skipping disabled plugin: ${pluginConfig.name}`); + continue; + } + + try { + // Resolve and validate path + const modulePath = resolvePluginPath(pluginConfig.module, this.options.configDir); + + // Load plugin module + const PluginClass = await loadPluginModule(modulePath, pluginConfig.name); + + // Instantiate + const plugin = new PluginClass(); + + // Store + const loadedPlugin: LoadedPlugin = { + plugin, + config: pluginConfig, + }; + this.plugins.set(pluginConfig.name, loadedPlugin); + + this.logger.info(`Custom plugin loaded: ${pluginConfig.name}`); + } catch (error) { + // Fail fast - cannot run with broken plugins + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_INITIALIZATION_FAILED, + ErrorScope.PLUGIN, + ErrorType.SYSTEM, + `Failed to load plugin '${pluginConfig.name}': ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + // 3. Initialize all plugins (call their initialize() method if exists) + for (const [name, loadedPlugin] of this.plugins.entries()) { + if (!loadedPlugin.config.enabled) continue; + + try { + if (loadedPlugin.plugin.initialize) { + await loadedPlugin.plugin.initialize(loadedPlugin.config.config || {}); + this.logger.debug(`Plugin initialized: ${name}`); + } + } catch (error) { + // Fail fast - plugin initialization failure is critical + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_INITIALIZATION_FAILED, + ErrorScope.PLUGIN, + ErrorType.SYSTEM, + `Plugin '${name}' initialization failed: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + // 4. Register plugins to their extension points + for (const loadedPlugin of this.plugins.values()) { + if (!loadedPlugin.config.enabled) continue; + this.registerToExtensionPoints(loadedPlugin); + } + + // 5. Sort plugins by priority for each extension point (low to high) + for (const [extensionPoint, plugins] of this.pluginsByExtensionPoint.entries()) { + plugins.sort((a, b) => a.config.priority - b.config.priority); + this.logger.debug( + `Extension point '${extensionPoint}': ${plugins.length} plugin(s) registered`, + { + plugins: plugins.map((p) => ({ + name: p.config.name, + priority: p.config.priority, + })), + } + ); + } + + this.initialized = true; + this.logger.info(`PluginManager initialized with ${this.plugins.size} plugin(s)`); + } + + /** + * Register a plugin to the extension points it implements + */ + private registerToExtensionPoints(loadedPlugin: LoadedPlugin): void { + const extensionPoints: ExtensionPoint[] = [ + 'beforeLLMRequest', + 'beforeToolCall', + 'afterToolResult', + 'beforeResponse', + ]; + + for (const point of extensionPoints) { + if (typeof loadedPlugin.plugin[point] === 'function') { + if (!this.pluginsByExtensionPoint.has(point)) { + this.pluginsByExtensionPoint.set(point, []); + } + this.pluginsByExtensionPoint.get(point)!.push(loadedPlugin); + } + } + } + + /** + * Execute all plugins at a specific extension point + * Plugins execute sequentially in priority order + * + * @param extensionPoint - Which extension point to execute + * @param payload - Payload for this extension point (must be an object) + * @param options - Options for building execution context + * @returns Modified payload after all plugins execute + * @throws {DextoRuntimeError} If a blocking plugin cancels execution or payload is not an object + */ + async executePlugins>( + extensionPoint: ExtensionPoint, + payload: T, + options: ExecutionContextOptions + ): Promise { + const plugins = this.pluginsByExtensionPoint.get(extensionPoint) || []; + if (plugins.length === 0) { + return payload; // No plugins for this extension point + } + + // Defensive runtime check: payload must be an object for spread operator + if (payload === null || typeof payload !== 'object') { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_INVALID_SHAPE, + ErrorScope.PLUGIN, + ErrorType.USER, + `Payload for ${extensionPoint} must be an object (got ${payload === null ? 'null' : typeof payload})`, + { extensionPoint, payloadType: typeof payload } + ); + } + + let currentPayload: T = { ...payload }; + + // Build execution context + const asyncCtx = getContext(); + const llmConfig = options.stateManager.getLLMConfig(options.sessionId); + + const context: PluginExecutionContext = { + sessionId: options.sessionId ?? undefined, + userId: asyncCtx?.userId ?? undefined, + tenantId: asyncCtx?.tenantId ?? undefined, + llmConfig, + logger: this.logger, + abortSignal: options.abortSignal ?? undefined, + agent: { + sessionManager: options.sessionManager, + mcpManager: options.mcpManager, + toolManager: options.toolManager, + stateManager: options.stateManager, + agentEventBus: this.options.agentEventBus, + storageManager: this.options.storageManager, + }, + }; + + // Execute plugins sequentially + for (const { plugin, config } of plugins) { + const method = plugin[extensionPoint]; + if (!method) continue; // Shouldn't happen, but be safe + + const startTime = Date.now(); + + try { + // Execute with timeout + // Use type assertion since we validated the method exists and has correct signature + const result = await this.executeWithTimeout( + (method as any).call(plugin, currentPayload, context), + config.name, + PluginManager.DEFAULT_TIMEOUT + ); + + const duration = Date.now() - startTime; + + // Log execution + this.logger.debug(`Plugin '${config.name}' executed at ${extensionPoint}`, { + ok: result.ok, + cancelled: result.cancel, + duration, + hasModifications: !!result.modify, + }); + + // Emit notices if any + if (result.notices && result.notices.length > 0) { + for (const notice of result.notices) { + const level = + notice.kind === 'block' || notice.kind === 'warn' ? 'warn' : 'info'; + this.logger[level](`Plugin notice (${notice.kind}): ${notice.message}`, { + plugin: config.name, + code: notice.code, + details: notice.details, + }); + } + } + + // Handle failure + if (!result.ok) { + this.logger.warn(`Plugin '${config.name}' returned error`, { + message: result.message, + }); + + if (config.blocking && result.cancel) { + // Blocking plugin wants to stop execution + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_BLOCKED_EXECUTION, + ErrorScope.PLUGIN, + ErrorType.FORBIDDEN, + result.message || `Operation blocked by plugin '${config.name}'`, + { + plugin: config.name, + extensionPoint, + notices: result.notices, + } + ); + } + + // Non-blocking: continue to next plugin + continue; + } + + // Apply modifications + if (result.modify) { + currentPayload = { ...currentPayload, ...result.modify }; + this.logger.debug(`Plugin '${config.name}' modified payload`, { + keys: Object.keys(result.modify), + }); + } + + // Check cancellation + if (result.cancel && config.blocking) { + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_BLOCKED_EXECUTION, + ErrorScope.PLUGIN, + ErrorType.FORBIDDEN, + result.message || `Operation cancelled by plugin '${config.name}'`, + { + plugin: config.name, + extensionPoint, + notices: result.notices, + } + ); + } + } catch (error) { + const duration = Date.now() - startTime; + + // Re-throw our own errors + if (error instanceof DextoRuntimeError) { + throw error; + } + + // Plugin threw exception + this.logger.error(`Plugin '${config.name}' threw error`, { + error: error instanceof Error ? error.message : String(error), + duration, + }); + + if (config.blocking) { + // Blocking plugin failed - stop execution + throw new DextoRuntimeError( + PluginErrorCode.PLUGIN_EXECUTION_FAILED, + ErrorScope.PLUGIN, + ErrorType.SYSTEM, + `Plugin '${config.name}' failed: ${ + error instanceof Error ? error.message : String(error) + }`, + { + plugin: config.name, + extensionPoint, + } + ); + } + + // Non-blocking: continue + this.logger.debug(`Non-blocking plugin error, continuing execution`); + } + } + + return currentPayload; + } + + /** + * Execute a promise with timeout, properly clearing timer on completion + * Prevents timer leaks and unhandled rejections from Promise.race + */ + private async executeWithTimeout( + promise: Promise, + pluginName: string, + ms: number + ): Promise { + let timer: NodeJS.Timeout | undefined; + return await new Promise((resolve, reject) => { + timer = setTimeout(() => { + reject( + new DextoRuntimeError( + PluginErrorCode.PLUGIN_EXECUTION_TIMEOUT, + ErrorScope.PLUGIN, + ErrorType.TIMEOUT, + `Plugin '${pluginName}' execution timed out after ${ms}ms` + ) + ); + }, ms); + promise.then( + (val) => { + if (timer) clearTimeout(timer); + resolve(val); + }, + (err) => { + if (timer) clearTimeout(timer); + reject(err); + } + ); + }); + } + + /** + * Cleanup all plugins + * Called when agent shuts down + */ + async cleanup(): Promise { + for (const [name, loadedPlugin] of this.plugins.entries()) { + if (loadedPlugin.plugin.cleanup) { + try { + await loadedPlugin.plugin.cleanup(); + this.logger.debug(`Plugin cleaned up: ${name}`); + } catch (error) { + this.logger.error(`Plugin cleanup failed: ${name}`, { + error: error instanceof Error ? error.message : String(error), + }); + } + } + } + this.logger.info('PluginManager cleanup complete'); + } + + /** + * Get plugin statistics + */ + getStats(): { + total: number; + enabled: number; + byExtensionPoint: Record; + } { + const enabled = Array.from(this.plugins.values()).filter((p) => p.config.enabled).length; + + const byExtensionPoint: Record = {}; + for (const [point, plugins] of this.pluginsByExtensionPoint.entries()) { + byExtensionPoint[point] = plugins.length; + } + + return { + total: this.plugins.size, + enabled, + byExtensionPoint: byExtensionPoint as Record, + }; + } +} diff --git a/dexto/packages/core/src/plugins/registrations/builtins.ts b/dexto/packages/core/src/plugins/registrations/builtins.ts new file mode 100644 index 00000000..77935bb8 --- /dev/null +++ b/dexto/packages/core/src/plugins/registrations/builtins.ts @@ -0,0 +1,43 @@ +import type { PluginManager } from '../manager.js'; +import type { ValidatedAgentConfig } from '../../agent/schemas.js'; +import { ContentPolicyPlugin } from '../builtins/content-policy.js'; +import { ResponseSanitizerPlugin } from '../builtins/response-sanitizer.js'; + +/** + * Register all built-in plugins with the PluginManager + * Called during agent initialization before custom plugins are loaded + * + * Built-in plugins are referenced by name in the config (e.g., contentPolicy, responseSanitizer) + * and activated based on presence of their configuration object. + * + * @param pluginManager - The PluginManager instance + * @param config - Validated agent configuration + */ +export function registerBuiltInPlugins(args: { + pluginManager: PluginManager; + config: ValidatedAgentConfig; +}): void { + // Register ContentPolicy plugin if configured + const cp = args.config.plugins?.contentPolicy; + if (cp && typeof cp === 'object' && cp.enabled !== false) { + args.pluginManager.registerBuiltin('content-policy', ContentPolicyPlugin, { + name: 'content-policy', + enabled: cp.enabled ?? true, + priority: cp.priority, + blocking: cp.blocking ?? true, + config: cp, + }); + } + + // Register ResponseSanitizer plugin if configured + const rs = args.config.plugins?.responseSanitizer; + if (rs && typeof rs === 'object' && rs.enabled !== false) { + args.pluginManager.registerBuiltin('response-sanitizer', ResponseSanitizerPlugin, { + name: 'response-sanitizer', + enabled: rs.enabled ?? true, + priority: rs.priority, + blocking: rs.blocking ?? false, + config: rs, + }); + } +} diff --git a/dexto/packages/core/src/plugins/registry.test.ts b/dexto/packages/core/src/plugins/registry.test.ts new file mode 100644 index 00000000..e1de9a50 --- /dev/null +++ b/dexto/packages/core/src/plugins/registry.test.ts @@ -0,0 +1,344 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { z } from 'zod'; +import { + PluginRegistry, + pluginRegistry, + type PluginProvider, + type PluginCreationContext, +} from './registry.js'; +import type { + DextoPlugin, + PluginResult, + BeforeLLMRequestPayload, + PluginExecutionContext, +} from './types.js'; +import { PluginErrorCode } from './error-codes.js'; +import { DextoRuntimeError } from '../errors/index.js'; + +// Test plugin implementation +class TestPlugin implements DextoPlugin { + constructor( + public config: any, + public context: PluginCreationContext + ) {} + + async beforeLLMRequest( + _payload: BeforeLLMRequestPayload, + _context: PluginExecutionContext + ): Promise { + return { ok: true }; + } +} + +// Test plugin config schema +const TestPluginConfigSchema = z.object({ + type: z.literal('test-plugin'), + message: z.string().default('hello'), +}); + +// Test plugin provider +const testPluginProvider: PluginProvider<'test-plugin', z.output> = { + type: 'test-plugin', + configSchema: TestPluginConfigSchema, + create(config, context) { + return new TestPlugin(config, context); + }, + metadata: { + displayName: 'Test Plugin', + description: 'A test plugin for unit testing', + extensionPoints: ['beforeLLMRequest'], + category: 'test', + }, +}; + +describe('PluginRegistry', () => { + let registry: PluginRegistry; + + beforeEach(() => { + registry = new PluginRegistry(); + }); + + describe('register', () => { + it('should register a plugin provider', () => { + registry.register(testPluginProvider); + expect(registry.has('test-plugin')).toBe(true); + }); + + it('should throw when registering duplicate provider', () => { + registry.register(testPluginProvider); + + expect(() => registry.register(testPluginProvider)).toThrow(DextoRuntimeError); + + try { + registry.register(testPluginProvider); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe( + PluginErrorCode.PLUGIN_PROVIDER_ALREADY_REGISTERED + ); + } + }); + + it('should allow registering multiple different providers', () => { + const provider1: PluginProvider = { + type: 'plugin-a', + configSchema: z.object({ type: z.literal('plugin-a') }), + create: () => ({ + async beforeLLMRequest() { + return { ok: true }; + }, + }), + }; + + const provider2: PluginProvider = { + type: 'plugin-b', + configSchema: z.object({ type: z.literal('plugin-b') }), + create: () => ({ + async beforeLLMRequest() { + return { ok: true }; + }, + }), + }; + + registry.register(provider1); + registry.register(provider2); + + expect(registry.size).toBe(2); + expect(registry.getTypes()).toEqual(['plugin-a', 'plugin-b']); + }); + }); + + describe('unregister', () => { + it('should unregister an existing provider', () => { + registry.register(testPluginProvider); + const result = registry.unregister('test-plugin'); + + expect(result).toBe(true); + expect(registry.has('test-plugin')).toBe(false); + }); + + it('should return false when unregistering non-existent provider', () => { + const result = registry.unregister('nonexistent'); + expect(result).toBe(false); + }); + }); + + describe('get', () => { + it('should return the provider if found', () => { + registry.register(testPluginProvider); + + const result = registry.get('test-plugin'); + expect(result).toEqual(testPluginProvider); + }); + + it('should return undefined if not found', () => { + const result = registry.get('nonexistent'); + expect(result).toBeUndefined(); + }); + }); + + describe('has', () => { + it('should return true for registered providers', () => { + registry.register(testPluginProvider); + expect(registry.has('test-plugin')).toBe(true); + }); + + it('should return false for non-registered providers', () => { + expect(registry.has('nonexistent')).toBe(false); + }); + }); + + describe('getTypes', () => { + it('should return empty array when no providers', () => { + expect(registry.getTypes()).toEqual([]); + }); + + it('should return all registered types', () => { + const provider1: PluginProvider = { + type: 'plugin-a', + configSchema: z.object({ type: z.literal('plugin-a') }), + create: () => ({ + async beforeLLMRequest() { + return { ok: true }; + }, + }), + }; + + const provider2: PluginProvider = { + type: 'plugin-b', + configSchema: z.object({ type: z.literal('plugin-b') }), + create: () => ({ + async beforeLLMRequest() { + return { ok: true }; + }, + }), + }; + + registry.register(provider1); + registry.register(provider2); + + expect(registry.getTypes()).toEqual(['plugin-a', 'plugin-b']); + }); + }); + + describe('getAll / getProviders', () => { + it('should return empty array when no providers', () => { + expect(registry.getAll()).toEqual([]); + expect(registry.getProviders()).toEqual([]); + }); + + it('should return all registered providers', () => { + registry.register(testPluginProvider); + + expect(registry.getAll()).toEqual([testPluginProvider]); + expect(registry.getProviders()).toEqual([testPluginProvider]); + }); + }); + + describe('size', () => { + it('should return 0 when empty', () => { + expect(registry.size).toBe(0); + }); + + it('should return correct count', () => { + const provider1: PluginProvider = { + type: 'plugin-a', + configSchema: z.object({ type: z.literal('plugin-a') }), + create: () => ({ + async beforeLLMRequest() { + return { ok: true }; + }, + }), + }; + + registry.register(provider1); + registry.register(testPluginProvider); + + expect(registry.size).toBe(2); + }); + }); + + describe('clear', () => { + it('should remove all providers', () => { + registry.register(testPluginProvider); + registry.clear(); + + expect(registry.size).toBe(0); + expect(registry.getTypes()).toEqual([]); + }); + }); + + describe('validateConfig', () => { + beforeEach(() => { + registry.register(testPluginProvider); + }); + + it('should validate against provider schema', () => { + const result = registry.validateConfig({ type: 'test-plugin', message: 'world' }); + expect(result).toEqual({ type: 'test-plugin', message: 'world' }); + }); + + it('should use default values from schema', () => { + const result = registry.validateConfig({ type: 'test-plugin' }); + expect(result).toEqual({ type: 'test-plugin', message: 'hello' }); + }); + + it('should throw if provider not found', () => { + expect(() => registry.validateConfig({ type: 'unknown' })).toThrow(DextoRuntimeError); + + try { + registry.validateConfig({ type: 'unknown' }); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe( + PluginErrorCode.PLUGIN_PROVIDER_NOT_FOUND + ); + } + }); + + it('should throw on schema validation failure', () => { + expect(() => registry.validateConfig({ type: 'test-plugin', message: 123 })).toThrow(); + }); + }); + + describe('plugin creation', () => { + it('should create plugin instance with correct config and context', () => { + registry.register(testPluginProvider); + const provider = registry.get('test-plugin')!; + + const context: PluginCreationContext = { + config: { extra: 'data' }, + blocking: true, + priority: 10, + }; + + const plugin = provider.create({ type: 'test-plugin', message: 'test' }, context); + + expect(plugin).toBeInstanceOf(TestPlugin); + expect((plugin as TestPlugin).config).toEqual({ type: 'test-plugin', message: 'test' }); + expect((plugin as TestPlugin).context).toEqual(context); + }); + + it('should create plugin with metadata', () => { + registry.register(testPluginProvider); + const provider = registry.get('test-plugin')!; + + expect(provider.metadata).toBeDefined(); + expect(provider.metadata?.displayName).toBe('Test Plugin'); + expect(provider.metadata?.description).toBe('A test plugin for unit testing'); + expect(provider.metadata?.extensionPoints).toEqual(['beforeLLMRequest']); + expect(provider.metadata?.category).toBe('test'); + }); + }); + + describe('edge cases', () => { + it('should handle empty string type', () => { + const emptyProvider: PluginProvider = { + type: '', + configSchema: z.object({ type: z.literal('') }), + create: () => ({ + async beforeLLMRequest() { + return { ok: true }; + }, + }), + }; + + registry.register(emptyProvider); + expect(registry.has('')).toBe(true); + }); + + it('should handle special characters in type', () => { + const specialProvider: PluginProvider = { + type: 'plugin-with_special.chars', + configSchema: z.object({ type: z.literal('plugin-with_special.chars') }), + create: () => ({ + async beforeLLMRequest() { + return { ok: true }; + }, + }), + }; + + registry.register(specialProvider); + expect(registry.has('plugin-with_special.chars')).toBe(true); + }); + + it('should handle re-registration after unregister', () => { + registry.register(testPluginProvider); + registry.unregister('test-plugin'); + registry.register(testPluginProvider); + + expect(registry.has('test-plugin')).toBe(true); + }); + }); +}); + +describe('pluginRegistry singleton', () => { + it('should be an instance of PluginRegistry', () => { + expect(pluginRegistry).toBeInstanceOf(PluginRegistry); + }); + + it('should be the same instance across imports', async () => { + const { pluginRegistry: registry2 } = await import('./registry.js'); + expect(pluginRegistry).toBe(registry2); + }); +}); diff --git a/dexto/packages/core/src/plugins/registry.ts b/dexto/packages/core/src/plugins/registry.ts new file mode 100644 index 00000000..16d6ff1b --- /dev/null +++ b/dexto/packages/core/src/plugins/registry.ts @@ -0,0 +1,142 @@ +import { z } from 'zod'; +import type { DextoPlugin } from './types.js'; +import { DextoRuntimeError, ErrorScope, ErrorType } from '../errors/index.js'; +import { PluginErrorCode } from './error-codes.js'; +import { BaseRegistry, type RegistryErrorFactory } from '../providers/base-registry.js'; + +/** + * Context passed to plugin providers when creating plugin instances. + * Provides access to configuration and optional services. + */ +export interface PluginCreationContext { + /** Plugin-specific configuration from YAML */ + config: Record; + /** Whether this plugin should block execution on errors */ + blocking: boolean; + /** Execution priority (lower runs first) */ + priority: number; +} + +/** + * Plugin provider interface. + * Allows external code to register plugin providers that create plugin instances. + * Follows the same pattern as BlobStoreProvider, CompressionProvider, and CustomToolProvider. + * + * @template TType - The provider type discriminator (must match config.type) + * @template TConfig - The provider configuration type (must include { type: TType }) + */ +export interface PluginProvider< + TType extends string = string, + TConfig extends { type: TType } = any, +> { + /** Unique type identifier matching the discriminator in config */ + type: TType; + + /** Zod schema for runtime validation of provider configuration */ + configSchema: z.ZodType; + + /** + * Factory function to create a plugin instance from validated configuration + * @param config - Validated configuration matching configSchema + * @param context - Plugin creation context with priority and blocking settings + * @returns A DextoPlugin instance + */ + create(config: TConfig, context: PluginCreationContext): DextoPlugin; + + /** Optional metadata for display and categorization */ + metadata?: { + displayName: string; + description: string; + /** Which extension points this plugin implements */ + extensionPoints?: Array< + 'beforeLLMRequest' | 'beforeToolCall' | 'afterToolResult' | 'beforeResponse' + >; + /** Category for grouping (e.g., 'security', 'logging', 'integration') */ + category?: string; + }; +} + +/** + * Error factory for plugin registry errors. + * Uses PluginErrorCode for consistent error handling. + */ +const pluginRegistryErrorFactory: RegistryErrorFactory = { + alreadyRegistered: (type: string) => + new DextoRuntimeError( + PluginErrorCode.PLUGIN_PROVIDER_ALREADY_REGISTERED, + ErrorScope.PLUGIN, + ErrorType.USER, + `Plugin provider '${type}' is already registered`, + { type }, + 'Each plugin provider type can only be registered once' + ), + notFound: (type: string, availableTypes: string[]) => + new DextoRuntimeError( + PluginErrorCode.PLUGIN_PROVIDER_NOT_FOUND, + ErrorScope.PLUGIN, + ErrorType.USER, + `Plugin provider '${type}' not found`, + { type, available: availableTypes }, + `Available plugin providers: ${availableTypes.join(', ') || 'none'}` + ), +}; + +/** + * Registry for plugin providers. + * Follows the same pattern as BlobStoreRegistry, CompressionRegistry, and CustomToolRegistry. + * + * Plugin providers can be registered from external code (CLI, apps, distributions) + * and are validated at runtime using their Zod schemas. + * + * Extends BaseRegistry for common registry functionality. + * + * @example + * ```typescript + * // Define a plugin provider + * const myPluginProvider: PluginProvider<'my-plugin', MyPluginConfig> = { + * type: 'my-plugin', + * configSchema: MyPluginConfigSchema, + * create(config, context) { + * return new MyPlugin(config, context); + * }, + * metadata: { + * displayName: 'My Plugin', + * description: 'Does something useful', + * extensionPoints: ['beforeLLMRequest'], + * category: 'custom', + * }, + * }; + * + * // Register in dexto.config.ts + * import { pluginRegistry } from '@dexto/core'; + * pluginRegistry.register(myPluginProvider); + * + * // Use in agent YAML + * plugins: + * registry: + * - type: my-plugin + * priority: 50 + * blocking: false + * config: + * key: value + * ``` + */ +export class PluginRegistry extends BaseRegistry { + constructor() { + super(pluginRegistryErrorFactory); + } + + /** + * Get all registered plugin providers. + * Alias for getAll() to match other registry patterns. + */ + getProviders(): PluginProvider[] { + return this.getAll(); + } +} + +/** + * Global singleton instance of the plugin registry. + * Plugin providers should be registered at application startup. + */ +export const pluginRegistry = new PluginRegistry(); diff --git a/dexto/packages/core/src/plugins/schemas.ts b/dexto/packages/core/src/plugins/schemas.ts new file mode 100644 index 00000000..b744d2e0 --- /dev/null +++ b/dexto/packages/core/src/plugins/schemas.ts @@ -0,0 +1,85 @@ +import { z } from 'zod'; + +/** + * Schema for registry-based plugin configuration. + * These plugins are loaded from the pluginRegistry (programmatically registered). + */ +export const RegistryPluginConfigSchema = z + .object({ + type: z.string().describe('Plugin provider type (must be registered in pluginRegistry)'), + enabled: z.boolean().default(true).describe('Whether this plugin is enabled'), + blocking: z.boolean().describe('If true, plugin errors will halt execution'), + priority: z.number().int().describe('Execution priority (lower runs first)'), + config: z.record(z.any()).optional().describe('Plugin-specific configuration'), + }) + .strict(); + +export type RegistryPluginConfig = z.output; + +/** + * Schema for custom plugin configuration (loaded from file paths) + */ +export const CustomPluginConfigSchema = z + .object({ + name: z.string().describe('Unique name for the plugin'), + module: z + .string() + .describe( + 'Absolute path to plugin module (use ${{dexto.agent_dir}} for agent-relative paths)' + ), + enabled: z.boolean().default(true).describe('Whether this plugin is enabled'), + blocking: z.boolean().describe('If true, plugin errors will halt execution'), + priority: z.number().int().describe('Execution priority (lower runs first)'), + config: z.record(z.any()).optional().describe('Plugin-specific configuration'), + }) + .strict(); + +/** + * Schema for built-in plugin configuration + * Built-in plugins don't need module paths - they're referenced by name + */ +export const BuiltInPluginConfigSchema = z + .object({ + priority: z.number().int().describe('Execution priority (lower runs first)'), + blocking: z.boolean().optional().describe('If true, plugin errors will halt execution'), + enabled: z.boolean().default(true).describe('Whether this plugin is enabled'), + // Plugin-specific config fields are defined per-plugin + }) + .passthrough() // Allow additional fields for plugin-specific config + .describe('Configuration for a built-in plugin'); + +/** + * Main plugins configuration schema + * Supports built-in plugins (by name), custom plugins (file paths), and registry plugins (programmatic) + */ +export const PluginsConfigSchema = z + .object({ + // Built-in plugins - referenced by name + contentPolicy: BuiltInPluginConfigSchema.optional().describe( + 'Content policy plugin for input validation and sanitization' + ), + responseSanitizer: BuiltInPluginConfigSchema.optional().describe( + 'Response sanitizer plugin for output sanitization' + ), + + // Custom plugins - array of plugin configurations (loaded from file paths) + custom: z + .array(CustomPluginConfigSchema) + .default([]) + .describe('Array of custom plugin configurations (loaded from file paths)'), + + // Registry plugins - array of plugin configurations (loaded from pluginRegistry) + registry: z + .array(RegistryPluginConfigSchema) + .default([]) + .describe('Array of registry plugin configurations (loaded from pluginRegistry)'), + }) + .strict() + .default({ + custom: [], + registry: [], + }) + .describe('Plugin system configuration'); + +export type PluginsConfig = z.input; +export type ValidatedPluginsConfig = z.output; diff --git a/dexto/packages/core/src/plugins/types.ts b/dexto/packages/core/src/plugins/types.ts new file mode 100644 index 00000000..23b94038 --- /dev/null +++ b/dexto/packages/core/src/plugins/types.ts @@ -0,0 +1,182 @@ +import type { ValidatedLLMConfig } from '../llm/schemas.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import type { SessionManager } from '../session/index.js'; +import type { MCPManager } from '../mcp/manager.js'; +import type { ToolManager } from '../tools/tool-manager.js'; +import type { AgentStateManager } from '../agent/state-manager.js'; +import type { AgentEventBus } from '../events/index.js'; +import type { StorageManager } from '../storage/index.js'; + +/** + * Extension point names - fixed for MVP + * These are the 4 hook sites from PR #385 converted to generic plugin extension points + */ +export type ExtensionPoint = + | 'beforeLLMRequest' + | 'beforeToolCall' + | 'afterToolResult' + | 'beforeResponse'; + +/** + * Plugin result - what plugins return from extension point methods + */ +export interface PluginResult { + /** Did plugin execute successfully? */ + ok: boolean; + + /** Partial modifications to apply to payload */ + modify?: Record; + + /** Should execution stop? (Only respected if plugin is blocking) */ + cancel?: boolean; + + /** User-facing message (shown when cancelled) */ + message?: string; + + /** Notices for logging/events */ + notices?: PluginNotice[]; +} + +/** + * Plugin notice - for logging and user feedback + */ +export interface PluginNotice { + kind: 'allow' | 'block' | 'warn' | 'info'; + code?: string; + message: string; + details?: Record; +} + +/** + * Execution context passed to every plugin method + * Contains runtime state and read-only access to agent services + */ +export interface PluginExecutionContext { + /** Current session ID */ + sessionId?: string | undefined; + + /** User ID (set by application layer via AsyncLocalStorage) */ + userId?: string | undefined; + + /** Tenant ID (set by application layer for multi-tenant deployments via AsyncLocalStorage) */ + tenantId?: string | undefined; + + /** Current LLM configuration */ + llmConfig: ValidatedLLMConfig; + + /** Logger scoped to this plugin execution */ + logger: IDextoLogger; + + /** Abort signal for cancellation */ + abortSignal?: AbortSignal | undefined; + + /** Reference to agent services (read-only access) */ + agent: { + readonly sessionManager: SessionManager; + readonly mcpManager: MCPManager; + readonly toolManager: ToolManager; + readonly stateManager: AgentStateManager; + readonly agentEventBus: AgentEventBus; + readonly storageManager: StorageManager; + }; +} + +/** + * Payload for beforeLLMRequest extension point + */ +export interface BeforeLLMRequestPayload { + text: string; + imageData?: { image: string; mimeType: string }; + fileData?: { data: string; mimeType: string; filename?: string }; + sessionId?: string; +} + +/** + * Payload for beforeToolCall extension point + */ +export interface BeforeToolCallPayload { + toolName: string; + args: any; + sessionId?: string; + callId?: string; +} + +/** + * Payload for afterToolResult extension point + */ +export interface AfterToolResultPayload { + toolName: string; + result: any; + success: boolean; + sessionId?: string; + callId?: string; +} + +/** + * Payload for beforeResponse extension point + */ +export interface BeforeResponsePayload { + content: string; + reasoning?: string; + provider: string; + model?: string; + tokenUsage?: { input: number; output: number }; + sessionId?: string; +} + +/** + * Main plugin interface - implement any subset of these methods + * All methods are optional - plugin must implement at least one extension point + */ +export interface DextoPlugin { + /** Called once at plugin initialization (before agent starts) */ + initialize?(config: Record): Promise; + + /** Extension point: before LLM request */ + beforeLLMRequest?( + payload: BeforeLLMRequestPayload, + context: PluginExecutionContext + ): Promise; + + /** Extension point: before tool call */ + beforeToolCall?( + payload: BeforeToolCallPayload, + context: PluginExecutionContext + ): Promise; + + /** Extension point: after tool result */ + afterToolResult?( + payload: AfterToolResultPayload, + context: PluginExecutionContext + ): Promise; + + /** Extension point: before response */ + beforeResponse?( + payload: BeforeResponsePayload, + context: PluginExecutionContext + ): Promise; + + /** Called when agent shuts down (cleanup) */ + cleanup?(): Promise; +} + +/** + * Plugin configuration from YAML (custom plugins) + */ +export interface PluginConfig { + name: string; + module: string; + enabled: boolean; + blocking: boolean; + priority: number; + config?: Record | undefined; +} + +/** + * Loaded plugin with its configuration + * Internal type used by PluginManager + */ +export interface LoadedPlugin { + plugin: DextoPlugin; + config: PluginConfig; +} diff --git a/dexto/packages/core/src/prompts/error-codes.ts b/dexto/packages/core/src/prompts/error-codes.ts new file mode 100644 index 00000000..899e3e88 --- /dev/null +++ b/dexto/packages/core/src/prompts/error-codes.ts @@ -0,0 +1,18 @@ +/** + * Prompt-specific error codes + * Includes prompt resolution, validation, and provider errors + */ +export enum PromptErrorCode { + // Prompt resolution errors + PROMPT_NOT_FOUND = 'prompt_not_found', + PROMPT_EMPTY_CONTENT = 'prompt_empty_content', + PROMPT_PROVIDER_NOT_FOUND = 'prompt_provider_not_found', + + // Validation errors + PROMPT_NAME_REQUIRED = 'prompt_name_required', + PROMPT_INVALID_NAME = 'prompt_invalid_name', + PROMPT_MISSING_TEXT = 'prompt_missing_text', + PROMPT_MISSING_REQUIRED_ARGUMENTS = 'prompt_missing_required_arguments', + PROMPT_ALREADY_EXISTS = 'prompt_already_exists', + PROMPT_CONFIG_INVALID = 'prompt_config_invalid', +} diff --git a/dexto/packages/core/src/prompts/errors.ts b/dexto/packages/core/src/prompts/errors.ts new file mode 100644 index 00000000..9c19af65 --- /dev/null +++ b/dexto/packages/core/src/prompts/errors.ts @@ -0,0 +1,145 @@ +import { DextoRuntimeError } from '@core/errors/DextoRuntimeError.js'; +import { DextoValidationError } from '@core/errors/DextoValidationError.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { PromptErrorCode } from './error-codes.js'; + +/** + * Prompt error factory with typed methods for creating prompt-specific errors + * Each method creates a properly typed error with appropriate scope + */ +export class PromptError { + /** + * Prompt not found error + */ + static notFound(name: string) { + return new DextoRuntimeError( + PromptErrorCode.PROMPT_NOT_FOUND, + ErrorScope.PROMPT, + ErrorType.NOT_FOUND, + `Prompt not found: ${name}`, + { name } + ); + } + + /** + * Missing prompt text validation error + */ + static missingText() { + return new DextoValidationError([ + { + code: PromptErrorCode.PROMPT_MISSING_TEXT, + message: 'Prompt missing text content', + scope: ErrorScope.PROMPT, + type: ErrorType.USER, + severity: 'error', + context: {}, + }, + ]); + } + + /** + * Missing required arguments validation error + */ + static missingRequiredArguments(missingNames: string[]) { + return new DextoValidationError([ + { + code: PromptErrorCode.PROMPT_MISSING_REQUIRED_ARGUMENTS, + message: `Missing required arguments: ${missingNames.join(', ')}`, + scope: ErrorScope.PROMPT, + type: ErrorType.USER, + severity: 'error', + context: { missingNames }, + }, + ]); + } + + /** + * Provider not found error + */ + static providerNotFound(source: string) { + return new DextoRuntimeError( + PromptErrorCode.PROMPT_PROVIDER_NOT_FOUND, + ErrorScope.PROMPT, + ErrorType.NOT_FOUND, + `No provider found for prompt source: ${source}`, + { source } + ); + } + + /** + * Missing prompt name in request (validation) + */ + static nameRequired() { + return new DextoValidationError([ + { + code: PromptErrorCode.PROMPT_NAME_REQUIRED, + message: 'Prompt name is required', + scope: ErrorScope.PROMPT, + type: ErrorType.USER, + severity: 'error', + context: {}, + }, + ]); + } + + /** + * Invalid prompt name format validation error + */ + static invalidName(name: string, guidance: string, context?: string, hint?: string) { + const contextPrefix = context ?? 'Prompt name'; + const hintSuffix = hint ? ` ${hint}` : ''; + return new DextoValidationError([ + { + code: PromptErrorCode.PROMPT_INVALID_NAME, + message: `${contextPrefix} '${name}' must be ${guidance}.${hintSuffix}`, + scope: ErrorScope.PROMPT, + type: ErrorType.USER, + severity: 'error', + context: { name, guidance }, + }, + ]); + } + + /** Duplicate prompt name validation error */ + static alreadyExists(name: string) { + return new DextoValidationError([ + { + code: PromptErrorCode.PROMPT_ALREADY_EXISTS, + message: `Prompt already exists: ${name}`, + scope: ErrorScope.PROMPT, + type: ErrorType.USER, + severity: 'error', + context: { name }, + }, + ]); + } + + /** + * Prompt resolved to empty content + */ + static emptyResolvedContent(name: string) { + return new DextoRuntimeError( + PromptErrorCode.PROMPT_EMPTY_CONTENT, + ErrorScope.PROMPT, + ErrorType.NOT_FOUND, + `Prompt resolved to empty content: ${name}`, + { name } + ); + } + + /** + * Prompts config validation failed + */ + static validationFailed(details: string) { + return new DextoValidationError([ + { + code: PromptErrorCode.PROMPT_CONFIG_INVALID, + message: `Invalid prompts configuration: ${details}`, + scope: ErrorScope.PROMPT, + type: ErrorType.USER, + severity: 'error', + context: { details }, + }, + ]); + } +} diff --git a/dexto/packages/core/src/prompts/index.ts b/dexto/packages/core/src/prompts/index.ts new file mode 100644 index 00000000..ff5ab9a6 --- /dev/null +++ b/dexto/packages/core/src/prompts/index.ts @@ -0,0 +1,23 @@ +export { PromptManager } from './prompt-manager.js'; +export { MCPPromptProvider } from './providers/mcp-prompt-provider.js'; +export { CustomPromptProvider } from './providers/custom-prompt-provider.js'; +export type { CreateCustomPromptInput } from './providers/custom-prompt-provider.js'; +export { PromptError } from './errors.js'; +export { PromptsSchema, InlinePromptSchema, FilePromptSchema } from './schemas.js'; +export type { + ValidatedPromptsConfig, + ValidatedInlinePrompt, + ValidatedFilePrompt, + ValidatedPrompt, + PromptsConfig, +} from './schemas.js'; +export type { + PromptInfo, + PromptSet, + PromptProvider, + PromptArgument, + PromptDefinition, + ResolvedPromptResult, +} from './types.js'; +export { flattenPromptResult, normalizePromptArgs, appendContext } from './utils.js'; +export type { FlattenedPromptResult } from './utils.js'; diff --git a/dexto/packages/core/src/prompts/name-validation.test.ts b/dexto/packages/core/src/prompts/name-validation.test.ts new file mode 100644 index 00000000..4af9af66 --- /dev/null +++ b/dexto/packages/core/src/prompts/name-validation.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; +import { + assertValidPromptName, + isValidPromptName, + PROMPT_NAME_GUIDANCE, +} from './name-validation.js'; + +describe('prompt name validation helpers', () => { + it('accepts valid kebab-case names', () => { + expect(isValidPromptName('code-review')).toBe(true); + expect(isValidPromptName('debug')).toBe(true); + expect(() => assertValidPromptName('explain-more')).not.toThrow(); + }); + + it('rejects names with uppercase letters or spaces', () => { + expect(isValidPromptName('CodeReview')).toBe(false); + expect(isValidPromptName('code review')).toBe(false); + expect(isValidPromptName('code_review')).toBe(false); + expect(() => assertValidPromptName('Code Review')).toThrowError(PROMPT_NAME_GUIDANCE); + }); + + it('rejects names with leading or trailing hyphens', () => { + expect(isValidPromptName('-code')).toBe(false); + expect(isValidPromptName('code-')).toBe(false); + expect(isValidPromptName('code--review')).toBe(false); + }); +}); diff --git a/dexto/packages/core/src/prompts/name-validation.ts b/dexto/packages/core/src/prompts/name-validation.ts new file mode 100644 index 00000000..1b7154e3 --- /dev/null +++ b/dexto/packages/core/src/prompts/name-validation.ts @@ -0,0 +1,26 @@ +import { PromptError } from './errors.js'; + +export const PROMPT_NAME_REGEX = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; + +export const PROMPT_NAME_GUIDANCE = + 'kebab-case (lowercase letters and numbers separated by single hyphens)'; + +export interface PromptNameValidationOptions { + context?: string; + hint?: string; +} + +export function isValidPromptName(name: string): boolean { + return PROMPT_NAME_REGEX.test(name); +} + +export function assertValidPromptName( + name: string, + options: PromptNameValidationOptions = {} +): void { + if (isValidPromptName(name)) { + return; + } + + throw PromptError.invalidName(name, PROMPT_NAME_GUIDANCE, options.context, options.hint); +} diff --git a/dexto/packages/core/src/prompts/prompt-manager.test.ts b/dexto/packages/core/src/prompts/prompt-manager.test.ts new file mode 100644 index 00000000..be1a9485 --- /dev/null +++ b/dexto/packages/core/src/prompts/prompt-manager.test.ts @@ -0,0 +1,162 @@ +import { describe, test, expect } from 'vitest'; +import { PromptManager } from './prompt-manager.js'; +import type { PromptDefinition } from './types.js'; +import { createSilentMockLogger } from '../logger/v2/test-utils.js'; + +const mockLogger = createSilentMockLogger(); + +function makeFakeMCPManager(capture: { lastArgs?: any }) { + const def: PromptDefinition = { + name: 'analyze-metrics', + description: 'Analyze metrics', + arguments: [ + { name: 'metric_type', required: true }, + { name: 'time_period', required: false }, + ], + }; + return { + getAllPromptMetadata() { + return [{ promptName: 'analyze-metrics', serverName: 'demo', definition: def }]; + }, + getPromptMetadata(_name: string) { + return def; + }, + async getPrompt(name: string, args?: Record) { + capture.lastArgs = args; + return { + messages: [{ role: 'user', content: { type: 'text', text: `Prompt: ${name}` } }], + } as any; + }, + } as any; +} + +describe('PromptManager MCP args mapping/filtering', () => { + test('maps positional to named and filters internal keys for MCP', async () => { + const capture: any = {}; + const fakeMCP = makeFakeMCPManager(capture); + const resourceManagerStub = { getBlobStore: () => undefined } as any; + const agentConfig: any = { prompts: [] }; + const eventBus: any = { on: () => {}, emit: () => {} }; + const dbStub: any = { + connect: async () => {}, + list: async () => [], + get: async () => undefined, + }; + + const pm = new PromptManager( + fakeMCP, + resourceManagerStub, + agentConfig, + eventBus, + dbStub, + mockLogger + ); + await pm.initialize(); + await pm.getPrompt('analyze-metrics', { + _positional: ['users', 'Q4 2024'], + _context: 'ignore-me', + extraneous: 'should-drop', + } as any); + + expect(capture.lastArgs).toEqual({ metric_type: 'users', time_period: 'Q4 2024' }); + }); +}); + +describe('PromptManager getPromptDefinition', () => { + test('returns context field from config prompts', async () => { + const fakeMCP = { + getAllPromptMetadata() { + return []; + }, + getPromptMetadata() { + return undefined; + }, + async getPrompt() { + return { messages: [] }; + }, + } as any; + const resourceManagerStub = { getBlobStore: () => undefined } as any; + const agentConfig: any = { + prompts: [ + { + type: 'inline', + id: 'fork-skill', + prompt: 'A skill with fork context', + description: 'Test fork skill', + context: 'fork', + }, + ], + }; + const eventBus: any = { on: () => {}, emit: () => {} }; + const dbStub: any = { + connect: async () => {}, + list: async () => [], + get: async () => undefined, + }; + + const pm = new PromptManager( + fakeMCP, + resourceManagerStub, + agentConfig, + eventBus, + dbStub, + mockLogger + ); + await pm.initialize(); + const def = await pm.getPromptDefinition('config:fork-skill'); + + expect(def).toMatchObject({ + name: 'config:fork-skill', + description: 'Test fork skill', + context: 'fork', + }); + }); + + test('returns undefined context when not specified', async () => { + const fakeMCP = { + getAllPromptMetadata() { + return []; + }, + getPromptMetadata() { + return undefined; + }, + async getPrompt() { + return { messages: [] }; + }, + } as any; + const resourceManagerStub = { getBlobStore: () => undefined } as any; + const agentConfig: any = { + prompts: [ + { + type: 'inline', + id: 'inline-skill', + prompt: 'A skill without context', + description: 'Test inline skill', + }, + ], + }; + const eventBus: any = { on: () => {}, emit: () => {} }; + const dbStub: any = { + connect: async () => {}, + list: async () => [], + get: async () => undefined, + }; + + const pm = new PromptManager( + fakeMCP, + resourceManagerStub, + agentConfig, + eventBus, + dbStub, + mockLogger + ); + await pm.initialize(); + const def = await pm.getPromptDefinition('config:inline-skill'); + + expect(def).toMatchObject({ + name: 'config:inline-skill', + description: 'Test inline skill', + }); + expect(def?.context).toBeUndefined(); + }); +}); diff --git a/dexto/packages/core/src/prompts/prompt-manager.ts b/dexto/packages/core/src/prompts/prompt-manager.ts new file mode 100644 index 00000000..9824e896 --- /dev/null +++ b/dexto/packages/core/src/prompts/prompt-manager.ts @@ -0,0 +1,603 @@ +import type { MCPManager } from '../mcp/manager.js'; +import type { PromptSet, PromptProvider, PromptInfo, ResolvedPromptResult } from './types.js'; +import type { GetPromptResult } from '@modelcontextprotocol/sdk/types.js'; +import type { ValidatedAgentConfig } from '../agent/schemas.js'; +import type { PromptsConfig } from './schemas.js'; +import type { AgentEventBus } from '../events/index.js'; +import { MCPPromptProvider } from './providers/mcp-prompt-provider.js'; +import { ConfigPromptProvider } from './providers/config-prompt-provider.js'; +import { + CustomPromptProvider, + type CreateCustomPromptInput, +} from './providers/custom-prompt-provider.js'; +import { PromptError } from './errors.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import type { ResourceManager } from '../resources/manager.js'; +import type { Database } from '../storage/database/types.js'; +import { normalizePromptArgs, flattenPromptResult } from './utils.js'; + +interface PromptCacheEntry { + providerName: string; + providerPromptName: string; + originalName: string; + info: PromptInfo; +} + +export class PromptManager { + private providers: Map = new Map(); + private configProvider: ConfigPromptProvider; + private promptIndex: Map | undefined; + private aliasMap: Map = new Map(); + private buildPromise: Promise | null = null; + private logger: IDextoLogger; + + constructor( + mcpManager: MCPManager, + resourceManager: ResourceManager, + agentConfig: ValidatedAgentConfig, + private readonly eventBus: AgentEventBus, + private readonly database: Database, + logger: IDextoLogger + ) { + this.logger = logger.createChild(DextoLogComponent.PROMPT); + this.configProvider = new ConfigPromptProvider(agentConfig, this.logger); + this.providers.set('mcp', new MCPPromptProvider(mcpManager, this.logger)); + this.providers.set('config', this.configProvider); + this.providers.set( + 'custom', + new CustomPromptProvider(this.database, resourceManager, this.logger) + ); + + this.logger.debug( + `PromptManager initialized with providers: ${Array.from(this.providers.keys()).join(', ')}` + ); + + const refresh = async (reason: string) => { + this.logger.debug(`PromptManager refreshing due to: ${reason}`); + await this.refresh(); + }; + + this.eventBus.on('mcp:server-connected', async (p) => { + if (p.success) { + await refresh(`mcpServerConnected:${p.name}`); + } + }); + this.eventBus.on('mcp:server-removed', async (p) => { + await refresh(`mcpServerRemoved:${p.serverName}`); + }); + this.eventBus.on('mcp:server-updated', async (p) => { + await refresh(`mcpServerUpdated:${p.serverName}`); + }); + + // Listen for MCP notifications for surgical updates + this.eventBus.on('mcp:prompts-list-changed', async (p) => { + await this.updatePromptsForServer(p.serverName, p.prompts); + this.logger.debug( + `🔄 Surgically updated prompts for server '${p.serverName}': [${p.prompts.join(', ')}]` + ); + }); + } + + async initialize(): Promise { + await this.ensureCache(); + this.logger.debug('PromptManager initialization complete'); + } + + async list(): Promise { + await this.ensureCache(); + const index = this.promptIndex ?? new Map(); + const result: PromptSet = {}; + for (const [key, entry] of index.entries()) { + result[key] = entry.info; + } + return result; + } + + async has(name: string): Promise { + const entry = await this.findEntry(name); + return entry !== undefined; + } + + async getPromptDefinition(name: string): Promise { + const entry = await this.findEntry(name); + if (!entry) return null; + const { info } = entry; + return { + name: info.name, + ...(info.title && { title: info.title }), + ...(info.description && { description: info.description }), + ...(info.arguments && { arguments: info.arguments }), + // Claude Code compatibility fields + ...(info.disableModelInvocation !== undefined && { + disableModelInvocation: info.disableModelInvocation, + }), + ...(info.userInvocable !== undefined && { userInvocable: info.userInvocable }), + ...(info.allowedTools !== undefined && { allowedTools: info.allowedTools }), + ...(info.model !== undefined && { model: info.model }), + ...(info.context !== undefined && { context: info.context }), + ...(info.agent !== undefined && { agent: info.agent }), + }; + } + + /** + * List prompts that should appear in the CLI slash command menu. + * Filters out prompts with `userInvocable: false`. + * These prompts are intended for user invocation via `/` commands. + */ + async listUserInvocablePrompts(): Promise { + await this.ensureCache(); + const index = this.promptIndex ?? new Map(); + const result: PromptSet = {}; + for (const [key, entry] of index.entries()) { + // Include prompt if userInvocable is not explicitly set to false + if (entry.info.userInvocable !== false) { + result[key] = entry.info; + } + } + return result; + } + + /** + * List prompts that can be auto-invoked by the LLM. + * Filters out prompts with `disableModelInvocation: true`. + * These prompts should appear in the system prompt as available skills. + */ + async listAutoInvocablePrompts(): Promise { + await this.ensureCache(); + const index = this.promptIndex ?? new Map(); + const result: PromptSet = {}; + for (const [key, entry] of index.entries()) { + // Include prompt if disableModelInvocation is not explicitly set to true + if (entry.info.disableModelInvocation !== true) { + result[key] = entry.info; + } + } + return result; + } + + /** + * Retrieve a prompt from the appropriate provider. + * + * Responsibilities: + * - Resolve the correct provider by prompt name (post-cache lookup) + * - Map positional arguments (`args._positional: string[]`) to named arguments based + * on the prompt's declared `arguments` schema (order-based mapping) + * - Forward `_context` and any named args to the provider + * + * Mapping rules: + * - If `PromptInfo.arguments` is defined (e.g., MCP or file prompts with `argument-hint:`), + * then each position in `_positional` fills the corresponding named arg if not already set. + * - Named arguments already present in `args` are not overwritten by positional tokens. + */ + async getPrompt(name: string, args?: Record): Promise { + const entry = await this.findEntry(name); + if (!entry) { + throw PromptError.notFound(name); + } + + const provider = this.providers.get(entry.providerName); + if (!provider) { + throw PromptError.providerNotFound(entry.providerName); + } + + // Map positional arguments to named arguments based on prompt's argument schema + // This bridges the gap between user input (positional, like $1 $2) + // and MCP protocol expectations (named arguments like { report_type: "metrics" }) + let finalArgs = args; + if (args?._positional && Array.isArray(args._positional) && args._positional.length > 0) { + const promptArgs = entry.info.arguments; + if (promptArgs && promptArgs.length > 0) { + finalArgs = { ...args }; + const positionalArgs = args._positional as unknown[]; + // Map positional args to named args based on the prompt's argument order + promptArgs.forEach((argDef, index) => { + if (index < positionalArgs.length && !finalArgs![argDef.name]) { + // Only set if not already provided as a named argument + const value = positionalArgs[index]; + finalArgs![argDef.name] = typeof value === 'string' ? value : String(value); + } + }); + } + } + + // Provider-specific argument filtering: + // - MCP: pass only declared arguments (strip internal keys and unknowns) + // - Others: pass through (file/custom use _positional/_context semantics) + let providerArgs: Record | undefined = finalArgs; + if (entry.providerName === 'mcp') { + const declared = new Set((entry.info.arguments ?? []).map((a) => a.name)); + const filtered: Record = {}; + for (const [key, value] of Object.entries(finalArgs ?? {})) { + if (key.startsWith('_')) continue; // strip internal keys + if (declared.size === 0 || declared.has(key)) { + filtered[key] = value; + } + } + providerArgs = filtered; + } + + return await provider.getPrompt(entry.providerPromptName, providerArgs); + } + + async resolvePromptKey(nameOrAlias: string): Promise { + await this.ensureCache(); + if (!this.promptIndex) return null; + + if (this.promptIndex.has(nameOrAlias)) { + return nameOrAlias; + } + + const normalized = nameOrAlias.startsWith('/') ? nameOrAlias.slice(1) : nameOrAlias; + const aliasMatch = this.aliasMap.get(nameOrAlias) ?? this.aliasMap.get(normalized); + return aliasMatch ?? null; + } + + async createCustomPrompt(input: CreateCustomPromptInput): Promise { + const provider = this.providers.get('custom'); + if (!provider || !(provider instanceof CustomPromptProvider)) { + throw PromptError.providerNotFound('custom'); + } + const prompt = await provider.createPrompt(input); + await this.refresh(); + return prompt; + } + + async deleteCustomPrompt(name: string): Promise { + const provider = this.providers.get('custom'); + if (!provider || !(provider instanceof CustomPromptProvider)) { + throw PromptError.providerNotFound('custom'); + } + await provider.deletePrompt(name); + await this.refresh(); + } + + /** + * Resolves a prompt to its text content with all arguments applied. + * This is a high-level method that handles: + * - Prompt key resolution (resolving aliases) + * - Argument normalization (including special `_context` field) + * - Passing `_context` through to providers so they can decide whether to append it + * (e.g., file prompts without placeholders will append `Context: ...`) + * - Prompt execution and flattening + * - Returning per-prompt overrides (allowedTools, model) for the invoker to apply + * + * @param name The prompt name or alias + * @param options Optional configuration for prompt resolution + * @returns Promise resolving to the resolved text, resource URIs, and optional overrides + */ + async resolvePrompt( + name: string, + options: { + context?: string; + args?: Record; + } = {} + ): Promise { + // Build args from options + const args: Record = { ...options.args }; + // Preserve `_context` on args for providers that need to decide whether to append it + if (options.context?.trim()) args._context = options.context.trim(); + + // Resolve provided name to a valid prompt key using promptManager + const resolvedName = (await this.resolvePromptKey(name)) ?? name; + + // Get prompt definition to extract per-prompt overrides + const promptDef = await this.getPromptDefinition(resolvedName); + + // Normalize args (converts to strings, extracts context) + const normalized = normalizePromptArgs(args); + + // Providers need `_context` to decide whether to append it (e.g., file prompts without placeholders) + const providerArgs = normalized.context + ? { ...normalized.args, _context: normalized.context } + : normalized.args; + + // Get and flatten the prompt result + // Note: PromptManager handles positional-to-named argument mapping internally + const promptResult = await this.getPrompt(resolvedName, providerArgs); + const flattened = flattenPromptResult(promptResult); + + // Context handling is done by the prompt providers themselves + // (they check for placeholders and decide whether to append context) + + // Validate result + if (!flattened.text && flattened.resourceUris.length === 0) { + throw PromptError.emptyResolvedContent(resolvedName); + } + + return { + text: flattened.text, + resources: flattened.resourceUris, + // Include per-prompt overrides from prompt definition + ...(promptDef?.allowedTools && { allowedTools: promptDef.allowedTools }), + ...(promptDef?.model && { model: promptDef.model }), + ...(promptDef?.context && { context: promptDef.context }), + ...(promptDef?.agent && { agent: promptDef.agent }), + }; + } + + async refresh(): Promise { + this.promptIndex = undefined; + this.aliasMap.clear(); + for (const provider of this.providers.values()) { + provider.invalidateCache(); + } + await this.ensureCache(); + this.logger.info('PromptManager refreshed'); + } + + /** + * Updates the config prompts at runtime. + * Call this after modifying the agent config file to reflect new prompts. + */ + updateConfigPrompts(prompts: PromptsConfig): void { + this.configProvider.updatePrompts(prompts); + this.promptIndex = undefined; + this.aliasMap.clear(); + this.logger.debug('Config prompts updated'); + } + + /** + * Surgically update prompts for a specific MCP server instead of full cache rebuild + */ + private async updatePromptsForServer(serverName: string, _newPrompts: string[]): Promise { + await this.ensureCache(); + if (!this.promptIndex) return; + + // Remove existing prompts from this server + this.removePromptsForServer(serverName); + + // Add new prompts from this server + const mcpProvider = this.providers.get('mcp'); + if (mcpProvider) { + try { + const { prompts } = await mcpProvider.listPrompts(); + const serverPrompts = prompts.filter( + (p) => + p.metadata && + typeof p.metadata === 'object' && + 'serverName' in p.metadata && + p.metadata.serverName === serverName + ); + + // Compute displayName counts including existing prompts for collision detection + const displayNameCounts = new Map(); + for (const entry of this.promptIndex.values()) { + const displayName = entry.info.displayName || entry.info.name; + displayNameCounts.set( + displayName, + (displayNameCounts.get(displayName) || 0) + 1 + ); + } + for (const prompt of serverPrompts) { + const displayName = prompt.displayName || prompt.name; + displayNameCounts.set( + displayName, + (displayNameCounts.get(displayName) || 0) + 1 + ); + } + + // Compute commandName and insert each new prompt + for (const prompt of serverPrompts) { + const displayName = prompt.displayName || prompt.name; + const hasCollision = (displayNameCounts.get(displayName) || 0) > 1; + prompt.commandName = hasCollision + ? `${prompt.source}:${displayName}` + : displayName; + this.insertPrompt(this.promptIndex, this.aliasMap, 'mcp', prompt); + } + } catch (error) { + this.logger.debug( + `Failed to get updated prompts for server '${serverName}': ${error}` + ); + } + } + } + + /** + * Remove all prompts from a specific server + */ + private removePromptsForServer(serverName: string): void { + if (!this.promptIndex) return; + + const keysToRemove: string[] = []; + for (const [key, entry] of this.promptIndex.entries()) { + if ( + entry.providerName === 'mcp' && + entry.info.metadata && + typeof entry.info.metadata === 'object' && + 'serverName' in entry.info.metadata && + entry.info.metadata.serverName === serverName + ) { + keysToRemove.push(key); + } + } + + // Remove from index and aliases + for (const key of keysToRemove) { + const entry = this.promptIndex.get(key); + if (entry) { + this.promptIndex.delete(key); + // Remove aliases that point to this key + for (const [aliasKey, aliasValue] of Array.from(this.aliasMap.entries())) { + if (aliasValue === key) { + this.aliasMap.delete(aliasKey); + } + } + } + } + } + + private sanitizePromptInfo(prompt: PromptInfo, providerName: string): PromptInfo { + const metadata = { ...(prompt.metadata ?? {}) } as Record; + delete metadata.content; + delete metadata.prompt; + // Keep filePath - needed for prompt deletion + delete metadata.messages; + + if (!metadata.originalName) { + metadata.originalName = prompt.name; + } + metadata.provider = providerName; + + const sanitized: PromptInfo = { ...prompt }; + if (Object.keys(metadata).length > 0) { + sanitized.metadata = metadata; + } else { + delete sanitized.metadata; + } + return sanitized; + } + + private async ensureCache(): Promise { + if (this.promptIndex) { + return; + } + if (this.buildPromise) { + await this.buildPromise; + return; + } + this.buildPromise = this.buildCache(); + try { + await this.buildPromise; + } finally { + this.buildPromise = null; + } + } + + private async buildCache(): Promise { + const index = new Map(); + const aliases = new Map(); + + // Phase 1: Collect all prompts from providers + const collectedPrompts: Array<{ providerName: string; prompt: PromptInfo }> = []; + + for (const [providerName, provider] of this.providers) { + try { + const { prompts } = await provider.listPrompts(); + for (const prompt of prompts) { + collectedPrompts.push({ providerName, prompt }); + } + } catch (error) { + this.logger.error( + `Failed to get prompts from ${providerName} provider: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + // Phase 2: Detect displayName collisions + const displayNameCounts = new Map(); + for (const { prompt } of collectedPrompts) { + const displayName = prompt.displayName || prompt.name; + displayNameCounts.set(displayName, (displayNameCounts.get(displayName) || 0) + 1); + } + + // Phase 3: Compute commandName for each prompt (collision-resolved) + for (const { prompt } of collectedPrompts) { + const displayName = prompt.displayName || prompt.name; + const hasCollision = (displayNameCounts.get(displayName) || 0) > 1; + + // Compute unique command name: use source prefix only when collision exists + prompt.commandName = hasCollision ? `${prompt.source}:${displayName}` : displayName; + } + + // Phase 4: Build index and aliases + for (const { providerName, prompt } of collectedPrompts) { + this.insertPrompt(index, aliases, providerName, prompt); + } + + this.promptIndex = index; + this.aliasMap = aliases; + + if (index.size > 0) { + const sample = Array.from(index.keys()).slice(0, 5); + this.logger.debug( + `📋 Prompt discovery: ${index.size} prompts. Sample: ${sample.join(', ')}` + ); + } + } + + private insertPrompt( + index: Map, + aliases: Map, + providerName: string, + prompt: PromptInfo + ): void { + const providerPromptName = prompt.name; + const prepared = this.sanitizePromptInfo(prompt, providerName); + let key = providerPromptName; + const originalName = providerPromptName; + + if (index.has(key)) { + const existing = index.get(key)!; + index.delete(key); + + const existingKey = `${existing.providerName}:${existing.originalName}`; + const updatedExisting: PromptCacheEntry = { + ...existing, + info: + existing.info.name === existingKey + ? existing.info + : { ...existing.info, name: existingKey }, + }; + index.set(existingKey, updatedExisting); + aliases.set(existing.originalName, existingKey); + key = `${providerName}:${originalName}`; + } + + const entryInfo = + prepared.name === key ? prepared : ({ ...prepared, name: key } as PromptInfo); + const entry: PromptCacheEntry = { + providerName, + providerPromptName, + originalName, + info: entryInfo, + }; + + index.set(key, entry); + aliases.set(originalName, key); + + const metadata = entryInfo.metadata as Record | undefined; + if (metadata) { + const aliasCandidates = new Set(); + if (typeof metadata.originalName === 'string') { + aliasCandidates.add(metadata.originalName); + } + if (typeof metadata.command === 'string') { + const command = metadata.command as string; + aliasCandidates.add(command); + if (command.startsWith('/')) { + aliasCandidates.add(command.slice(1)); + } + } + + for (const candidate of aliasCandidates) { + if (candidate && !aliases.has(candidate)) { + aliases.set(candidate, key); + } + } + } + + // Add commandName as alias for resolution (allows resolving by user-facing command) + if (entryInfo.commandName && !aliases.has(entryInfo.commandName)) { + aliases.set(entryInfo.commandName, key); + } + } + + private async findEntry(name: string): Promise { + await this.ensureCache(); + if (!this.promptIndex) return undefined; + + if (this.promptIndex.has(name)) { + return this.promptIndex.get(name); + } + + const normalized = name.startsWith('/') ? name.slice(1) : name; + const alias = this.aliasMap.get(name) ?? this.aliasMap.get(normalized); + if (alias && this.promptIndex.has(alias)) { + return this.promptIndex.get(alias); + } + + return undefined; + } +} diff --git a/dexto/packages/core/src/prompts/providers/__fixtures__/claude-skill.md b/dexto/packages/core/src/prompts/providers/__fixtures__/claude-skill.md new file mode 100644 index 00000000..01d69fcc --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/__fixtures__/claude-skill.md @@ -0,0 +1,10 @@ +--- +name: test-skill +description: A test skill using Claude Code SKILL.md format with name field. +--- + +# Test Skill + +## Instructions + +This is a test skill file using the name: field instead of id: in frontmatter. diff --git a/dexto/packages/core/src/prompts/providers/__fixtures__/full-frontmatter.md b/dexto/packages/core/src/prompts/providers/__fixtures__/full-frontmatter.md new file mode 100644 index 00000000..08a2e2d8 --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/__fixtures__/full-frontmatter.md @@ -0,0 +1,14 @@ +--- +id: test-prompt +title: Test Prompt Title +description: A test prompt with full frontmatter +category: testing +priority: 10 +argument-hint: "[style] [length?]" +--- + +# Test Prompt + +This is the prompt content. + +Please analyze the following with $ARGUMENTS style. diff --git a/dexto/packages/core/src/prompts/providers/__fixtures__/invalid-name.md b/dexto/packages/core/src/prompts/providers/__fixtures__/invalid-name.md new file mode 100644 index 00000000..6b320d44 --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/__fixtures__/invalid-name.md @@ -0,0 +1,6 @@ +--- +id: Invalid_Name_With_Underscores +title: Invalid Name Test +--- + +This prompt has an invalid name that should be rejected. diff --git a/dexto/packages/core/src/prompts/providers/__fixtures__/minimal.md b/dexto/packages/core/src/prompts/providers/__fixtures__/minimal.md new file mode 100644 index 00000000..1f95bb21 --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/__fixtures__/minimal.md @@ -0,0 +1,5 @@ +# Minimal Prompt + +This is a minimal prompt without frontmatter. + +The title should be extracted from the heading above. diff --git a/dexto/packages/core/src/prompts/providers/__fixtures__/my-test-skill/SKILL.md b/dexto/packages/core/src/prompts/providers/__fixtures__/my-test-skill/SKILL.md new file mode 100644 index 00000000..0e2392b8 --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/__fixtures__/my-test-skill/SKILL.md @@ -0,0 +1,8 @@ +--- +description: Test skill using directory name as id +context: fork +--- + +# My Test Skill + +This skill should use the parent directory name (my-test-skill) as its id. diff --git a/dexto/packages/core/src/prompts/providers/__fixtures__/partial-frontmatter.md b/dexto/packages/core/src/prompts/providers/__fixtures__/partial-frontmatter.md new file mode 100644 index 00000000..66358d56 --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/__fixtures__/partial-frontmatter.md @@ -0,0 +1,8 @@ +--- +id: partial-test +description: Only id and description provided +--- + +# Partial Frontmatter + +Content with only some frontmatter fields. diff --git a/dexto/packages/core/src/prompts/providers/__fixtures__/skill-with-tools.md b/dexto/packages/core/src/prompts/providers/__fixtures__/skill-with-tools.md new file mode 100644 index 00000000..be83c00d --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/__fixtures__/skill-with-tools.md @@ -0,0 +1,9 @@ +--- +name: skill-with-tools +description: A skill with allowed-tools for testing Claude Code tool name normalization. +allowed-tools: [bash, Read, WRITE, edit, custom--keep_as_is] +--- + +# Skill With Tools + +This skill has allowed-tools that should be normalized to Dexto tool names. diff --git a/dexto/packages/core/src/prompts/providers/config-prompt-provider.test.ts b/dexto/packages/core/src/prompts/providers/config-prompt-provider.test.ts new file mode 100644 index 00000000..1fa7a705 --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/config-prompt-provider.test.ts @@ -0,0 +1,654 @@ +import { describe, test, expect } from 'vitest'; +import { ConfigPromptProvider } from './config-prompt-provider.js'; +import type { ValidatedAgentConfig } from '../../agent/schemas.js'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; +import { createSilentMockLogger } from '../../logger/v2/test-utils.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const FIXTURES_DIR = join(__dirname, '__fixtures__'); + +const mockLogger = createSilentMockLogger(); + +function makeAgentConfig(prompts: any[]): ValidatedAgentConfig { + return { prompts } as ValidatedAgentConfig; +} + +describe('ConfigPromptProvider', () => { + describe('inline prompts', () => { + test('lists inline prompts correctly', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'test-prompt', + title: 'Test Prompt', + description: 'A test prompt', + prompt: 'This is the prompt content', + category: 'testing', + priority: 1, + showInStarters: true, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.listPrompts(); + + expect(result.prompts).toHaveLength(1); + expect(result.prompts[0]).toMatchObject({ + name: 'config:test-prompt', + title: 'Test Prompt', + description: 'A test prompt', + source: 'config', + }); + expect(result.prompts[0]?.metadata).toMatchObject({ + type: 'inline', + category: 'testing', + priority: 1, + showInStarters: true, + originalId: 'test-prompt', + }); + }); + + test('gets inline prompt content', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'simple', + title: 'Simple', + description: 'Simple prompt', + prompt: 'Hello world', + category: 'general', + priority: 0, + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.getPrompt('config:simple'); + + expect(result.messages).toHaveLength(1); + expect((result.messages?.[0]?.content as any).text).toBe('Hello world'); + }); + + test('sorts prompts by priority (higher first)', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'low', + prompt: 'Low priority', + priority: 1, + }, + { + type: 'inline', + id: 'high', + prompt: 'High priority', + priority: 10, + }, + { + type: 'inline', + id: 'medium', + prompt: 'Medium priority', + priority: 5, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.listPrompts(); + + expect(result.prompts.map((p) => p.name)).toEqual([ + 'config:high', + 'config:medium', + 'config:low', + ]); + }); + + test('throws error for non-existent prompt', async () => { + const config = makeAgentConfig([]); + const provider = new ConfigPromptProvider(config, mockLogger); + + await expect(provider.getPrompt('config:nonexistent')).rejects.toThrow(); + }); + }); + + describe('applyArguments', () => { + test('appends Context at END when no placeholders', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 's1', + title: 'T', + description: 'D', + prompt: 'Starter content', + category: 'cat', + priority: 1, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const res = await provider.getPrompt('config:s1', { _context: 'CTX' } as any); + const text = (res.messages?.[0]?.content as any).text as string; + + expect(text).toBe('Starter content\n\nContext: CTX'); + }); + + test('does not append when positional placeholders present ($1)', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 's2', + title: 'T', + description: 'D', + prompt: 'Use $1 style', + category: 'cat', + priority: 1, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const res = await provider.getPrompt('config:s2', { + _positional: ['value'], + _context: 'CTX', + } as any); + const text = (res.messages?.[0]?.content as any).text as string; + + // Placeholder expanded, context not appended + expect(text).toBe('Use value style'); + }); + + test('expands $ARGUMENTS and does not append context when placeholders used', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'p1', + prompt: 'Content: $ARGUMENTS', + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const res = await provider.getPrompt('config:p1', { + _positional: ['one', 'two'], + _context: 'should-not-append', + }); + + const text = (res.messages?.[0]?.content as any).text as string; + expect(text).toContain('Content: one two'); + expect(text.includes('should-not-append')).toBe(false); + }); + + test('appends Arguments at END when no placeholders and no context', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'np2', + prompt: 'Alpha', + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const res = await provider.getPrompt('config:np2', { a: '1', b: '2' } as any); + const text = (res.messages?.[0]?.content as any).text as string; + + expect(text).toBe('Alpha\n\nArguments: a: 1, b: 2'); + }); + + test('ignores internal keys starting with underscore', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'test', + prompt: 'Test', + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const res = await provider.getPrompt('config:test', { + _internal: 'ignored', + visible: 'shown', + } as any); + const text = (res.messages?.[0]?.content as any).text as string; + + expect(text).toBe('Test\n\nArguments: visible: shown'); + expect(text).not.toContain('_internal'); + }); + }); + + describe('getPromptDefinition', () => { + test('returns prompt definition for existing prompt', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'def-test', + title: 'Definition Test', + description: 'Test description', + prompt: 'Content', + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const def = await provider.getPromptDefinition('config:def-test'); + + expect(def).toEqual({ + name: 'config:def-test', + title: 'Definition Test', + description: 'Test description', + }); + }); + + test('returns null for non-existent prompt', async () => { + const config = makeAgentConfig([]); + const provider = new ConfigPromptProvider(config, mockLogger); + const def = await provider.getPromptDefinition('config:missing'); + + expect(def).toBeNull(); + }); + }); + + describe('cache management', () => { + test('invalidateCache clears the cache', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'cached', + prompt: 'Cached content', + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + + // First access builds cache + const result1 = await provider.listPrompts(); + expect(result1.prompts).toHaveLength(1); + + // Invalidate + provider.invalidateCache(); + + // Still works after invalidation (rebuilds cache) + const result2 = await provider.listPrompts(); + expect(result2.prompts).toHaveLength(1); + }); + + test('updatePrompts replaces prompts', async () => { + const config1 = makeAgentConfig([ + { + type: 'inline', + id: 'first', + prompt: 'First content', + }, + ]); + + const provider = new ConfigPromptProvider(config1, mockLogger); + let result = await provider.listPrompts(); + expect(result.prompts).toHaveLength(1); + expect(result.prompts[0]?.name).toBe('config:first'); + + // Update prompts + const newPrompts = [ + { + type: 'inline' as const, + id: 'second', + prompt: 'Second content', + }, + { + type: 'inline' as const, + id: 'third', + prompt: 'Third content', + }, + ]; + + provider.updatePrompts(newPrompts); + result = await provider.listPrompts(); + expect(result.prompts).toHaveLength(2); + expect(result.prompts.map((p) => p.name)).toContain('config:second'); + expect(result.prompts.map((p) => p.name)).toContain('config:third'); + }); + }); + + describe('getSource', () => { + test('returns "config"', () => { + const config = makeAgentConfig([]); + const provider = new ConfigPromptProvider(config, mockLogger); + expect(provider.getSource()).toBe('config'); + }); + }); + + describe('file prompts', () => { + test('loads file with full frontmatter', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'full-frontmatter.md'), + showInStarters: true, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.listPrompts(); + + expect(result.prompts).toHaveLength(1); + expect(result.prompts[0]).toMatchObject({ + name: 'config:test-prompt', + title: 'Test Prompt Title', + description: 'A test prompt with full frontmatter', + source: 'config', + }); + expect(result.prompts[0]?.metadata).toMatchObject({ + type: 'file', + category: 'testing', + priority: 10, + showInStarters: true, + originalId: 'test-prompt', + }); + // Should have parsed arguments from argument-hint + expect(result.prompts[0]?.arguments).toEqual([ + { name: 'style', required: true }, + { name: 'length', required: false }, + ]); + }); + + test('loads file without frontmatter (uses filename as id)', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'minimal.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.listPrompts(); + + expect(result.prompts).toHaveLength(1); + expect(result.prompts[0]).toMatchObject({ + name: 'config:minimal', + title: 'Minimal Prompt', // Extracted from # heading + source: 'config', + }); + }); + + test('loads file with partial frontmatter', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'partial-frontmatter.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.listPrompts(); + + expect(result.prompts).toHaveLength(1); + expect(result.prompts[0]).toMatchObject({ + name: 'config:partial-test', + title: 'Partial Frontmatter', // From heading since not in frontmatter + description: 'Only id and description provided', + source: 'config', + }); + }); + + test('loads Claude Code SKILL.md format (uses name: instead of id:)', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'claude-skill.md'), + namespace: 'my-plugin', + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.listPrompts(); + + expect(result.prompts).toHaveLength(1); + expect(result.prompts[0]).toMatchObject({ + // name: field in frontmatter should be used as id + name: 'config:my-plugin:test-skill', + // displayName is now just the skill id (without namespace prefix) + // commandName (computed by PromptManager) handles collision resolution + displayName: 'test-skill', + description: 'A test skill using Claude Code SKILL.md format with name field.', + source: 'config', + }); + }); + + test('gets file prompt content', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'minimal.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.getPrompt('config:minimal'); + + const text = (result.messages?.[0]?.content as any).text as string; + expect(text).toContain('Minimal Prompt'); + expect(text).toContain('minimal prompt without frontmatter'); + }); + + test('skips file with invalid prompt name', async () => { + const warnings: string[] = []; + const warnLogger = { + ...mockLogger, + warn: (msg: string) => warnings.push(msg), + createChild: () => warnLogger, + }; + + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'invalid-name.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, warnLogger as any); + const result = await provider.listPrompts(); + + // Should be skipped due to invalid name + expect(result.prompts).toHaveLength(0); + expect(warnings.some((w) => w.includes('Invalid prompt name'))).toBe(true); + }); + + test('skips non-existent file gracefully', async () => { + const warnings: string[] = []; + const warnLogger = { + ...mockLogger, + warn: (msg: string) => warnings.push(msg), + createChild: () => warnLogger, + }; + + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'does-not-exist.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, warnLogger as any); + const result = await provider.listPrompts(); + + expect(result.prompts).toHaveLength(0); + expect(warnings.some((w) => w.includes('not found'))).toBe(true); + }); + + test('mixed inline and file prompts', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'inline-one', + prompt: 'Inline content', + priority: 5, + }, + { + type: 'file', + file: join(FIXTURES_DIR, 'full-frontmatter.md'), + showInStarters: true, + }, + { + type: 'inline', + id: 'inline-two', + prompt: 'Another inline', + priority: 1, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.listPrompts(); + + expect(result.prompts).toHaveLength(3); + // Sorted by priority: file (10), inline-one (5), inline-two (1) + expect(result.prompts.map((p) => p.name)).toEqual([ + 'config:test-prompt', // priority 10 from file + 'config:inline-one', // priority 5 + 'config:inline-two', // priority 1 + ]); + }); + + test('applies arguments to file prompt content', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'full-frontmatter.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.getPrompt('config:test-prompt', { + _positional: ['detailed'], + }); + + const text = (result.messages?.[0]?.content as any).text as string; + // $ARGUMENTS should be expanded + expect(text).toContain('detailed style'); + }); + + test('SKILL.md uses parent directory name as id (Claude Code convention)', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'my-test-skill', 'SKILL.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.listPrompts(); + + expect(result.prompts).toHaveLength(1); + expect(result.prompts[0]).toMatchObject({ + // Should use directory name "my-test-skill" as id, not "SKILL" + name: 'config:my-test-skill', + displayName: 'my-test-skill', + description: 'Test skill using directory name as id', + source: 'config', + }); + }); + + test('parses context: fork from SKILL.md frontmatter', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'my-test-skill', 'SKILL.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const result = await provider.listPrompts(); + + expect(result.prompts).toHaveLength(1); + // context should be parsed from frontmatter + expect(result.prompts[0]?.context).toBe('fork'); + }); + + test('getPromptDefinition includes context field', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'my-test-skill', 'SKILL.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const def = await provider.getPromptDefinition('config:my-test-skill'); + + expect(def).toMatchObject({ + name: 'config:my-test-skill', + context: 'fork', + }); + }); + }); + + describe('Claude Code tool name normalization', () => { + test('normalizes Claude Code tool names to Dexto format (case-insensitive)', async () => { + const config = makeAgentConfig([ + { + type: 'file', + file: join(FIXTURES_DIR, 'skill-with-tools.md'), + showInStarters: false, + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const def = await provider.getPromptDefinition('config:skill-with-tools'); + + // Should normalize: bash, Read, WRITE, edit -> Dexto names + // Should preserve: custom--keep_as_is (already Dexto format) + expect(def).not.toBeNull(); + expect(def!.allowedTools).toEqual([ + 'custom--bash_exec', + 'custom--read_file', + 'custom--write_file', + 'custom--edit_file', + 'custom--keep_as_is', + ]); + }); + + test('normalizes inline prompt allowed-tools', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'inline-with-tools', + title: 'Inline With Tools', + prompt: 'Test prompt', + 'allowed-tools': ['BASH', 'Grep', 'glob', 'mcp--some_server'], + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const def = await provider.getPromptDefinition('config:inline-with-tools'); + + expect(def).not.toBeNull(); + expect(def!.allowedTools).toEqual([ + 'custom--bash_exec', + 'custom--grep_content', + 'custom--glob_files', + 'mcp--some_server', // MCP tools pass through unchanged + ]); + }); + + test('preserves unknown tool names unchanged', async () => { + const config = makeAgentConfig([ + { + type: 'inline', + id: 'unknown-tools', + title: 'Unknown Tools', + prompt: 'Test prompt', + 'allowed-tools': ['unknown_tool', 'another-custom', 'internal--foo'], + }, + ]); + + const provider = new ConfigPromptProvider(config, mockLogger); + const def = await provider.getPromptDefinition('config:unknown-tools'); + + expect(def).not.toBeNull(); + expect(def!.allowedTools).toEqual(['unknown_tool', 'another-custom', 'internal--foo']); + }); + }); +}); diff --git a/dexto/packages/core/src/prompts/providers/config-prompt-provider.ts b/dexto/packages/core/src/prompts/providers/config-prompt-provider.ts new file mode 100644 index 00000000..edf9a9a5 --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/config-prompt-provider.ts @@ -0,0 +1,555 @@ +import type { PromptProvider, PromptInfo, PromptDefinition, PromptListResult } from '../types.js'; +import type { GetPromptResult } from '@modelcontextprotocol/sdk/types.js'; +import type { ValidatedAgentConfig } from '../../agent/schemas.js'; +import type { InlinePrompt, FilePrompt, PromptsConfig } from '../schemas.js'; +import { PromptsSchema } from '../schemas.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { PromptError } from '../errors.js'; +import { expandPlaceholders } from '../utils.js'; +import { assertValidPromptName } from '../name-validation.js'; +import { readFile, realpath } from 'fs/promises'; +import { existsSync } from 'fs'; +import { basename, dirname, relative, sep } from 'path'; + +/** + * Mapping from Claude Code tool names to Dexto tool names. + * Used for .claude/commands/ compatibility. + * + * Claude Code uses short names like "bash", "read", "write" in allowed-tools. + * Dexto uses prefixed names like "custom--bash_exec", "custom--read_file". + * + * Keys are lowercase for case-insensitive lookup. + * + * TODO: Add additional Claude Code tool mappings as needed (e.g., list, search, run, notebook, etc.) + */ +const CLAUDE_CODE_TOOL_MAP: Record = { + // Bash/process tools + bash: 'custom--bash_exec', + + // Filesystem tools + read: 'custom--read_file', + write: 'custom--write_file', + edit: 'custom--edit_file', + glob: 'custom--glob_files', + grep: 'custom--grep_content', + + // Internal tools + task: 'internal--delegate_task', +}; + +/** + * Normalize tool names from Claude Code format to Dexto format. + * Uses case-insensitive lookup for Claude Code tool names. + * Unknown tools are passed through unchanged. + */ +function normalizeAllowedTools(tools: string[]): string[] { + return tools.map((tool) => CLAUDE_CODE_TOOL_MAP[tool.toLowerCase()] ?? tool); +} + +/** + * Config Prompt Provider - Unified provider for prompts from agent configuration + * + * Handles both inline prompts (text defined directly in config) and file-based prompts + * (loaded from markdown files). This replaces the old StarterPromptProvider and + * FilePromptProvider with a single, unified approach. + * + * Prompts with showInStarters: true are displayed as clickable buttons in the WebUI. + */ +export class ConfigPromptProvider implements PromptProvider { + private prompts: PromptsConfig = []; + private promptsCache: PromptInfo[] = []; + private promptContent: Map = new Map(); + private cacheValid: boolean = false; + private logger: IDextoLogger; + + constructor(agentConfig: ValidatedAgentConfig, logger: IDextoLogger) { + this.logger = logger.createChild(DextoLogComponent.PROMPT); + this.prompts = agentConfig.prompts; + this.buildPromptsCache(); + } + + getSource(): string { + return 'config'; + } + + invalidateCache(): void { + this.cacheValid = false; + this.promptsCache = []; + this.promptContent.clear(); + this.logger.debug('ConfigPromptProvider cache invalidated'); + } + + updatePrompts(prompts: PromptsConfig): void { + const result = PromptsSchema.safeParse(prompts); + if (!result.success) { + const errorMsg = result.error.issues.map((i) => i.message).join(', '); + this.logger.error(`Invalid prompts config: ${errorMsg}`); + throw PromptError.validationFailed(errorMsg); + } + this.prompts = result.data; + this.invalidateCache(); + this.buildPromptsCache(); + } + + async listPrompts(_cursor?: string): Promise { + if (!this.cacheValid) { + await this.buildPromptsCache(); + } + + return { + prompts: this.promptsCache, + }; + } + + async getPrompt(name: string, args?: Record): Promise { + if (!this.cacheValid) { + await this.buildPromptsCache(); + } + + const promptInfo = this.promptsCache.find((p) => p.name === name); + if (!promptInfo) { + throw PromptError.notFound(name); + } + + let content = this.promptContent.get(name); + if (!content) { + throw PromptError.missingText(); + } + + // Apply arguments + content = this.applyArguments(content, args); + + return { + description: promptInfo.description, + messages: [ + { + role: 'user', + content: { + type: 'text', + text: content, + }, + }, + ], + }; + } + + async getPromptDefinition(name: string): Promise { + if (!this.cacheValid) { + await this.buildPromptsCache(); + } + + const promptInfo = this.promptsCache.find((p) => p.name === name); + if (!promptInfo) { + return null; + } + + return { + name: promptInfo.name, + ...(promptInfo.title && { title: promptInfo.title }), + ...(promptInfo.description && { description: promptInfo.description }), + ...(promptInfo.arguments && { arguments: promptInfo.arguments }), + // Claude Code compatibility fields + ...(promptInfo.disableModelInvocation !== undefined && { + disableModelInvocation: promptInfo.disableModelInvocation, + }), + ...(promptInfo.userInvocable !== undefined && { + userInvocable: promptInfo.userInvocable, + }), + ...(promptInfo.allowedTools !== undefined && { + allowedTools: promptInfo.allowedTools, + }), + ...(promptInfo.model !== undefined && { model: promptInfo.model }), + ...(promptInfo.context !== undefined && { context: promptInfo.context }), + ...(promptInfo.agent !== undefined && { agent: promptInfo.agent }), + }; + } + + private async buildPromptsCache(): Promise { + const cache: PromptInfo[] = []; + const contentMap = new Map(); + + for (const prompt of this.prompts ?? []) { + try { + if (prompt.type === 'inline') { + const { info, content } = this.processInlinePrompt(prompt); + cache.push(info); + contentMap.set(info.name, content); + } else if (prompt.type === 'file') { + const result = await this.processFilePrompt(prompt); + if (result) { + cache.push(result.info); + contentMap.set(result.info.name, result.content); + } + } + } catch (error) { + this.logger.warn( + `Failed to process prompt: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + // Sort by priority (higher numbers first) + cache.sort((a, b) => { + const priorityA = (a.metadata?.priority as number) ?? 0; + const priorityB = (b.metadata?.priority as number) ?? 0; + return priorityB - priorityA; + }); + + this.promptsCache = cache; + this.promptContent = contentMap; + this.cacheValid = true; + + this.logger.debug(`Cached ${cache.length} config prompts`); + } + + private processInlinePrompt(prompt: InlinePrompt): { + info: PromptInfo; + content: string; + } { + const promptName = `config:${prompt.id}`; + const promptInfo: PromptInfo = { + name: promptName, + displayName: prompt.id, + title: prompt.title, + description: prompt.description, + source: 'config', + // Claude Code compatibility fields + disableModelInvocation: prompt['disable-model-invocation'], + userInvocable: prompt['user-invocable'], + allowedTools: prompt['allowed-tools'] + ? normalizeAllowedTools(prompt['allowed-tools']) + : undefined, + model: prompt.model, + context: prompt.context, + agent: prompt.agent, + metadata: { + type: 'inline', + category: prompt.category, + priority: prompt.priority, + showInStarters: prompt.showInStarters, + originalId: prompt.id, + }, + }; + + return { info: promptInfo, content: prompt.prompt }; + } + + private async processFilePrompt( + prompt: FilePrompt + ): Promise<{ info: PromptInfo; content: string } | null> { + const filePath = prompt.file; + + if (!existsSync(filePath)) { + this.logger.warn(`Prompt file not found: ${filePath}`); + return null; + } + + // Security: Validate file path to prevent symlink escapes + try { + const resolvedDir = await realpath(dirname(filePath)); + const resolvedFile = await realpath(filePath); + + // Check if resolved file is within the expected directory + const rel = relative(resolvedDir, resolvedFile); + if (rel.startsWith('..' + sep) || rel === '..') { + this.logger.warn( + `Skipping prompt file '${filePath}': path traversal attempt detected (resolved outside directory)` + ); + return null; + } + } catch (realpathError) { + this.logger.warn( + `Skipping prompt file '${filePath}': unable to resolve path (${realpathError instanceof Error ? realpathError.message : String(realpathError)})` + ); + return null; + } + + try { + const rawContent = await readFile(filePath, 'utf-8'); + const parsed = this.parseMarkdownPrompt(rawContent, filePath); + + // Validate the parsed prompt name + try { + assertValidPromptName(parsed.id, { + context: `file prompt '${filePath}'`, + hint: "Use kebab-case in the 'id:' frontmatter field or file name.", + }); + } catch (validationError) { + this.logger.warn( + `Invalid prompt name in '${filePath}': ${validationError instanceof Error ? validationError.message : String(validationError)}` + ); + return null; + } + + // Config-level fields override frontmatter values + const disableModelInvocation = + prompt['disable-model-invocation'] ?? parsed.disableModelInvocation; + const userInvocable = prompt['user-invocable'] ?? parsed.userInvocable; + const rawAllowedTools = prompt['allowed-tools'] ?? parsed.allowedTools; + const allowedTools = rawAllowedTools + ? normalizeAllowedTools(rawAllowedTools) + : undefined; + const model = prompt.model ?? parsed.model; + const context = prompt.context ?? parsed.context; + const agent = prompt.agent ?? parsed.agent; + + const displayName = parsed.id; + const promptName = prompt.namespace + ? `config:${prompt.namespace}:${parsed.id}` + : `config:${parsed.id}`; + + const promptInfo: PromptInfo = { + name: promptName, + displayName, + title: parsed.title, + description: parsed.description, + source: 'config', + ...(parsed.arguments && { arguments: parsed.arguments }), + // Claude Code compatibility fields + ...(disableModelInvocation !== undefined && { disableModelInvocation }), + ...(userInvocable !== undefined && { userInvocable }), + ...(allowedTools !== undefined && { allowedTools }), + ...(model !== undefined && { model }), + ...(context !== undefined && { context }), + ...(agent !== undefined && { agent }), + metadata: { + type: 'file', + filePath: filePath, + category: parsed.category, + priority: parsed.priority, + showInStarters: prompt.showInStarters, + originalId: parsed.id, + ...(prompt.namespace && { namespace: prompt.namespace }), + }, + }; + + return { info: promptInfo, content: parsed.content }; + } catch (error) { + this.logger.warn( + `Failed to read prompt file ${filePath}: ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } + } + + private parseMarkdownPrompt( + rawContent: string, + filePath: string + ): { + id: string; + title: string; + description: string; + content: string; + category?: string; + priority?: number; + arguments?: Array<{ name: string; required: boolean; description?: string }>; + // Claude Code compatibility fields + disableModelInvocation?: boolean; + userInvocable?: boolean; + allowedTools?: string[]; + model?: string; + context?: 'inline' | 'fork'; + agent?: string; + } { + const lines = rawContent.trim().split('\n'); + // Use path utilities for cross-platform compatibility (Windows uses backslashes) + const fileName = basename(filePath, '.md') || 'unknown'; + const parentDir = basename(dirname(filePath)) || 'unknown'; + + // For SKILL.md files, use parent directory name as the id (Claude Code convention) + // e.g., .claude/skills/my-skill/SKILL.md -> id = "my-skill" + const defaultId = fileName.toUpperCase() === 'SKILL' ? parentDir : fileName; + + let id = defaultId; + let title = defaultId; + let description = `File prompt: ${defaultId}`; + let category: string | undefined; + let priority: number | undefined; + let argumentHint: string | undefined; + // Claude Code compatibility fields + let disableModelInvocation: boolean | undefined; + let userInvocable: boolean | undefined; + let allowedTools: string[] | undefined; + let model: string | undefined; + let context: 'inline' | 'fork' | undefined; + let agent: string | undefined; + let contentBody: string; + + // Parse frontmatter if present + if (lines[0]?.trim() === '---') { + let frontmatterEnd = 0; + for (let i = 1; i < lines.length; i++) { + if (lines[i]?.trim() === '---') { + frontmatterEnd = i; + break; + } + } + + if (frontmatterEnd > 0) { + const frontmatterLines = lines.slice(1, frontmatterEnd); + contentBody = lines.slice(frontmatterEnd + 1).join('\n'); + + for (const line of frontmatterLines) { + const match = (key: string) => { + const regex = new RegExp(`${key}:\\s*(?:['"](.+)['"]|(.+))`); + const m = line.match(regex); + return m ? (m[1] || m[2] || '').trim() : null; + }; + + const matchBool = (key: string): boolean | undefined => { + const val = match(key); + if (val === 'true') return true; + if (val === 'false') return false; + return undefined; + }; + + if (line.includes('id:')) { + const val = match('id'); + if (val) id = val; + } else if (line.includes('name:') && !line.includes('display-name:')) { + // Claude Code SKILL.md uses 'name:' instead of 'id:' + // Only use if id hasn't been explicitly set via 'id:' field + const val = match('name'); + if (val && id === defaultId) id = val; + } else if (line.includes('title:')) { + const val = match('title'); + if (val) title = val; + } else if (line.includes('description:')) { + const val = match('description'); + if (val) description = val; + } else if (line.includes('category:')) { + const val = match('category'); + if (val) category = val; + } else if (line.includes('priority:')) { + const val = match('priority'); + if (val) priority = parseInt(val, 10); + } else if (line.includes('argument-hint:')) { + const val = match('argument-hint'); + if (val) argumentHint = val; + } else if (line.includes('disable-model-invocation:')) { + disableModelInvocation = matchBool('disable-model-invocation'); + } else if (line.includes('user-invocable:')) { + userInvocable = matchBool('user-invocable'); + } else if (line.includes('model:')) { + const val = match('model'); + if (val) model = val; + } else if (line.includes('context:')) { + const val = match('context'); + if (val === 'fork' || val === 'inline') context = val; + } else if (line.includes('agent:')) { + const val = match('agent'); + if (val) agent = val; + } + // Note: allowed-tools parsing requires special handling for arrays + // Will be parsed as YAML array in a separate pass below + } + + // Parse allowed-tools as inline YAML array format: [item1, item2] or [] + // Note: Multiline YAML array format (- item) is not supported + const frontmatterText = frontmatterLines.join('\n'); + const allowedToolsMatch = frontmatterText.match(/allowed-tools:\s*\[([^\]]*)\]/); + if (allowedToolsMatch) { + const rawContent = allowedToolsMatch[1]?.trim() ?? ''; + allowedTools = + rawContent.length === 0 + ? [] + : rawContent + .split(',') + .map((s) => s.trim().replace(/^['"]|['"]$/g, '')) + .filter((s) => s.length > 0); + } + } else { + contentBody = rawContent; + } + } else { + contentBody = rawContent; + } + + // Extract title from first heading if not in frontmatter + if (title === defaultId) { + for (const line of contentBody.trim().split('\n')) { + if (line.trim().startsWith('#')) { + title = line.trim().replace(/^#+\s*/, ''); + break; + } + } + } + + // Parse argument hints into structured arguments + const parsedArguments = argumentHint ? this.parseArgumentHint(argumentHint) : undefined; + + return { + id, + title, + description, + content: contentBody.trim(), + ...(category !== undefined && { category }), + ...(priority !== undefined && { priority }), + ...(parsedArguments !== undefined && { arguments: parsedArguments }), + ...(disableModelInvocation !== undefined && { disableModelInvocation }), + ...(userInvocable !== undefined && { userInvocable }), + ...(allowedTools !== undefined && { allowedTools }), + ...(model !== undefined && { model }), + ...(context !== undefined && { context }), + ...(agent !== undefined && { agent }), + }; + } + + private parseArgumentHint( + hint: string + ): Array<{ name: string; required: boolean; description?: string }> { + const args: Array<{ name: string; required: boolean; description?: string }> = []; + const argPattern = /\[([^\]]+)\]/g; + let match; + + while ((match = argPattern.exec(hint)) !== null) { + const argText = match[1]; + if (!argText) continue; + + const isOptional = argText.endsWith('?'); + const name = isOptional ? argText.slice(0, -1).trim() : argText.trim(); + + if (name) { + args.push({ + name, + required: !isOptional, + }); + } + } + + return args; + } + + private applyArguments(content: string, args?: Record): string { + // Detect whether content uses positional placeholders + const detectionTarget = content.replaceAll('$$', ''); + const usesPositionalPlaceholders = + /\$[1-9](?!\d)/.test(detectionTarget) || detectionTarget.includes('$ARGUMENTS'); + + // First expand positional placeholders + const expanded = expandPlaceholders(content, args).trim(); + + if (!args || typeof args !== 'object' || Object.keys(args).length === 0) { + return expanded; + } + + // If the prompt doesn't use placeholders, append formatted arguments + if (!usesPositionalPlaceholders) { + if ((args as Record)._context) { + const contextString = String((args as Record)._context); + return `${expanded}\n\nContext: ${contextString}`; + } + + const argEntries = Object.entries(args).filter(([key]) => !key.startsWith('_')); + if (argEntries.length > 0) { + const formattedArgs = argEntries + .map(([key, value]) => `${key}: ${value}`) + .join(', '); + return `${expanded}\n\nArguments: ${formattedArgs}`; + } + } + + return expanded; + } +} diff --git a/dexto/packages/core/src/prompts/providers/custom-prompt-provider.test.ts b/dexto/packages/core/src/prompts/providers/custom-prompt-provider.test.ts new file mode 100644 index 00000000..95fea7d0 --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/custom-prompt-provider.test.ts @@ -0,0 +1,48 @@ +import { describe, test, expect, beforeAll } from 'vitest'; +import { CustomPromptProvider } from './custom-prompt-provider.js'; +import { MemoryDatabaseStore } from '../../storage/database/memory-database-store.js'; + +const mockLogger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + silly: () => {}, + trackException: () => {}, + createChild: () => mockLogger, + destroy: async () => {}, +} as any; + +describe('CustomPromptProvider', () => { + let db: MemoryDatabaseStore; + const resourceManagerStub = { getBlobStore: () => undefined } as any; + + beforeAll(async () => { + db = new MemoryDatabaseStore(); + await db.connect(); + }); + + test('appends Context at END when no placeholders', async () => { + const provider = new CustomPromptProvider(db as any, resourceManagerStub, mockLogger); + await provider.createPrompt({ name: 'c1', content: 'Simple content' }); + const res = await provider.getPrompt('c1', { _context: 'CTX' } as any); + const text = (res.messages?.[0]?.content as any).text as string; + expect(text).toBe('Simple content\n\nContext: CTX'); + }); + + test('replaces named placeholders and does not append when used', async () => { + const provider = new CustomPromptProvider(db as any, resourceManagerStub, mockLogger); + await provider.createPrompt({ + name: 'c2', + content: 'Process: {{data}} with mode {{mode}}', + arguments: [{ name: 'data', required: true }, { name: 'mode' }], + }); + const res = await provider.getPrompt('c2', { + data: 'dataset', + mode: 'fast', + _context: 'CTX', + } as any); + const text = (res.messages?.[0]?.content as any).text as string; + expect(text).toBe('Process: dataset with mode fast'); + }); +}); diff --git a/dexto/packages/core/src/prompts/providers/custom-prompt-provider.ts b/dexto/packages/core/src/prompts/providers/custom-prompt-provider.ts new file mode 100644 index 00000000..0d2b877a --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/custom-prompt-provider.ts @@ -0,0 +1,360 @@ +import type { + PromptProvider, + PromptInfo, + PromptDefinition, + PromptListResult, + PromptArgument, +} from '../types.js'; +import type { GetPromptResult } from '@modelcontextprotocol/sdk/types.js'; +import type { Database } from '../../storage/database/types.js'; +import type { ResourceManager } from '../../resources/manager.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { expandPlaceholders } from '../utils.js'; +import { PromptError } from '../errors.js'; + +const CUSTOM_PROMPT_KEY_PREFIX = 'prompt:custom:'; + +interface StoredCustomPrompt { + id: string; + name: string; + title?: string; + description?: string; + content: string; + arguments?: PromptArgument[]; + resourceUri?: string; + resourceMetadata?: { + originalName?: string; + mimeType?: string; + }; + createdAt: number; + updatedAt: number; +} + +export interface CreateCustomPromptInput { + name: string; + title?: string; + description?: string; + content: string; + arguments?: PromptArgument[]; + resource?: { + data: string; + mimeType: string; + filename?: string; + }; +} + +export class CustomPromptProvider implements PromptProvider { + private cacheValid = false; + private promptsCache: PromptInfo[] = []; + private promptRecords: Map = new Map(); + private logger: IDextoLogger; + + constructor( + private database: Database, + private resourceManager: ResourceManager, + logger: IDextoLogger + ) { + this.logger = logger.createChild(DextoLogComponent.PROMPT); + } + + getSource(): string { + return 'custom'; + } + + invalidateCache(): void { + this.cacheValid = false; + this.promptsCache = []; + this.promptRecords.clear(); + } + + async listPrompts(_cursor?: string): Promise { + if (!this.cacheValid) { + await this.buildCache(); + } + + return { prompts: this.promptsCache }; + } + + async getPrompt(name: string, args?: Record): Promise { + if (!this.cacheValid) { + await this.buildCache(); + } + const record = this.promptRecords.get(name); + if (!record) { + throw PromptError.notFound(name); + } + + // Validate required arguments + if (record.arguments && record.arguments.length > 0) { + const requiredArgs = record.arguments.filter((arg) => arg.required); + const missingArgs = requiredArgs + .filter((arg) => !args || !(arg.name in args)) + .map((arg) => arg.name); + if (missingArgs.length > 0) { + throw PromptError.missingRequiredArguments(missingArgs); + } + } + + const messages: GetPromptResult['messages'] = []; + // First expand positional placeholders ($ARGUMENTS, $1..$9, $$) + const expanded = expandPlaceholders(record.content, args); + const textContent = this.applyArguments(expanded, args, record.arguments); + messages.push({ + role: 'user', + content: { + type: 'text', + text: textContent, + }, + }); + + if (record.resourceUri) { + try { + const blobService = this.resourceManager.getBlobStore(); + const blobData = await blobService.retrieve(record.resourceUri, 'base64'); + if (blobData.format === 'base64') { + messages.push({ + role: 'user', + content: { + type: 'resource', + resource: { + uri: record.resourceUri, + blob: blobData.data, + mimeType: + record.resourceMetadata?.mimeType || + blobData.metadata?.mimeType || + 'application/octet-stream', + }, + }, + }); + } + } catch (error) { + this.logger.warn( + `Failed to load blob resource for custom prompt ${name}: ${String(error)}` + ); + } + } + + return { + description: record.description, + messages, + }; + } + + async getPromptDefinition(name: string): Promise { + if (!this.cacheValid) { + await this.buildCache(); + } + const record = this.promptRecords.get(name); + if (!record) return null; + return { + name: record.name, + ...(record.title && { title: record.title }), + ...(record.description && { description: record.description }), + ...(record.arguments && { arguments: record.arguments }), + }; + } + + async createPrompt(input: CreateCustomPromptInput): Promise { + const id = this.slugify(input.name); + if (!id) { + throw PromptError.nameRequired(); + } + + if (!input.content || input.content.trim().length === 0) { + throw PromptError.missingText(); + } + + if (!this.cacheValid) { + await this.buildCache(); + } + + if (this.promptRecords.has(id)) { + throw PromptError.alreadyExists(id); + } + + let resourceUri: string | undefined; + let resourceMetadata: StoredCustomPrompt['resourceMetadata'] | undefined; + + if (input.resource) { + try { + const blobService = this.resourceManager.getBlobStore(); + const { data, mimeType, filename } = input.resource; + const blobRef = await blobService.store(data, { + mimeType, + originalName: filename, + source: 'system', + }); + resourceUri = blobRef.uri; + const meta: { originalName?: string; mimeType?: string } = {}; + const originalName = blobRef.metadata.originalName ?? filename; + if (originalName) { + meta.originalName = originalName; + } + if (blobRef.metadata.mimeType) { + meta.mimeType = blobRef.metadata.mimeType; + } + resourceMetadata = Object.keys(meta).length > 0 ? meta : undefined; + } catch (error) { + this.logger.warn(`Failed to store custom prompt resource: ${String(error)}`); + } + } + + const now = Date.now(); + const record: StoredCustomPrompt = { + id, + name: id, + content: input.content, + createdAt: now, + updatedAt: now, + ...(input.title ? { title: input.title } : {}), + ...(input.description ? { description: input.description } : {}), + ...(input.arguments ? { arguments: input.arguments } : {}), + ...(resourceUri ? { resourceUri } : {}), + ...(resourceMetadata ? { resourceMetadata } : {}), + }; + + await this.database.set(this.toKey(id), record); + this.invalidateCache(); + await this.buildCache(); + + const prompt = this.promptsCache.find((p) => p.name === id); + if (!prompt) { + throw PromptError.notFound(id); + } + return prompt; + } + + async deletePrompt(name: string): Promise { + if (!this.cacheValid) { + await this.buildCache(); + } + const record = this.promptRecords.get(name); + if (!record) { + throw PromptError.notFound(name); + } + + await this.database.delete(this.toKey(name)); + if (record.resourceUri) { + try { + const blobService = this.resourceManager.getBlobStore(); + await blobService.delete(record.resourceUri); + } catch (error) { + this.logger.warn( + `Failed to delete blob for custom prompt ${name}: ${String(error)}` + ); + } + } + this.invalidateCache(); + } + + private async buildCache(): Promise { + try { + const keys = await this.database.list(CUSTOM_PROMPT_KEY_PREFIX); + const prompts: PromptInfo[] = []; + this.promptRecords.clear(); + for (const key of keys) { + try { + const record = await this.database.get(key); + if (!record) continue; + this.promptRecords.set(record.name, record); + const metadata: Record = { + originalName: + record.resourceMetadata?.originalName && + record.resourceMetadata.originalName.trim().length > 0 + ? record.resourceMetadata.originalName + : record.name, + }; + + if (record.resourceUri) { + metadata.resourceUri = record.resourceUri; + } + + if (record.resourceMetadata?.mimeType) { + metadata.mimeType = record.resourceMetadata.mimeType; + } + + metadata.createdAt = record.createdAt; + metadata.updatedAt = record.updatedAt; + + prompts.push({ + name: record.name, + displayName: record.name, + title: record.title, + description: record.description, + source: 'custom', + ...(record.arguments && { arguments: record.arguments }), + metadata, + }); + } catch (error) { + this.logger.warn(`Failed to load custom prompt from ${key}: ${String(error)}`); + } + } + this.promptsCache = prompts; + this.cacheValid = true; + } catch (error) { + this.logger.error(`Failed to build custom prompts cache: ${String(error)}`); + this.promptsCache = []; + this.cacheValid = false; + } + } + + private toKey(id: string): string { + return `${CUSTOM_PROMPT_KEY_PREFIX}${id}`; + } + + private slugify(name: string): string { + const slug = name + .toLowerCase() + .trim() + .replace(/[^a-z0-9\s\-_]/g, '') + .replace(/[\s\-_]+/g, '-') + .replace(/^-+|-+$/g, ''); + return slug; + } + + private applyArguments( + content: string, + args?: Record, + declaredArgs?: PromptArgument[] + ): string { + if (!args || Object.keys(args).length === 0) { + return content; + } + + // Replace named placeholders {{name}} + let result = content; + for (const [key, value] of Object.entries(args)) { + if (key.startsWith('_')) continue; // skip special keys + const placeholder = `{{${key}}}`; + result = result.replaceAll(placeholder, String(value)); + } + + // Determine if placeholders were used in the template + const usesPositional = /\$[1-9]/.test(content) || content.includes('$ARGUMENTS'); + let usesNamed = false; + if (declaredArgs && declaredArgs.length > 0) { + usesNamed = declaredArgs.some((a) => a.name && content.includes(`{{${a.name}}}`)); + } else { + // Fallback heuristic: any {{...}} token counts as named placeholder usage + usesNamed = content.includes('{{') && content.includes('}}'); + } + + const placeholdersUsed = usesPositional || usesNamed; + + // If no placeholders are used, append context/arguments at the END + if (!placeholdersUsed) { + if ((args as any)._context) { + const contextString = String((args as any)._context); + return `${result}\n\nContext: ${contextString}`; + } + const argEntries = Object.entries(args).filter(([k]) => !k.startsWith('_')); + if (argEntries.length > 0) { + const formattedArgs = argEntries.map(([k, v]) => `${k}: ${v}`).join(', '); + return `${result}\n\nArguments: ${formattedArgs}`; + } + } + + return result; + } +} diff --git a/dexto/packages/core/src/prompts/providers/mcp-prompt-provider.ts b/dexto/packages/core/src/prompts/providers/mcp-prompt-provider.ts new file mode 100644 index 00000000..66111fe6 --- /dev/null +++ b/dexto/packages/core/src/prompts/providers/mcp-prompt-provider.ts @@ -0,0 +1,101 @@ +import type { MCPManager } from '../../mcp/manager.js'; +import type { PromptProvider, PromptInfo, PromptDefinition, PromptListResult } from '../types.js'; +import type { GetPromptResult } from '@modelcontextprotocol/sdk/types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; + +/** + * MCP Prompt Provider - Provides prompts from connected MCP servers + * + * This provider acts as a bridge between the PromptManager and MCPManager, + * exposing prompts from all connected MCP servers through a unified interface. + * It leverages MCPManager's built-in prompt metadata cache for efficient access. + */ +export class MCPPromptProvider implements PromptProvider { + private mcpManager: MCPManager; + private logger: IDextoLogger; + + constructor(mcpManager: MCPManager, logger: IDextoLogger) { + this.mcpManager = mcpManager; + this.logger = logger; + } + + /** + * Get the source identifier for this provider + */ + getSource(): string { + return 'mcp'; + } + + /** + * Invalidate the prompts cache (no-op as MCPManager handles caching) + */ + invalidateCache(): void { + // MCPManager handles cache invalidation through event notifications + this.logger.debug('MCPPromptProvider cache invalidation (handled by MCPManager)'); + } + + /** + * List all available prompts from MCP servers using MCPManager's cache + */ + async listPrompts(_cursor?: string): Promise { + const cachedPrompts = this.mcpManager.getAllPromptMetadata(); + + const prompts: PromptInfo[] = cachedPrompts.map( + ({ promptName, serverName, definition }) => { + const promptInfo: PromptInfo = { + name: promptName, + displayName: promptName, + title: + definition.title || definition.description || `MCP prompt: ${promptName}`, + description: definition.description || `MCP prompt: ${promptName}`, + ...(definition.arguments && { arguments: definition.arguments }), + source: 'mcp', + metadata: { + serverName, + originalName: promptName, + ...definition, + }, + }; + return promptInfo; + } + ); + + this.logger.debug(`📝 Listed ${prompts.length} MCP prompts from cache`); + + return { + prompts, + }; + } + + /** + * Get a specific prompt by name + */ + async getPrompt(name: string, args?: Record): Promise { + this.logger.debug(`📝 Reading MCP prompt: ${name}`); + return await this.mcpManager.getPrompt(name, args); + } + + /** + * Get prompt definition (metadata only) from MCPManager's cache + */ + async getPromptDefinition(name: string): Promise { + try { + const definition = this.mcpManager.getPromptMetadata(name); + if (!definition) { + return null; + } + + return { + name: definition.name, + ...(definition.title && { title: definition.title }), + ...(definition.description && { description: definition.description }), + ...(definition.arguments && { arguments: definition.arguments }), + }; + } catch (error) { + this.logger.debug( + `Failed to get prompt definition for '${name}': ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } + } +} diff --git a/dexto/packages/core/src/prompts/schemas.ts b/dexto/packages/core/src/prompts/schemas.ts new file mode 100644 index 00000000..94c40ee6 --- /dev/null +++ b/dexto/packages/core/src/prompts/schemas.ts @@ -0,0 +1,207 @@ +/** + * Zod schemas for prompt-related configurations + * + * Unified prompt system with discriminated union: + * - type: 'inline' - Prompt text defined directly in config + * - type: 'file' - Prompt loaded from a markdown file + * + * Both support showInStarters flag to control WebUI button display. + */ + +import { z } from 'zod'; +import { PROMPT_NAME_REGEX, PROMPT_NAME_GUIDANCE } from './name-validation.js'; + +/** + * Schema for inline prompts - text defined directly in config + */ +export const InlinePromptSchema = z + .object({ + type: z.literal('inline').describe('Inline prompt type'), + id: z + .string() + .min(1) + .max(64) + .regex(PROMPT_NAME_REGEX, `Prompt id must be ${PROMPT_NAME_GUIDANCE}`) + .describe('Kebab-case slug id for the prompt (e.g., quick-start)'), + title: z.string().optional().describe('Display title for the prompt'), + description: z + .string() + .optional() + .default('') + .describe('Description shown on hover or in the UI'), + prompt: z.string().describe('The actual prompt text'), + category: z + .string() + .optional() + .default('general') + .describe('Category for organizing prompts (e.g., general, coding, analysis, tools)'), + priority: z + .number() + .optional() + .default(0) + .describe('Higher numbers appear first in the list'), + showInStarters: z + .boolean() + .optional() + .default(false) + .describe('Show as a clickable button in WebUI starter prompts'), + // Claude Code compatibility fields (Phase 1) + 'disable-model-invocation': z + .boolean() + .optional() + .default(false) + .describe('Exclude from auto-invocation list in system prompt'), + 'user-invocable': z + .boolean() + .optional() + .default(true) + .describe('Show in slash command menu (false = hidden but auto-invocable by LLM)'), + // Per-prompt overrides (Phase 2) + 'allowed-tools': z + .array(z.string()) + .optional() + .describe('Tools allowed when this prompt is active (overrides global policies)'), + model: z.string().optional().describe('Model to use when this prompt is invoked'), + // Execution context (Phase 2) + context: z + .enum(['inline', 'fork']) + .optional() + .default('inline') + .describe( + "Execution context: 'inline' runs in current session (default), 'fork' spawns isolated subagent" + ), + // Agent for fork execution + agent: z + .string() + .optional() + .describe('Agent ID from registry to use for fork execution (e.g., "explore-agent")'), + }) + .strict() + .describe('Inline prompt with text defined directly in config'); + +/** + * Schema for file-based prompts - loaded from markdown files + */ +export const FilePromptSchema = z + .object({ + type: z.literal('file').describe('File-based prompt type'), + file: z + .string() + .describe( + 'Path to markdown file containing prompt (supports ${{dexto.agent_dir}} template)' + ), + showInStarters: z + .boolean() + .optional() + .default(false) + .describe('Show as a clickable button in WebUI starter prompts'), + // Claude Code compatibility fields (Phase 1) - can override frontmatter + 'disable-model-invocation': z + .boolean() + .optional() + .describe('Exclude from auto-invocation list in system prompt'), + 'user-invocable': z + .boolean() + .optional() + .describe('Show in slash command menu (false = hidden but auto-invocable by LLM)'), + // Per-prompt overrides (Phase 2) - can override frontmatter + 'allowed-tools': z + .array(z.string()) + .optional() + .describe('Tools allowed when this prompt is active (overrides global policies)'), + model: z.string().optional().describe('Model to use when this prompt is invoked'), + // Execution context (Phase 2) - can override frontmatter + context: z + .enum(['inline', 'fork']) + .optional() + .describe( + "Execution context: 'inline' runs in current session (default), 'fork' spawns isolated subagent" + ), + // Agent for fork execution - can override frontmatter + agent: z + .string() + .optional() + .describe('Agent ID from registry to use for fork execution (e.g., "explore-agent")'), + // Plugin namespace (Phase 3) - for prefixing command names + namespace: z + .string() + .optional() + .describe('Plugin namespace for command prefixing (e.g., plugin-name:command)'), + }) + .strict() + .describe('File-based prompt loaded from a markdown file'); + +/** + * Unified prompts schema - array of inline or file-based prompts + * Replaces the old StarterPromptsSchema + */ +export const PromptsSchema = z + .array(z.discriminatedUnion('type', [InlinePromptSchema, FilePromptSchema])) + .superRefine((arr, ctx) => { + // Check for duplicate inline prompt IDs + const seen = new Map(); + arr.forEach((p, idx) => { + if (p.type === 'inline') { + if (seen.has(p.id)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Duplicate prompt id: ${p.id}`, + path: [idx, 'id'], + }); + } else { + seen.set(p.id, idx); + } + } + }); + }) + .transform((arr) => + arr.map((p) => { + if (p.type === 'inline') { + // Auto-generate title from id if not provided + return { ...p, title: p.title ?? p.id.replace(/-/g, ' ') }; + } + return p; + }) + ) + .default([]) + .describe('Agent prompts - inline text or file-based'); + +/** + * Type for a single inline prompt (validated) + */ +export type ValidatedInlinePrompt = z.output; + +/** + * Type for a single file-based prompt (validated) + */ +export type ValidatedFilePrompt = z.output; + +/** + * Type for a single prompt (either inline or file) + */ +export type ValidatedPrompt = ValidatedInlinePrompt | ValidatedFilePrompt; + +/** + * Input type for a single inline prompt (before validation) + */ +export type InlinePrompt = z.input; + +/** + * Input type for a single file-based prompt (before validation) + */ +export type FilePrompt = z.input; + +/** + * Input type for a single prompt (before validation) + */ +export type Prompt = InlinePrompt | FilePrompt; + +/** + * Validated prompts configuration type + */ +export type ValidatedPromptsConfig = z.output; + +/** + * Input type for prompts configuration (before validation) + */ +export type PromptsConfig = z.input; diff --git a/dexto/packages/core/src/prompts/types.ts b/dexto/packages/core/src/prompts/types.ts new file mode 100644 index 00000000..764073b3 --- /dev/null +++ b/dexto/packages/core/src/prompts/types.ts @@ -0,0 +1,130 @@ +import type { GetPromptResult } from '@modelcontextprotocol/sdk/types.js'; + +/** + * MCP-compliant prompt argument definition + * Matches the MCP SDK's Prompt.arguments structure + */ +export interface PromptArgument { + name: string; + description?: string | undefined; + required?: boolean | undefined; +} + +/** + * MCP-compliant prompt definition with Dexto extensions + * Base structure matches MCP SDK's Prompt, extended with Claude Code compatibility fields + */ +export interface PromptDefinition { + name: string; + title?: string | undefined; + description?: string | undefined; + arguments?: PromptArgument[] | undefined; + // Claude Code compatibility fields (Phase 1) + /** Exclude from auto-invocation list in system prompt */ + disableModelInvocation?: boolean | undefined; + /** Show in slash command menu (false = hidden but auto-invocable by LLM) */ + userInvocable?: boolean | undefined; + // Per-prompt overrides (Phase 2) + /** Tools allowed when this prompt is active (overrides global policies) */ + allowedTools?: string[] | undefined; + /** Model to use when this prompt is invoked */ + model?: string | undefined; + /** Execution context: 'inline' runs in current session, 'fork' spawns isolated subagent */ + context?: 'inline' | 'fork' | undefined; + /** Agent ID from registry to use for fork execution */ + agent?: string | undefined; +} + +/** + * Enhanced prompt info with MCP-compliant structure + * + * ## Naming Convention + * + * Prompts have three name fields that serve different purposes: + * + * - **name**: Internal identifier used for resolution. May include prefixes like + * "config:namespace:id" for config prompts or just "promptName" for MCP/custom. + * + * - **displayName**: User-friendly base name without system prefixes. Set by providers + * to just the skill/prompt id (e.g., "plan" not "config:tools:plan"). For MCP and + * custom prompts, this equals `name` since they have no internal prefixes. + * + * - **commandName**: Collision-resolved slash command name computed by PromptManager. + * If multiple prompts share the same displayName, commandName adds a source prefix + * (e.g., "config:plan" vs "mcp:plan"). Otherwise, commandName equals displayName. + * + * UI components should use `commandName` for display and execution. + */ +export interface PromptInfo extends PromptDefinition { + source: 'mcp' | 'config' | 'custom'; + /** Base display name set by provider (e.g., "plan"). May equal `name` for simple prompts. */ + displayName?: string | undefined; + /** Collision-resolved command name computed by PromptManager (e.g., "plan" or "config:plan") */ + commandName?: string | undefined; + /** Execution context: 'inline' runs in current session, 'fork' spawns isolated subagent */ + context?: 'inline' | 'fork' | undefined; + /** Agent ID from registry to use for fork execution */ + agent?: string | undefined; + metadata?: Record; +} + +/** + * Set of prompts indexed by name + */ +export type PromptSet = Record; + +/** + * Result for prompt listing (pagination not currently implemented) + */ +export interface PromptListResult { + prompts: PromptInfo[]; + nextCursor?: string | undefined; +} + +/** + * Result type for resolvePrompt including optional per-prompt overrides + */ +export interface ResolvedPromptResult { + /** The resolved prompt text with arguments applied */ + text: string; + /** Resource URIs referenced by the prompt */ + resources: string[]; + /** Tools allowed when this prompt is active (overrides global policies) */ + allowedTools?: string[] | undefined; + /** Model to use when this prompt is invoked */ + model?: string | undefined; + /** Execution context: 'inline' runs in current session, 'fork' spawns isolated subagent */ + context?: 'inline' | 'fork' | undefined; + /** Agent ID from registry to use for fork execution */ + agent?: string | undefined; +} + +/** + * Interface for prompt providers + */ +export interface PromptProvider { + /** + * Get the source identifier for this provider + */ + getSource(): string; + + /** + * Invalidate the provider's internal cache + */ + invalidateCache(): void; + + /** + * List all available prompts from this provider + */ + listPrompts(cursor?: string): Promise; + + /** + * Get a specific prompt by name + */ + getPrompt(name: string, args?: Record): Promise; + + /** + * Get prompt definition (metadata only) + */ + getPromptDefinition(name: string): Promise; +} diff --git a/dexto/packages/core/src/prompts/utils.test.ts b/dexto/packages/core/src/prompts/utils.test.ts new file mode 100644 index 00000000..70ea4317 --- /dev/null +++ b/dexto/packages/core/src/prompts/utils.test.ts @@ -0,0 +1,520 @@ +import { describe, test, expect } from 'vitest'; +import { + flattenPromptResult, + normalizePromptArgs, + appendContext, + expandPlaceholders, +} from './utils.js'; +import type { GetPromptResult } from '@modelcontextprotocol/sdk/types.js'; + +describe('flattenPromptResult', () => { + test('should flatten text content from messages', () => { + const result: GetPromptResult = { + messages: [ + { + role: 'user', + content: { type: 'text', text: 'Hello world' }, + }, + ], + }; + + const flattened = flattenPromptResult(result); + + expect(flattened.text).toBe('Hello world'); + expect(flattened.resourceUris).toEqual([]); + }); + + test('should flatten multiple text parts with newline separation', () => { + const result = { + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'First line' }, + { type: 'text', text: 'Second line' }, + ], + }, + ], + } as unknown as GetPromptResult; + + const flattened = flattenPromptResult(result); + + expect(flattened.text).toBe('First line\nSecond line'); + expect(flattened.resourceUris).toEqual([]); + }); + + test('should extract resource URIs and text from resource content', () => { + const result = { + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Check this file' }, + { + type: 'resource', + resource: { + uri: 'file:///path/to/file.txt', + text: 'File content here', + }, + }, + ], + }, + ], + } as unknown as GetPromptResult; + + const flattened = flattenPromptResult(result); + + // No reference line appended; resource URIs returned separately + expect(flattened.text).toBe('Check this file\nFile content here'); + expect(flattened.resourceUris).toEqual(['file:///path/to/file.txt']); + }); + + test('should deduplicate resource URIs', () => { + const result = { + messages: [ + { + role: 'user', + content: [ + { + type: 'resource', + resource: { + uri: 'file:///same.txt', + text: 'Content 1', + }, + }, + { + type: 'resource', + resource: { + uri: 'file:///same.txt', + text: 'Content 2', + }, + }, + ], + }, + ], + } as unknown as GetPromptResult; + + const flattened = flattenPromptResult(result); + + expect(flattened.resourceUris).toEqual(['file:///same.txt']); + // No reference line appended; resource URIs returned separately + expect(flattened.text).toBe('Content 1\nContent 2'); + }); + + test('should handle string content directly', () => { + const result = { + messages: [ + { + role: 'user', + content: 'Simple string content', + }, + ], + } as unknown as GetPromptResult; + + const flattened = flattenPromptResult(result); + + expect(flattened.text).toBe('Simple string content'); + expect(flattened.resourceUris).toEqual([]); + }); + + test('should handle array of mixed content types', () => { + const result = { + messages: [ + { + role: 'user', + content: [ + 'Direct string', + { type: 'text', text: 'Text object' }, + { + type: 'resource', + resource: { + uri: 'mcp://server/resource', + text: 'Resource text', + }, + }, + ], + }, + ], + } as unknown as GetPromptResult; + + const flattened = flattenPromptResult(result); + + // No reference line appended; resource URIs returned separately + expect(flattened.text).toBe('Direct string\nText object\nResource text'); + expect(flattened.resourceUris).toEqual(['mcp://server/resource']); + }); + + test('should handle resources without text', () => { + const result = { + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Main content' }, + { + type: 'resource', + resource: { + uri: 'blob:abc123', + }, + }, + ], + }, + ], + } as unknown as GetPromptResult; + + const flattened = flattenPromptResult(result); + + // No reference line appended; resource URIs returned separately + expect(flattened.text).toBe('Main content'); + expect(flattened.resourceUris).toEqual(['blob:abc123']); + }); + + test('should ignore non-text content types (image, etc.)', () => { + const result = { + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Text content' }, + { type: 'image', data: 'base64data' }, + ], + }, + ], + } as unknown as GetPromptResult; + + const flattened = flattenPromptResult(result); + + expect(flattened.text).toBe('Text content'); + expect(flattened.resourceUris).toEqual([]); + }); + + test('should handle empty messages array', () => { + const result: GetPromptResult = { + messages: [], + }; + + const flattened = flattenPromptResult(result); + + expect(flattened.text).toBe(''); + expect(flattened.resourceUris).toEqual([]); + }); + + test('should handle multiple messages', () => { + const result: GetPromptResult = { + messages: [ + { + role: 'user', + content: { type: 'text', text: 'Message 1' }, + }, + { + role: 'assistant', + content: { type: 'text', text: 'Message 2' }, + }, + ], + }; + + const flattened = flattenPromptResult(result); + + expect(flattened.text).toBe('Message 1\nMessage 2'); + }); + + test('should filter out empty text parts', () => { + const result = { + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: '' }, + { type: 'text', text: 'Valid content' }, + { type: 'text', text: '' }, + ], + }, + ], + } as unknown as GetPromptResult; + + const flattened = flattenPromptResult(result); + + expect(flattened.text).toBe('Valid content'); + }); + + test('should ignore resources with empty URIs', () => { + const result = { + messages: [ + { + role: 'user', + content: [ + { + type: 'resource', + resource: { + uri: '', + text: 'Content without URI', + }, + }, + { + type: 'resource', + resource: { + uri: 'file:///valid.txt', + text: 'Valid resource', + }, + }, + ], + }, + ], + } as unknown as GetPromptResult; + + const flattened = flattenPromptResult(result); + + // No reference line appended; resource URIs returned separately + expect(flattened.text).toBe('Content without URI\nValid resource'); + expect(flattened.resourceUris).toEqual(['file:///valid.txt']); + }); +}); + +describe('expandPlaceholders', () => { + test('replaces $ARGUMENTS with all positional tokens when no explicit placeholders used', () => { + const out = expandPlaceholders('Run: $ARGUMENTS', { _positional: ['a', 'b', 'c'] }); + expect(out).toBe('Run: a b c'); + }); + + test('replaces $1..$3 with corresponding tokens and leaves missing empty', () => { + const out = expandPlaceholders('A:$1 B:$2 C:$3 D:$4', { _positional: ['x', 'y'] }); + expect(out).toBe('A:x B:y C: D:'); + }); + + test('respects $$ escape', () => { + const out = expandPlaceholders('Price $$1 and token $1', { _positional: ['X'] }); + expect(out).toBe('Price $1 and token X'); + }); + + test('$ARGUMENTS includes only remaining args after $1 and $2', () => { + const template = 'Style: $1, Length: $2, Content: $ARGUMENTS'; + const args = { _positional: ['technical', '100', 'machine learning'] }; + const result = expandPlaceholders(template, args); + expect(result).toBe('Style: technical, Length: 100, Content: machine learning'); + }); + + test('$ARGUMENTS is empty when all args consumed by explicit placeholders', () => { + const template = '$1 $2 $3 rest: $ARGUMENTS'; + const args = { _positional: ['a', 'b', 'c'] }; + const result = expandPlaceholders(template, args); + expect(result).toBe('a b c rest: '); + }); + + test('$ARGUMENTS works with sparse placeholders ($1 and $3)', () => { + const template = 'First: $1, Third: $3, Rest: $ARGUMENTS'; + const args = { _positional: ['a', 'b', 'c', 'd'] }; + const result = expandPlaceholders(template, args); + // maxExplicitIndex = 3, so ARGUMENTS includes from index 3 onwards + expect(result).toBe('First: a, Third: c, Rest: d'); + }); + + test('$ARGUMENTS works correctly with more positional args than explicit placeholders', () => { + const template = 'Arg1: $1, Remaining: $ARGUMENTS'; + const args = { _positional: ['first', 'second', 'third', 'fourth'] }; + const result = expandPlaceholders(template, args); + expect(result).toBe('Arg1: first, Remaining: second third fourth'); + }); + + test('handles template with only $ARGUMENTS and no explicit placeholders', () => { + const template = 'All args: $ARGUMENTS'; + const args = { _positional: ['one', 'two', 'three'] }; + const result = expandPlaceholders(template, args); + expect(result).toBe('All args: one two three'); + }); + + test('handles empty positional array with $ARGUMENTS', () => { + const template = 'Content: $ARGUMENTS'; + const args = { _positional: [] }; + const result = expandPlaceholders(template, args); + expect(result).toBe('Content: '); + }); + + test('real-world summarize.md use case', () => { + const template = 'Using style **$1** with length **$2**.\nContent: $ARGUMENTS'; + const args = { _positional: ['technical', '100', 'machine learning'] }; + const result = expandPlaceholders(template, args); + expect(result).toBe( + 'Using style **technical** with length **100**.\nContent: machine learning' + ); + }); +}); + +describe('normalizePromptArgs', () => { + test('should convert all values to strings', () => { + const input = { + name: 'John', + age: 30, + active: true, + }; + + const result = normalizePromptArgs(input); + + expect(result.args).toEqual({ + name: 'John', + age: '30', + active: 'true', + }); + expect(result.context).toBeUndefined(); + }); + + test('should extract _context field separately', () => { + const input = { + query: 'search term', + _context: 'Additional context here', + }; + + const result = normalizePromptArgs(input); + + expect(result.args).toEqual({ + query: 'search term', + }); + expect(result.context).toBe('Additional context here'); + }); + + test('should trim _context field', () => { + const input = { + _context: ' trimmed context ', + }; + + const result = normalizePromptArgs(input); + + expect(result.context).toBe('trimmed context'); + }); + + test('should ignore empty _context field', () => { + const input = { + query: 'test', + _context: ' ', + }; + + const result = normalizePromptArgs(input); + + expect(result.args).toEqual({ + query: 'test', + }); + expect(result.context).toBeUndefined(); + }); + + test('should stringify objects and arrays', () => { + const input = { + config: { nested: true }, + tags: ['tag1', 'tag2'], + }; + + const result = normalizePromptArgs(input); + + expect(result.args).toEqual({ + config: '{"nested":true}', + tags: '["tag1","tag2"]', + }); + }); + + test('should skip undefined and null values', () => { + const input = { + defined: 'value', + undefinedKey: undefined, + nullKey: null, + }; + + const result = normalizePromptArgs(input); + + expect(result.args).toEqual({ + defined: 'value', + }); + }); + + test('should handle empty input', () => { + const result = normalizePromptArgs({}); + + expect(result.args).toEqual({}); + expect(result.context).toBeUndefined(); + }); + + test('should handle non-JSON-serializable values gracefully', () => { + const circular: any = { name: 'obj' }; + circular.self = circular; + + const input = { + regular: 'value', + circular: circular, + }; + + const result = normalizePromptArgs(input); + + expect(result.args.regular).toBe('value'); + expect(result.args.circular).toBe('[object Object]'); // Falls back to String() + }); + + test('should preserve string values unchanged', () => { + const input = { + text: 'unchanged', + number: '42', + }; + + const result = normalizePromptArgs(input); + + expect(result.args).toEqual({ + text: 'unchanged', + number: '42', + }); + }); +}); + +describe('appendContext', () => { + test('should append context with double newline separator', () => { + const result = appendContext('Main text', 'Additional context'); + + expect(result).toBe('Main text\n\nAdditional context'); + }); + + test('should return text unchanged when context is undefined', () => { + const result = appendContext('Main text', undefined); + + expect(result).toBe('Main text'); + }); + + test('should return text unchanged when context is empty', () => { + const result = appendContext('Main text', ''); + + expect(result).toBe('Main text'); + }); + + test('should return text unchanged when context is whitespace only', () => { + const result = appendContext('Main text', ' '); + + expect(result).toBe('Main text'); + }); + + test('should return context when text is empty', () => { + const result = appendContext('', 'Context only'); + + expect(result).toBe('Context only'); + }); + + test('should return context when text is whitespace only', () => { + const result = appendContext(' ', 'Context only'); + + expect(result).toBe('Context only'); + }); + + test('should handle both empty text and empty context', () => { + const result = appendContext('', ''); + + expect(result).toBe(''); + }); + + test('should handle undefined text gracefully', () => { + const result = appendContext(undefined as any, 'Context'); + + expect(result).toBe('Context'); + }); + + test('should preserve multiline text and context', () => { + const text = 'Line 1\nLine 2'; + const context = 'Context line 1\nContext line 2'; + + const result = appendContext(text, context); + + expect(result).toBe('Line 1\nLine 2\n\nContext line 1\nContext line 2'); + }); +}); diff --git a/dexto/packages/core/src/prompts/utils.ts b/dexto/packages/core/src/prompts/utils.ts new file mode 100644 index 00000000..fe3f04e0 --- /dev/null +++ b/dexto/packages/core/src/prompts/utils.ts @@ -0,0 +1,243 @@ +import type { GetPromptResult } from '@modelcontextprotocol/sdk/types.js'; + +export interface FlattenedPromptResult { + text: string; + resourceUris: string[]; +} + +/** + * Validates that a resource URI uses an allowed scheme. + * Prevents injection attacks via javascript:, data:, etc. + * + * Allowed schemes: + * - mcp: (MCP server resources) + * - blob: (blob storage) + * - file: (filesystem resources) + * - http/https: (web resources) + */ +function isValidResourceUri(uri: string): boolean { + try { + // Handle URIs with scheme + if (uri.includes(':')) { + const scheme = uri.split(':')[0]?.toLowerCase(); + if (!scheme) return false; + const allowedSchemes = ['mcp', 'blob', 'file', 'http', 'https']; + return allowedSchemes.includes(scheme); + } + // Allow relative URIs (no scheme) + return true; + } catch { + return false; + } +} + +function handleContent( + content: unknown, + accumulator: { + textParts: string[]; + resourceUris: string[]; + } +): void { + if (content == null) { + return; + } + + if (typeof content === 'string') { + accumulator.textParts.push(content); + return; + } + + if (Array.isArray(content)) { + for (const part of content) { + handleContent(part, accumulator); + } + return; + } + + if (typeof content === 'object') { + const candidate = content as { type?: string }; + switch (candidate.type) { + case 'text': { + const textCandidate = content as { text?: unknown }; + if (typeof textCandidate.text === 'string') { + accumulator.textParts.push(textCandidate.text); + } + return; + } + case 'resource': { + // Embedded resource: content is included directly + const resourceContent = content as { + resource?: { + uri?: string; + text?: unknown; + }; + }; + const resource = resourceContent.resource; + if (resource) { + if (typeof resource.text === 'string') { + accumulator.textParts.push(resource.text); + } + if (typeof resource.uri === 'string' && resource.uri.length > 0) { + accumulator.resourceUris.push(resource.uri); + } + } + return; + } + case 'resource_link': { + // Resource link: pointer to be fetched separately + // Add a text marker so UI knows to fetch this resource + const linkContent = content as { + resource?: { + uri?: string; + }; + }; + const resource = linkContent.resource; + if (resource && typeof resource.uri === 'string' && resource.uri.length > 0) { + // Security: Validate URI scheme to prevent injection attacks + if (isValidResourceUri(resource.uri)) { + accumulator.textParts.push(`@<${resource.uri}>`); + accumulator.resourceUris.push(resource.uri); + } + } + return; + } + default: + // Non-textual content types (image/audio/file) are ignored here; callers can + // use resource URIs or other channels to handle them explicitly. + return; + } + } +} + +export function flattenPromptResult(result: GetPromptResult): FlattenedPromptResult { + const accumulator = { textParts: [] as string[], resourceUris: [] as string[] }; + const messages = Array.isArray(result.messages) ? result.messages : []; + + for (const message of messages) { + const maybeContent = (message as { content?: unknown }).content; + handleContent(maybeContent, accumulator); + } + + const uniqueUris = Array.from(new Set(accumulator.resourceUris)); + const joinedText = accumulator.textParts + .map((part) => (typeof part === 'string' ? part : '')) + .filter((part) => part.length > 0) + .join('\n') + .trim(); + + // Note: We don't append @ references here because: + // 1. For embedded resources (type: 'resource'), the content is already in the text + // 2. resourceUris array is returned for metadata/tracking purposes only + // 3. The UI should not try to fetch these URIs as separate attachments + return { + text: joinedText, + resourceUris: uniqueUris, + }; +} + +/** + * Normalize prompt arguments by converting all values to strings and extracting context. + * Handles the special `_context` field used for natural language after slash commands. + */ +export function normalizePromptArgs(input: Record): { + args: Record; + context?: string | undefined; +} { + const args: Record = {}; + let context: string | undefined; + + for (const [key, value] of Object.entries(input)) { + if (key === '_context') { + if (typeof value === 'string' && value.trim().length > 0) { + const trimmed = value.trim(); + context = trimmed; + // Don't add _context to args - it's handled separately + // ToDo: handle arg parsing for prompts + } + continue; + } + + // Preserve _positional array as-is for prompt expansion and mapping + if (key === '_positional') { + (args as any)._positional = value; + continue; + } + + if (typeof value === 'string') { + args[key] = value; + } else if (value !== undefined && value !== null) { + try { + args[key] = JSON.stringify(value); + } catch { + args[key] = String(value); + } + } + } + + return { args, context }; +} + +/** + * Append context to text, handling empty cases gracefully. + */ +export function appendContext(text: string, context?: string): string { + if (!context || context.trim().length === 0) { + return text ?? ''; + } + if (!text || text.trim().length === 0) { + return context; + } + return `${text}\n\n${context}`; +} + +/** + * Expand simple placeholder syntax in a template using positional args. + * Supported tokens: + * - $ARGUMENTS → remaining positional tokens (after $1..$9) joined by a single space + * - $1..$9 → nth positional token or empty string if missing + * - $$ → literal dollar sign + * + * Notes: + * - Positional tokens are expected under args._positional as string[] + * - Key/value args are ignored here (handled separately by providers if needed) + * - $ARGUMENTS only includes args not consumed by explicit $N placeholders + */ +export function expandPlaceholders(content: string, args?: Record): string { + if (!content) return ''; + + const positional = Array.isArray((args as any)?._positional) + ? ((args as any)._positional as string[]) + : []; + + // Protect escaped dollars + const ESC = '__DOLLAR__PLACEHOLDER__'; + let out = content.replaceAll('$$', ESC); + + // Find highest $N placeholder used in template (1-9) + let maxExplicitIndex = 0; + for (let i = 1; i <= 9; i++) { + if (out.includes(`$${i}`)) { + maxExplicitIndex = i; + } + } + + // $ARGUMENTS → remaining positional args after explicit $N placeholders + if (out.includes('$ARGUMENTS')) { + const remainingArgs = positional.slice(maxExplicitIndex); + out = out.replaceAll('$ARGUMENTS', remainingArgs.join(' ')); + } + + // $1..$9 → corresponding positional token + for (let i = 1; i <= 9; i++) { + const token = `$${i}`; + if (out.includes(token)) { + const val = positional[i - 1] ?? ''; + // Use split/join to avoid $-replacement semantics in regex + out = out.split(token).join(val); + } + } + + // Restore $$ + out = out.replaceAll(ESC, '$'); + return out; +} diff --git a/dexto/packages/core/src/providers/base-registry.test.ts b/dexto/packages/core/src/providers/base-registry.test.ts new file mode 100644 index 00000000..b8e5722d --- /dev/null +++ b/dexto/packages/core/src/providers/base-registry.test.ts @@ -0,0 +1,351 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { z } from 'zod'; +import { + BaseRegistry, + defaultErrorFactory, + type BaseProvider, + type ConfigurableProvider, + type RegistryErrorFactory, +} from './base-registry.js'; + +// Test provider types +interface SimpleProvider extends BaseProvider { + type: string; + value: number; +} + +interface ConfigurableTestProvider extends ConfigurableProvider { + type: string; + configSchema: z.ZodType; + metadata?: { displayName: string }; +} + +describe('BaseRegistry', () => { + describe('with default error factory', () => { + let registry: BaseRegistry; + + beforeEach(() => { + registry = new BaseRegistry(); + }); + + describe('register', () => { + it('should register a provider', () => { + const provider: SimpleProvider = { type: 'test', value: 42 }; + registry.register(provider); + expect(registry.has('test')).toBe(true); + }); + + it('should throw when registering duplicate provider', () => { + const provider: SimpleProvider = { type: 'test', value: 42 }; + registry.register(provider); + + expect(() => registry.register(provider)).toThrow( + "Provider 'test' is already registered" + ); + }); + + it('should allow registering multiple different providers', () => { + registry.register({ type: 'a', value: 1 }); + registry.register({ type: 'b', value: 2 }); + registry.register({ type: 'c', value: 3 }); + + expect(registry.size).toBe(3); + expect(registry.getTypes()).toEqual(['a', 'b', 'c']); + }); + }); + + describe('unregister', () => { + it('should unregister an existing provider', () => { + registry.register({ type: 'test', value: 42 }); + const result = registry.unregister('test'); + + expect(result).toBe(true); + expect(registry.has('test')).toBe(false); + }); + + it('should return false when unregistering non-existent provider', () => { + const result = registry.unregister('nonexistent'); + expect(result).toBe(false); + }); + }); + + describe('get', () => { + it('should return the provider if found', () => { + const provider: SimpleProvider = { type: 'test', value: 42 }; + registry.register(provider); + + const result = registry.get('test'); + expect(result).toEqual(provider); + }); + + it('should return undefined if not found', () => { + const result = registry.get('nonexistent'); + expect(result).toBeUndefined(); + }); + }); + + describe('has', () => { + it('should return true for registered providers', () => { + registry.register({ type: 'test', value: 42 }); + expect(registry.has('test')).toBe(true); + }); + + it('should return false for non-registered providers', () => { + expect(registry.has('nonexistent')).toBe(false); + }); + }); + + describe('getTypes', () => { + it('should return empty array when no providers', () => { + expect(registry.getTypes()).toEqual([]); + }); + + it('should return all registered types', () => { + registry.register({ type: 'a', value: 1 }); + registry.register({ type: 'b', value: 2 }); + + expect(registry.getTypes()).toEqual(['a', 'b']); + }); + }); + + describe('getAll', () => { + it('should return empty array when no providers', () => { + expect(registry.getAll()).toEqual([]); + }); + + it('should return all registered providers', () => { + const a: SimpleProvider = { type: 'a', value: 1 }; + const b: SimpleProvider = { type: 'b', value: 2 }; + registry.register(a); + registry.register(b); + + expect(registry.getAll()).toEqual([a, b]); + }); + }); + + describe('size', () => { + it('should return 0 when empty', () => { + expect(registry.size).toBe(0); + }); + + it('should return correct count', () => { + registry.register({ type: 'a', value: 1 }); + registry.register({ type: 'b', value: 2 }); + expect(registry.size).toBe(2); + }); + }); + + describe('clear', () => { + it('should remove all providers', () => { + registry.register({ type: 'a', value: 1 }); + registry.register({ type: 'b', value: 2 }); + registry.clear(); + + expect(registry.size).toBe(0); + expect(registry.getTypes()).toEqual([]); + }); + }); + }); + + describe('with custom error factory', () => { + class CustomError extends Error { + constructor( + message: string, + public code: string + ) { + super(message); + } + } + + const customErrorFactory: RegistryErrorFactory = { + alreadyRegistered: (type: string) => new CustomError(`Duplicate: ${type}`, 'DUPLICATE'), + notFound: (type: string, available: string[]) => + new CustomError(`Missing: ${type}, have: ${available}`, 'NOT_FOUND'), + }; + + let registry: BaseRegistry; + + beforeEach(() => { + registry = new BaseRegistry(customErrorFactory); + }); + + it('should use custom error for duplicate registration', () => { + registry.register({ type: 'test', value: 42 }); + + try { + registry.register({ type: 'test', value: 99 }); + expect.fail('Should have thrown'); + } catch (error) { + expect(error).toBeInstanceOf(CustomError); + expect((error as CustomError).code).toBe('DUPLICATE'); + expect((error as CustomError).message).toBe('Duplicate: test'); + } + }); + + it('should use custom error for validateConfig not found', () => { + try { + registry.validateConfig({ type: 'unknown' }); + expect.fail('Should have thrown'); + } catch (error) { + expect(error).toBeInstanceOf(CustomError); + expect((error as CustomError).code).toBe('NOT_FOUND'); + } + }); + }); + + describe('validateConfig', () => { + let registry: BaseRegistry; + + beforeEach(() => { + registry = new BaseRegistry(); + }); + + it('should throw if config has no type field', () => { + expect(() => registry.validateConfig({})).toThrow(); + }); + + it('should throw if config type is not a string', () => { + expect(() => registry.validateConfig({ type: 123 })).toThrow(); + }); + + it('should throw if provider not found', () => { + expect(() => registry.validateConfig({ type: 'unknown' })).toThrow( + "Provider 'unknown' not found" + ); + }); + + it('should throw if provider has no configSchema', () => { + // Register a provider without configSchema + const providerWithoutSchema = { + type: 'no-schema', + } as ConfigurableTestProvider; + registry.register(providerWithoutSchema); + + expect(() => registry.validateConfig({ type: 'no-schema' })).toThrow( + "Provider 'no-schema' does not support config validation" + ); + }); + + it('should validate against provider schema', () => { + const schema = z.object({ + type: z.literal('my-type'), + value: z.number(), + }); + + registry.register({ + type: 'my-type', + configSchema: schema, + }); + + const result = registry.validateConfig({ type: 'my-type', value: 42 }); + expect(result).toEqual({ type: 'my-type', value: 42 }); + }); + + it('should throw on schema validation failure', () => { + const schema = z.object({ + type: z.literal('my-type'), + value: z.number(), + }); + + registry.register({ + type: 'my-type', + configSchema: schema, + }); + + expect(() => + registry.validateConfig({ type: 'my-type', value: 'not-a-number' }) + ).toThrow(); + }); + }); + + describe('defaultErrorFactory', () => { + it('should create alreadyRegistered error', () => { + const error = defaultErrorFactory.alreadyRegistered('test-type'); + expect(error).toBeInstanceOf(Error); + expect(error.message).toBe("Provider 'test-type' is already registered"); + }); + + it('should create notFound error with available types', () => { + const error = defaultErrorFactory.notFound('unknown', ['a', 'b', 'c']); + expect(error).toBeInstanceOf(Error); + expect(error.message).toBe("Provider 'unknown' not found. Available: a, b, c"); + }); + + it('should create notFound error with no available types', () => { + const error = defaultErrorFactory.notFound('unknown', []); + expect(error).toBeInstanceOf(Error); + expect(error.message).toBe("Provider 'unknown' not found. Available: none"); + }); + }); + + describe('type safety', () => { + it('should maintain provider type through get()', () => { + interface TypedProvider extends BaseProvider { + type: string; + specificMethod: () => string; + } + + const registry = new BaseRegistry(); + registry.register({ + type: 'typed', + specificMethod: () => 'hello', + }); + + const provider = registry.get('typed'); + expect(provider?.specificMethod()).toBe('hello'); + }); + + it('should maintain provider type through getAll()', () => { + interface TypedProvider extends BaseProvider { + type: string; + value: number; + } + + const registry = new BaseRegistry(); + registry.register({ type: 'a', value: 1 }); + registry.register({ type: 'b', value: 2 }); + + const providers = registry.getAll(); + const sum = providers.reduce((acc, p) => acc + p.value, 0); + expect(sum).toBe(3); + }); + }); + + describe('edge cases', () => { + let registry: BaseRegistry; + + beforeEach(() => { + registry = new BaseRegistry(); + }); + + it('should handle empty string type', () => { + registry.register({ type: '', value: 42 }); + expect(registry.has('')).toBe(true); + expect(registry.get('')?.value).toBe(42); + }); + + it('should handle special characters in type', () => { + registry.register({ type: 'type-with_special.chars', value: 42 }); + expect(registry.has('type-with_special.chars')).toBe(true); + }); + + it('should handle re-registration after unregister', () => { + registry.register({ type: 'test', value: 1 }); + registry.unregister('test'); + registry.register({ type: 'test', value: 2 }); + + expect(registry.get('test')?.value).toBe(2); + }); + + it('should handle clear then re-register', () => { + registry.register({ type: 'a', value: 1 }); + registry.register({ type: 'b', value: 2 }); + registry.clear(); + registry.register({ type: 'c', value: 3 }); + + expect(registry.size).toBe(1); + expect(registry.has('a')).toBe(false); + expect(registry.has('c')).toBe(true); + }); + }); +}); diff --git a/dexto/packages/core/src/providers/base-registry.ts b/dexto/packages/core/src/providers/base-registry.ts new file mode 100644 index 00000000..afb76708 --- /dev/null +++ b/dexto/packages/core/src/providers/base-registry.ts @@ -0,0 +1,208 @@ +import { z } from 'zod'; + +/** + * Base provider interface - all providers must have a type identifier. + */ +export interface BaseProvider { + /** Unique type identifier for this provider */ + type: string; +} + +/** + * Provider with config schema - for providers that support validateConfig. + */ +export interface ConfigurableProvider extends BaseProvider { + /** Zod schema for validating provider configuration */ + configSchema: z.ZodType; +} + +/** + * Error factory interface for customizing registry errors. + * Each registry can provide its own error implementations. + */ +export interface RegistryErrorFactory { + /** Called when attempting to register a provider that already exists */ + alreadyRegistered(type: string): Error; + /** Called when looking up a provider that doesn't exist (for validateConfig) */ + notFound(type: string, availableTypes: string[]): Error; +} + +/** + * Default error factory that throws plain Error instances. + * Used when no custom error factory is provided. + */ +export const defaultErrorFactory: RegistryErrorFactory = { + alreadyRegistered: (type: string) => new Error(`Provider '${type}' is already registered`), + notFound: (type: string, availableTypes: string[]) => + new Error( + `Provider '${type}' not found. Available: ${availableTypes.join(', ') || 'none'}` + ), +}; + +/** + * Generic base registry for provider patterns. + * + * This class provides common registry functionality used across Dexto's + * provider system (blob storage, compression, custom tools, etc.). + * + * Features: + * - Type-safe provider registration and retrieval + * - Duplicate registration prevention + * - Customizable error handling via error factory + * - Optional config validation for providers with schemas + * + * @template TProvider - The provider type (must extend BaseProvider) + * + * @example + * ```typescript + * // Define your provider interface + * interface MyProvider extends BaseProvider { + * type: string; + * configSchema: z.ZodType; + * create(config: any): MyInstance; + * } + * + * // Create a registry + * class MyRegistry extends BaseRegistry { + * constructor() { + * super({ + * alreadyRegistered: (type) => new MyError(`Provider ${type} exists`), + * notFound: (type, available) => new MyError(`Unknown: ${type}`), + * }); + * } + * } + * + * // Use the registry + * const registry = new MyRegistry(); + * registry.register(myProvider); + * const provider = registry.get('my-type'); + * ``` + */ +export class BaseRegistry { + protected providers = new Map(); + protected errorFactory: RegistryErrorFactory; + + /** + * Create a new registry instance. + * + * @param errorFactory - Optional custom error factory for registry errors. + * If not provided, uses default Error instances. + */ + constructor(errorFactory: RegistryErrorFactory = defaultErrorFactory) { + this.errorFactory = errorFactory; + } + + /** + * Register a provider. + * + * @param provider - The provider to register + * @throws Error if a provider with the same type is already registered + */ + register(provider: TProvider): void { + if (this.providers.has(provider.type)) { + throw this.errorFactory.alreadyRegistered(provider.type); + } + this.providers.set(provider.type, provider); + } + + /** + * Unregister a provider. + * + * @param type - The provider type to unregister + * @returns true if the provider was unregistered, false if it wasn't registered + */ + unregister(type: string): boolean { + return this.providers.delete(type); + } + + /** + * Get a registered provider by type. + * + * @param type - The provider type identifier + * @returns The provider if found, undefined otherwise + */ + get(type: string): TProvider | undefined { + return this.providers.get(type); + } + + /** + * Check if a provider is registered. + * + * @param type - The provider type identifier + * @returns true if registered, false otherwise + */ + has(type: string): boolean { + return this.providers.has(type); + } + + /** + * Get all registered provider types. + * + * @returns Array of provider type identifiers + */ + getTypes(): string[] { + return Array.from(this.providers.keys()); + } + + /** + * Get all registered providers. + * + * @returns Array of providers + */ + getAll(): TProvider[] { + return Array.from(this.providers.values()); + } + + /** + * Get the number of registered providers. + * + * @returns Count of registered providers + */ + get size(): number { + return this.providers.size; + } + + /** + * Clear all registered providers. + * Primarily useful for testing. + */ + clear(): void { + this.providers.clear(); + } + + /** + * Validate a configuration against a provider's schema. + * + * This method is only available for registries with providers that have + * a configSchema property. It: + * 1. Extracts the 'type' field to identify the provider + * 2. Looks up the provider in the registry + * 3. Validates the full config against the provider's schema + * 4. Returns the validated configuration + * + * @param config - Raw configuration object with a 'type' field + * @returns Validated configuration object + * @throws Error if type is missing, provider not found, or validation fails + */ + validateConfig(config: unknown): any { + // First, validate that we have a type field + const typeSchema = z.object({ type: z.string() }).passthrough(); + const parsed = typeSchema.parse(config); + + // Look up the provider + const provider = this.providers.get(parsed.type); + if (!provider) { + throw this.errorFactory.notFound(parsed.type, this.getTypes()); + } + + // Check if provider has configSchema + if (!('configSchema' in provider) || !provider.configSchema) { + throw new Error( + `Provider '${parsed.type}' does not support config validation (no configSchema)` + ); + } + + // Validate against provider schema + return (provider as ConfigurableProvider).configSchema.parse(config); + } +} diff --git a/dexto/packages/core/src/providers/discovery.integration.test.ts b/dexto/packages/core/src/providers/discovery.integration.test.ts new file mode 100644 index 00000000..1a8ee6dd --- /dev/null +++ b/dexto/packages/core/src/providers/discovery.integration.test.ts @@ -0,0 +1,187 @@ +import { describe, it, expect } from 'vitest'; +import { listAllProviders, getProvidersByCategory, hasProvider } from './discovery.js'; + +/** + * Integration tests for Provider Discovery API + * These tests verify that the discovery API correctly interacts with all registries + * and that built-in providers are properly registered. + */ +describe('Provider Discovery API - Integration', () => { + describe('Built-in Provider Registration', () => { + it('should have blob storage providers registered on module import', () => { + const providers = listAllProviders(); + + // Verify built-in blob providers are registered + expect(providers.blob.length).toBeGreaterThanOrEqual(2); + + const types = providers.blob.map((p) => p.type); + expect(types).toContain('local'); + expect(types).toContain('in-memory'); + }); + + it('should have compaction providers registered on module import', () => { + const providers = listAllProviders(); + + // Verify built-in compaction providers are registered + expect(providers.compaction.length).toBeGreaterThanOrEqual(2); + + const types = providers.compaction.map((p) => p.type); + expect(types).toContain('reactive-overflow'); + expect(types).toContain('noop'); + }); + + it('should have valid metadata for built-in providers', () => { + const providers = listAllProviders(); + + // Check blob providers have metadata + for (const provider of providers.blob) { + expect(provider.type).toBeTruthy(); + expect(provider.category).toBe('blob'); + // Metadata is optional but if present should have displayName or description + if (provider.metadata) { + const hasDisplayName = provider.metadata.displayName !== undefined; + const hasDescription = provider.metadata.description !== undefined; + expect(hasDisplayName || hasDescription).toBe(true); + } + } + + // Check compaction providers have metadata + for (const provider of providers.compaction) { + expect(provider.type).toBeTruthy(); + expect(provider.category).toBe('compaction'); + if (provider.metadata) { + const hasDisplayName = provider.metadata.displayName !== undefined; + const hasDescription = provider.metadata.description !== undefined; + expect(hasDisplayName || hasDescription).toBe(true); + } + } + }); + }); + + describe('Cross-Registry Queries', () => { + it('should correctly separate providers by category', () => { + const allProviders = listAllProviders(); + + // All blob providers should have blob category + allProviders.blob.forEach((p) => { + expect(p.category).toBe('blob'); + }); + + // All compaction providers should have compaction category + allProviders.compaction.forEach((p) => { + expect(p.category).toBe('compaction'); + }); + + // All custom tool providers should have customTools category + allProviders.customTools.forEach((p) => { + expect(p.category).toBe('customTools'); + }); + }); + + it('should return consistent results between listAllProviders and getProvidersByCategory', () => { + const allProviders = listAllProviders(); + + const blobViaList = allProviders.blob; + const blobViaCategory = getProvidersByCategory('blob'); + expect(blobViaList).toEqual(blobViaCategory); + + const compactionViaList = allProviders.compaction; + const compactionViaCategory = getProvidersByCategory('compaction'); + expect(compactionViaList).toEqual(compactionViaCategory); + + const customToolsViaList = allProviders.customTools; + const customToolsViaCategory = getProvidersByCategory('customTools'); + expect(customToolsViaList).toEqual(customToolsViaCategory); + }); + + it('should have consistent results between hasProvider and listAllProviders', () => { + const allProviders = listAllProviders(); + + // For each blob provider, hasProvider should return true + for (const provider of allProviders.blob) { + expect(hasProvider('blob', provider.type)).toBe(true); + } + + // For each compaction provider, hasProvider should return true + for (const provider of allProviders.compaction) { + expect(hasProvider('compaction', provider.type)).toBe(true); + } + + // For each custom tool provider, hasProvider should return true + for (const provider of allProviders.customTools) { + expect(hasProvider('customTools', provider.type)).toBe(true); + } + + // Non-existent providers should return false + expect(hasProvider('blob', 'nonexistent-provider-xyz')).toBe(false); + expect(hasProvider('compaction', 'nonexistent-provider-xyz')).toBe(false); + expect(hasProvider('customTools', 'nonexistent-provider-xyz')).toBe(false); + }); + }); + + describe('Real-World Scenarios', () => { + it('should support debugging scenario: list all available providers', () => { + const providers = listAllProviders(); + + // Verify we can iterate through all providers for debugging + const summary = { + blobCount: providers.blob.length, + compactionCount: providers.compaction.length, + customToolsCount: providers.customTools.length, + total: + providers.blob.length + + providers.compaction.length + + providers.customTools.length, + }; + + expect(summary.blobCount).toBeGreaterThanOrEqual(2); + expect(summary.compactionCount).toBeGreaterThanOrEqual(2); + expect(summary.total).toBeGreaterThanOrEqual(4); + }); + + it('should support validation scenario: check required providers exist', () => { + // Scenario: App requires local blob storage and reactive-overflow compaction + const requiredProviders = [ + { category: 'blob' as const, type: 'local' }, + { category: 'compaction' as const, type: 'reactive-overflow' }, + ]; + + for (const { category, type } of requiredProviders) { + const exists = hasProvider(category, type); + expect(exists).toBe(true); + } + }); + + it('should support UI scenario: display provider options with metadata', () => { + const blobProviders = getProvidersByCategory('blob'); + + // Verify we can build a UI from provider data + const uiOptions = blobProviders.map((provider) => ({ + id: provider.type, + label: provider.metadata?.displayName || provider.type, + description: provider.metadata?.description || 'No description', + })); + + expect(uiOptions.length).toBeGreaterThanOrEqual(2); + + // Verify all UI options have required fields + for (const option of uiOptions) { + expect(option.id).toBeTruthy(); + expect(option.label).toBeTruthy(); + expect(option.description).toBeTruthy(); + } + }); + + it('should support provider selection scenario: find best available provider', () => { + const blobProviders = getProvidersByCategory('blob'); + + // Scenario: Select first cloud provider, fallback to local + const cloudProvider = blobProviders.find((p) => p.metadata?.requiresNetwork === true); + + const selectedProvider = cloudProvider?.type || 'local'; + + // Should select a valid provider + expect(hasProvider('blob', selectedProvider)).toBe(true); + }); + }); +}); diff --git a/dexto/packages/core/src/providers/discovery.test.ts b/dexto/packages/core/src/providers/discovery.test.ts new file mode 100644 index 00000000..9af37a96 --- /dev/null +++ b/dexto/packages/core/src/providers/discovery.test.ts @@ -0,0 +1,185 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { listAllProviders, getProvidersByCategory, hasProvider } from './discovery.js'; +import { blobStoreRegistry } from '../storage/blob/index.js'; +import { compactionRegistry } from '../context/compaction/index.js'; +import { customToolRegistry } from '../tools/custom-tool-registry.js'; +import type { CustomToolProvider } from '../tools/custom-tool-registry.js'; +import { z } from 'zod'; + +describe('Provider Discovery API', () => { + // Store original registry state + const _originalBlobProviders = blobStoreRegistry.getTypes(); + const _originalCompressionProviders = compactionRegistry.getTypes(); + const _originalCustomToolProviders = customToolRegistry.getTypes(); + + beforeEach(() => { + // Note: We don't clear registries because built-in providers are registered + // on module import. Tests work with the existing state. + }); + + afterEach(() => { + // Clean up any test providers we added + // (Built-in providers remain registered) + }); + + describe('listAllProviders', () => { + it('should return all registered providers grouped by category', () => { + const providers = listAllProviders(); + + expect(providers).toHaveProperty('blob'); + expect(providers).toHaveProperty('compaction'); + expect(providers).toHaveProperty('customTools'); + + expect(Array.isArray(providers.blob)).toBe(true); + expect(Array.isArray(providers.compaction)).toBe(true); + expect(Array.isArray(providers.customTools)).toBe(true); + }); + + it('should include built-in blob providers', () => { + const providers = listAllProviders(); + + // Built-in blob providers: 'local' and 'in-memory' + const types = providers.blob.map((p) => p.type); + expect(types).toContain('local'); + expect(types).toContain('in-memory'); + }); + + it('should include built-in compaction providers', () => { + const providers = listAllProviders(); + + // Built-in compaction providers: 'reactive-overflow' and 'noop' + const types = providers.compaction.map((p) => p.type); + expect(types).toContain('reactive-overflow'); + expect(types).toContain('noop'); + }); + + it('should include provider metadata when available', () => { + const providers = listAllProviders(); + + // Check that blob providers have metadata + const localProvider = providers.blob.find((p) => p.type === 'local'); + expect(localProvider).toBeDefined(); + expect(localProvider?.category).toBe('blob'); + }); + + it('should include custom tool providers', () => { + // Register a test custom tool provider + const testProvider: CustomToolProvider = { + type: 'test-tool', + configSchema: z.object({ type: z.literal('test-tool') }), + create: () => [], + metadata: { + displayName: 'Test Tool', + description: 'A test tool provider', + }, + }; + + customToolRegistry.register(testProvider); + + const providers = listAllProviders(); + const testTool = providers.customTools.find((p) => p.type === 'test-tool'); + + expect(testTool).toBeDefined(); + expect(testTool?.category).toBe('customTools'); + expect(testTool?.metadata?.displayName).toBe('Test Tool'); + + // Cleanup + customToolRegistry.unregister('test-tool'); + }); + }); + + describe('getProvidersByCategory', () => { + it('should return only blob providers when category is blob', () => { + const providers = getProvidersByCategory('blob'); + + expect(Array.isArray(providers)).toBe(true); + expect(providers.length).toBeGreaterThan(0); + providers.forEach((p) => { + expect(p.category).toBe('blob'); + }); + }); + + it('should return only compaction providers when category is compaction', () => { + const providers = getProvidersByCategory('compaction'); + + expect(Array.isArray(providers)).toBe(true); + expect(providers.length).toBeGreaterThan(0); + providers.forEach((p) => { + expect(p.category).toBe('compaction'); + }); + }); + + it('should return only custom tool providers when category is customTools', () => { + const providers = getProvidersByCategory('customTools'); + + expect(Array.isArray(providers)).toBe(true); + // May be empty if no custom tools registered + providers.forEach((p) => { + expect(p.category).toBe('customTools'); + }); + }); + }); + + describe('hasProvider', () => { + it('should return true for registered blob providers', () => { + expect(hasProvider('blob', 'local')).toBe(true); + expect(hasProvider('blob', 'in-memory')).toBe(true); + }); + + it('should return false for unregistered blob providers', () => { + expect(hasProvider('blob', 'nonexistent')).toBe(false); + }); + + it('should return true for registered compaction providers', () => { + expect(hasProvider('compaction', 'reactive-overflow')).toBe(true); + expect(hasProvider('compaction', 'noop')).toBe(true); + }); + + it('should return false for unregistered compaction providers', () => { + expect(hasProvider('compaction', 'nonexistent')).toBe(false); + }); + + it('should work correctly for custom tool providers', () => { + // Initially should not exist + expect(hasProvider('customTools', 'test-tool-2')).toBe(false); + + // Register a test provider + const testProvider: CustomToolProvider = { + type: 'test-tool-2', + configSchema: z.object({ type: z.literal('test-tool-2') }), + create: () => [], + }; + + customToolRegistry.register(testProvider); + expect(hasProvider('customTools', 'test-tool-2')).toBe(true); + + // Cleanup + customToolRegistry.unregister('test-tool-2'); + expect(hasProvider('customTools', 'test-tool-2')).toBe(false); + }); + }); + + describe('DiscoveredProvider structure', () => { + it('should have correct structure for blob providers', () => { + const providers = getProvidersByCategory('blob'); + const localProvider = providers.find((p) => p.type === 'local'); + + expect(localProvider).toBeDefined(); + expect(localProvider).toHaveProperty('type'); + expect(localProvider).toHaveProperty('category'); + expect(localProvider?.type).toBe('local'); + expect(localProvider?.category).toBe('blob'); + }); + + it('should have correct structure for compaction providers', () => { + const providers = getProvidersByCategory('compaction'); + const noopProvider = providers.find((p) => p.type === 'noop'); + + expect(noopProvider).toBeDefined(); + expect(noopProvider).toHaveProperty('type'); + expect(noopProvider).toHaveProperty('category'); + expect(noopProvider?.type).toBe('noop'); + expect(noopProvider?.category).toBe('compaction'); + }); + }); +}); diff --git a/dexto/packages/core/src/providers/discovery.ts b/dexto/packages/core/src/providers/discovery.ts new file mode 100644 index 00000000..c47c6cbe --- /dev/null +++ b/dexto/packages/core/src/providers/discovery.ts @@ -0,0 +1,196 @@ +import { blobStoreRegistry } from '../storage/blob/index.js'; +import { databaseRegistry } from '../storage/database/index.js'; +import { compactionRegistry } from '../context/compaction/index.js'; +import { customToolRegistry } from '../tools/custom-tool-registry.js'; +import { INTERNAL_TOOL_NAMES } from '../tools/internal-tools/constants.js'; +import { INTERNAL_TOOL_REGISTRY } from '../tools/internal-tools/registry.js'; + +/** + * Information about a registered provider. + */ +export interface DiscoveredProvider { + /** Provider type identifier (e.g., 'local', 's3', 'reactive-overflow') */ + type: string; + + /** Provider category */ + category: 'blob' | 'database' | 'compaction' | 'customTools'; + + /** Optional metadata about the provider */ + metadata?: + | { + displayName?: string; + description?: string; + [key: string]: any; + } + | undefined; +} + +/** + * Information about an internal tool for discovery. + */ +export interface InternalToolDiscovery { + /** Tool name identifier (e.g., 'search_history', 'ask_user') */ + name: string; + + /** Human-readable description of what the tool does */ + description: string; +} + +/** + * Discovery result with providers grouped by category. + */ +export interface ProviderDiscovery { + /** Blob storage providers */ + blob: DiscoveredProvider[]; + + /** Database providers */ + database: DiscoveredProvider[]; + + /** Compaction strategy providers */ + compaction: DiscoveredProvider[]; + + /** Custom tool providers */ + customTools: DiscoveredProvider[]; + + /** Internal tools available for configuration */ + internalTools: InternalToolDiscovery[]; +} + +/** + * Provider category type. + */ +export type ProviderCategory = 'blob' | 'database' | 'compaction' | 'customTools'; + +/** + * List all registered providers across all registries. + * + * This function is useful for debugging and building UIs that need to display + * available providers. It queries all provider registries and returns a + * comprehensive view of what's currently registered. + * + * @returns Object with providers grouped by category + * + * @example + * ```typescript + * const providers = listAllProviders(); + * console.log('Available blob providers:', providers.blob); + * console.log('Available compaction providers:', providers.compaction); + * console.log('Available custom tool providers:', providers.customTools); + * ``` + */ +export function listAllProviders(): ProviderDiscovery { + // Get blob store providers + const blobProviders = blobStoreRegistry.getProviders().map((provider) => { + const info: DiscoveredProvider = { + type: provider.type, + category: 'blob', + }; + if (provider.metadata) { + info.metadata = provider.metadata; + } + return info; + }); + + // Get compaction providers + const compactionProviders = compactionRegistry.getAll().map((provider) => { + const info: DiscoveredProvider = { + type: provider.type, + category: 'compaction', + }; + if (provider.metadata) { + info.metadata = provider.metadata; + } + return info; + }); + + // Get custom tool providers + const customToolProviders = customToolRegistry.getTypes().map((type) => { + const provider = customToolRegistry.get(type); + const info: DiscoveredProvider = { + type, + category: 'customTools', + }; + if (provider?.metadata) { + info.metadata = provider.metadata; + } + return info; + }); + + // Get database providers + const databaseProviders = databaseRegistry.getProviders().map((provider) => { + const info: DiscoveredProvider = { + type: provider.type, + category: 'database', + }; + if (provider.metadata) { + info.metadata = provider.metadata; + } + return info; + }); + + // Get internal tools + const internalTools: InternalToolDiscovery[] = INTERNAL_TOOL_NAMES.map((name) => ({ + name, + description: INTERNAL_TOOL_REGISTRY[name].description, + })); + + return { + blob: blobProviders, + database: databaseProviders, + compaction: compactionProviders, + customTools: customToolProviders, + internalTools, + }; +} + +/** + * Get all providers for a specific category. + * + * @param category - The provider category to query + * @returns Array of provider information for the specified category + * + * @example + * ```typescript + * const blobProviders = getProvidersByCategory('blob'); + * blobProviders.forEach(p => { + * console.log(`${p.type}: ${p.metadata?.displayName}`); + * }); + * ``` + */ +export function getProvidersByCategory(category: ProviderCategory): DiscoveredProvider[] { + const allProviders = listAllProviders(); + return allProviders[category]; +} + +/** + * Check if a specific provider exists in a category. + * + * @param category - The provider category to check + * @param type - The provider type identifier + * @returns True if the provider is registered, false otherwise + * + * @example + * ```typescript + * if (hasProvider('blob', 's3')) { + * console.log('S3 blob storage is available'); + * } + * + * if (hasProvider('compaction', 'reactive-overflow')) { + * console.log('Reactive overflow compaction is available'); + * } + * ``` + */ +export function hasProvider(category: ProviderCategory, type: string): boolean { + switch (category) { + case 'blob': + return blobStoreRegistry.has(type); + case 'database': + return databaseRegistry.has(type); + case 'compaction': + return compactionRegistry.has(type); + case 'customTools': + return customToolRegistry.has(type); + default: + return false; + } +} diff --git a/dexto/packages/core/src/providers/index.ts b/dexto/packages/core/src/providers/index.ts new file mode 100644 index 00000000..b6b24b52 --- /dev/null +++ b/dexto/packages/core/src/providers/index.ts @@ -0,0 +1,32 @@ +/** + * Provider Infrastructure + * + * This module provides: + * 1. BaseRegistry - Generic base class for building type-safe provider registries + * 2. Discovery API - Utilities for querying registered providers across all registries + * + * Useful for: + * - Building custom provider registries with consistent behavior + * - Debugging: See what providers are available at runtime + * - UIs: Build dynamic interfaces that show available providers + * - Configuration validation: Check if required providers are registered + * + * @example + * ```typescript + * import { BaseRegistry, listAllProviders, hasProvider } from '@dexto/core'; + * + * // Create a custom registry + * class MyRegistry extends BaseRegistry { + * constructor() { + * super(myErrorFactory); + * } + * } + * + * // List all providers + * const providers = listAllProviders(); + * console.log('Blob providers:', providers.blob); + * ``` + */ + +export * from './base-registry.js'; +export * from './discovery.js'; diff --git a/dexto/packages/core/src/resources/error-codes.ts b/dexto/packages/core/src/resources/error-codes.ts new file mode 100644 index 00000000..c9250f75 --- /dev/null +++ b/dexto/packages/core/src/resources/error-codes.ts @@ -0,0 +1,13 @@ +export const ResourceErrorCodes = { + INVALID_URI_FORMAT: 'resource_invalid_uri_format', + EMPTY_URI: 'resource_empty_uri', + RESOURCE_NOT_FOUND: 'resource_not_found', + PROVIDER_NOT_INITIALIZED: 'resource_provider_not_initialized', + PROVIDER_NOT_AVAILABLE: 'resource_provider_not_available', + READ_FAILED: 'resource_read_failed', + ACCESS_DENIED: 'resource_access_denied', + NO_SUITABLE_PROVIDER: 'resource_no_suitable_provider', + PROVIDER_ERROR: 'resource_provider_error', +} as const; + +export type ResourceErrorCode = (typeof ResourceErrorCodes)[keyof typeof ResourceErrorCodes]; diff --git a/dexto/packages/core/src/resources/errors.ts b/dexto/packages/core/src/resources/errors.ts new file mode 100644 index 00000000..d1fc2b5b --- /dev/null +++ b/dexto/packages/core/src/resources/errors.ts @@ -0,0 +1,144 @@ +import { DextoRuntimeError } from '@core/errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { ResourceErrorCodes } from './error-codes.js'; + +/** + * Resource management error factory + * Creates properly typed errors for resource operations + */ +export class ResourceError { + private static redactUri(uri: string): string { + try { + const u = new URL(uri); + if (u.username) u.username = '***'; + if (u.password) u.password = '***'; + u.searchParams.forEach((_, k) => { + if (/token|key|secret|sig|pwd|password/i.test(k)) u.searchParams.set(k, '***'); + }); + return u.toString(); + } catch { + return uri + .replace(/\/\/([^@]+)@/, '//***@') + .replace(/((?:token|key|secret|sig|pwd|password)=)[^&]*/gi, '$1***'); + } + } + + private static toMessageAndRaw(reason: unknown): { message: string; raw: unknown } { + if (reason instanceof Error) { + return { + message: reason.message, + raw: { name: reason.name, message: reason.message, stack: reason.stack }, + }; + } + if (typeof reason === 'string') return { message: reason, raw: reason }; + try { + return { message: JSON.stringify(reason), raw: reason }; + } catch { + return { message: String(reason), raw: reason }; + } + } + // URI format and parsing errors + static invalidUriFormat(uri: string, expected?: string) { + return new DextoRuntimeError( + ResourceErrorCodes.INVALID_URI_FORMAT, + ErrorScope.RESOURCE, + ErrorType.USER, + `Invalid resource URI format: '${ResourceError.redactUri(uri)}'${expected ? ` (expected ${expected})` : ''}`, + { uri: ResourceError.redactUri(uri), uriRaw: uri, expected }, + expected ? `Use format: ${expected}` : 'Check the resource URI format' + ); + } + + static emptyUri() { + return new DextoRuntimeError( + ResourceErrorCodes.EMPTY_URI, + ErrorScope.RESOURCE, + ErrorType.USER, + 'Resource URI cannot be empty', + {}, + 'Provide a valid resource URI' + ); + } + + // Resource discovery and access errors + static resourceNotFound(uri: string) { + return new DextoRuntimeError( + ResourceErrorCodes.RESOURCE_NOT_FOUND, + ErrorScope.RESOURCE, + ErrorType.NOT_FOUND, + `Resource not found: '${ResourceError.redactUri(uri)}'`, + { uri: ResourceError.redactUri(uri), uriRaw: uri }, + 'Check that the resource exists and is accessible' + ); + } + + static providerNotInitialized(providerType: string, uri: string) { + return new DextoRuntimeError( + ResourceErrorCodes.PROVIDER_NOT_INITIALIZED, + ErrorScope.RESOURCE, + ErrorType.SYSTEM, + `${providerType} resource provider not initialized for: '${ResourceError.redactUri(uri)}'`, + { providerType, uri: ResourceError.redactUri(uri), uriRaw: uri }, + 'Ensure the resource provider is properly configured' + ); + } + + static providerNotAvailable(providerType: string) { + return new DextoRuntimeError( + ResourceErrorCodes.PROVIDER_NOT_AVAILABLE, + ErrorScope.RESOURCE, + ErrorType.SYSTEM, + `${providerType} resource provider is not available`, + { providerType }, + 'Check resource provider configuration and availability' + ); + } + + // Content access errors + static readFailed(uri: string, reason: unknown) { + const { message: reasonMsg, raw: reasonRaw } = ResourceError.toMessageAndRaw(reason); + return new DextoRuntimeError( + ResourceErrorCodes.READ_FAILED, + ErrorScope.RESOURCE, + ErrorType.SYSTEM, + `Failed to read resource '${ResourceError.redactUri(uri)}': ${reasonMsg}`, + { uri: ResourceError.redactUri(uri), uriRaw: uri, reason: reasonMsg, reasonRaw }, + 'Check resource permissions and availability' + ); + } + + static accessDenied(uri: string) { + return new DextoRuntimeError( + ResourceErrorCodes.ACCESS_DENIED, + ErrorScope.RESOURCE, + ErrorType.FORBIDDEN, + `Access denied to resource: '${ResourceError.redactUri(uri)}'`, + { uri: ResourceError.redactUri(uri), uriRaw: uri }, + 'Ensure you have permission to access this resource' + ); + } + + // Provider coordination errors + static noSuitableProvider(uri: string) { + return new DextoRuntimeError( + ResourceErrorCodes.NO_SUITABLE_PROVIDER, + ErrorScope.RESOURCE, + ErrorType.NOT_FOUND, + `No suitable provider found for resource: '${ResourceError.redactUri(uri)}'`, + { uri: ResourceError.redactUri(uri), uriRaw: uri }, + 'Check that the resource type is supported' + ); + } + + static providerError(providerType: string, operation: string, reason: unknown) { + const { message: reasonMsg, raw: reasonRaw } = ResourceError.toMessageAndRaw(reason); + return new DextoRuntimeError( + ResourceErrorCodes.PROVIDER_ERROR, + ErrorScope.RESOURCE, + ErrorType.SYSTEM, + `${providerType} provider failed during ${operation}: ${reasonMsg}`, + { providerType, operation, reason: reasonMsg, reasonRaw }, + 'Check provider configuration and logs for details' + ); + } +} diff --git a/dexto/packages/core/src/resources/handlers/blob-handler.ts b/dexto/packages/core/src/resources/handlers/blob-handler.ts new file mode 100644 index 00000000..2b1e1d23 --- /dev/null +++ b/dexto/packages/core/src/resources/handlers/blob-handler.ts @@ -0,0 +1,259 @@ +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { ResourceError } from '../errors.js'; +import type { ResourceMetadata } from '../types.js'; +import type { ReadResourceResult } from '@modelcontextprotocol/sdk/types.js'; +import type { BlobStore, StoredBlobMetadata } from '../../storage/blob/types.js'; +import type { ValidatedBlobResourceConfig } from '../schemas.js'; +import type { InternalResourceHandler, InternalResourceServices } from './types.js'; + +export class BlobResourceHandler implements InternalResourceHandler { + private config: ValidatedBlobResourceConfig; + private blobStore: BlobStore; + private logger: IDextoLogger; + + constructor(config: ValidatedBlobResourceConfig, blobStore: BlobStore, logger: IDextoLogger) { + this.config = config; + this.blobStore = blobStore; + this.logger = logger.createChild(DextoLogComponent.RESOURCE); + } + + getType(): string { + return 'blob'; + } + + async initialize(_services: InternalResourceServices): Promise { + // Config and blobStore are set in constructor + this.logger.debug('BlobResourceHandler initialized with BlobStore'); + } + + async listResources(): Promise { + this.logger.debug('🔍 BlobResourceHandler.listResources() called'); + + try { + const stats = await this.blobStore.getStats(); + this.logger.debug( + `📊 BlobStore stats: ${stats.count} blobs, backend: ${stats.backendType}` + ); + const resources: ResourceMetadata[] = []; + + // List individual blobs from the store + try { + const blobs = await this.blobStore.listBlobs(); + this.logger.debug(`📄 Found ${blobs.length} individual blobs`); + + for (const blob of blobs) { + // Filter out 'system' source blobs (prompt .md files, custom prompt attachments). + // These are internal config accessed via prompt system, not @-referenceable resources. + // Prevents clutter and confusion in resource autocomplete. + if (blob.metadata.source === 'system') { + continue; + } + + // Generate a user-friendly name with proper extension + const displayName = this.generateBlobDisplayName(blob.metadata, blob.id); + const friendlyType = this.getFriendlyType(blob.metadata.mimeType); + + resources.push({ + uri: blob.uri, + name: displayName, + description: `${friendlyType} (${this.formatSize(blob.metadata.size)})${blob.metadata.source ? ` • ${blob.metadata.source}` : ''}`, + source: 'internal', + size: blob.metadata.size, + mimeType: blob.metadata.mimeType, + lastModified: new Date(blob.metadata.createdAt), + metadata: { + type: 'blob', + source: blob.metadata.source, + hash: blob.metadata.hash, + createdAt: blob.metadata.createdAt, + originalName: blob.metadata.originalName, + }, + }); + } + } catch (error) { + this.logger.warn(`Failed to list individual blobs: ${String(error)}`); + } + + this.logger.debug(`✅ BlobResourceHandler returning ${resources.length} resources`); + return resources; + } catch (error) { + this.logger.warn(`Failed to list blob resources: ${String(error)}`); + return []; + } + } + + canHandle(uri: string): boolean { + return uri.startsWith('blob:'); + } + + async readResource(uri: string): Promise { + if (!this.canHandle(uri)) { + throw ResourceError.noSuitableProvider(uri); + } + + try { + // Extract blob ID from URI (remove 'blob:' prefix) + const blobId = uri.substring(5); + + // Validate blob ID + if (!blobId) { + throw ResourceError.readFailed(uri, new Error('Invalid blob URI: missing blob ID')); + } + + // Special case: blob store info + if (blobId === 'store') { + const stats = await this.blobStore.getStats(); + return { + contents: [ + { + uri, + mimeType: 'application/json', + text: JSON.stringify(stats, null, 2), + }, + ], + }; + } + + // Retrieve actual blob data using blob ID + const result = await this.blobStore.retrieve(blobId, 'base64'); + + return { + contents: [ + { + uri, + mimeType: result.metadata.mimeType || 'application/octet-stream', + blob: result.data as string, // base64 data from retrieve call + }, + ], + _meta: { + size: result.metadata.size, + createdAt: result.metadata.createdAt, + originalName: result.metadata.originalName, + source: result.metadata.source, + }, + }; + } catch (error) { + if (error instanceof ResourceError) { + throw error; + } + throw ResourceError.readFailed(uri, error); + } + } + + async refresh(): Promise { + // BlobStore doesn't need refresh as it's not file-system based scanning + // But we can perform cleanup of old blobs if configured + try { + await this.blobStore.cleanup(); + this.logger.debug('Blob store cleanup completed'); + } catch (error) { + this.logger.warn(`Blob store cleanup failed: ${String(error)}`); + } + } + + getBlobStore(): BlobStore { + return this.blobStore; + } + + /** + * Generate a user-friendly display name for a blob with proper file extension + */ + private generateBlobDisplayName(metadata: StoredBlobMetadata, _blobId: string): string { + // If we have an original name with extension, use it + if (metadata.originalName && metadata.originalName.includes('.')) { + return metadata.originalName; + } + + // Generate a name based on MIME type and content + let baseName = + metadata.originalName || this.generateNameFromType(metadata.mimeType, metadata.source); + const extension = this.getExtensionFromMimeType(metadata.mimeType); + + // Add extension if not present + if (extension && !baseName.toLowerCase().endsWith(extension)) { + baseName += extension; + } + + return baseName; + } + + /** + * Generate a descriptive base name from MIME type and source + */ + private generateNameFromType(mimeType: string, source?: string): string { + if (mimeType.startsWith('image/')) { + if (source === 'user') return 'uploaded-image'; + if (source === 'tool') return 'generated-image'; + return 'image'; + } + if (mimeType.startsWith('text/')) { + if (source === 'tool') return 'tool-output'; + return 'text-file'; + } + if (mimeType.startsWith('application/pdf')) { + return 'document'; + } + if (mimeType.startsWith('audio/')) { + return 'audio-file'; + } + if (mimeType.startsWith('video/')) { + return 'video-file'; + } + + // Default based on source + if (source === 'user') return 'user-upload'; + if (source === 'tool') return 'tool-result'; + return 'file'; + } + + /** + * Get file extension from MIME type + */ + private getExtensionFromMimeType(mimeType: string): string { + const mimeToExt: Record = { + 'image/jpeg': '.jpg', + 'image/png': '.png', + 'image/gif': '.gif', + 'image/webp': '.webp', + 'image/svg+xml': '.svg', + 'text/plain': '.txt', + 'text/markdown': '.md', + 'text/html': '.html', + 'text/css': '.css', + 'application/json': '.json', + 'application/pdf': '.pdf', + 'application/xml': '.xml', + 'audio/mpeg': '.mp3', + 'audio/wav': '.wav', + 'video/mp4': '.mp4', + 'video/webm': '.webm', + }; + + return mimeToExt[mimeType] || ''; + } + + /** + * Convert MIME type to user-friendly type description + */ + private getFriendlyType(mimeType: string): string { + if (mimeType.startsWith('image/')) return 'Image'; + if (mimeType.startsWith('text/')) return 'Text File'; + if (mimeType.startsWith('audio/')) return 'Audio File'; + if (mimeType.startsWith('video/')) return 'Video File'; + if (mimeType === 'application/pdf') return 'PDF Document'; + if (mimeType === 'application/json') return 'JSON Data'; + return 'File'; + } + + /** + * Format file size in human-readable format + */ + private formatSize(bytes: number): string { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]; + } +} diff --git a/dexto/packages/core/src/resources/handlers/factory.ts b/dexto/packages/core/src/resources/handlers/factory.ts new file mode 100644 index 00000000..0e039b7f --- /dev/null +++ b/dexto/packages/core/src/resources/handlers/factory.ts @@ -0,0 +1,37 @@ +import { ResourceError } from '../errors.js'; +import { FileSystemResourceHandler } from './filesystem-handler.js'; +import { BlobResourceHandler } from './blob-handler.js'; +import type { InternalResourceServices, InternalResourceHandler } from './types.js'; +import type { ValidatedInternalResourceConfig } from '../schemas.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; + +/** + * Factory function for creating internal resource handlers + */ +export function createInternalResourceHandler( + config: ValidatedInternalResourceConfig, + services: InternalResourceServices, + logger: IDextoLogger +): InternalResourceHandler { + const type = config.type; + if (type === 'filesystem') { + // Pass blob storage path to filesystem handler to avoid scanning blob directories + const blobStoragePath = services.blobStore.getStoragePath(); + return new FileSystemResourceHandler(config, logger, blobStoragePath); + } + if (type === 'blob') { + return new BlobResourceHandler(config, services.blobStore, logger); + } + throw ResourceError.providerError( + 'Internal', + 'createInternalResourceHandler', + `Unsupported internal resource handler type: ${type}` + ); +} + +/** + * Get all supported internal resource handler types + */ +export function getInternalResourceHandlerTypes(): string[] { + return ['filesystem', 'blob']; +} diff --git a/dexto/packages/core/src/resources/handlers/filesystem-handler.ts b/dexto/packages/core/src/resources/handlers/filesystem-handler.ts new file mode 100644 index 00000000..204799ac --- /dev/null +++ b/dexto/packages/core/src/resources/handlers/filesystem-handler.ts @@ -0,0 +1,432 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { ResourceError } from '../errors.js'; +import type { ResourceMetadata } from '../types.js'; +import type { ReadResourceResult } from '@modelcontextprotocol/sdk/types.js'; +import type { ValidatedFileSystemResourceConfig } from '../schemas.js'; +import type { InternalResourceHandler, InternalResourceServices } from './types.js'; + +export class FileSystemResourceHandler implements InternalResourceHandler { + private config: ValidatedFileSystemResourceConfig; + private resourcesCache: Map = new Map(); + private visitedPaths: Set = new Set(); + private fileCount: number = 0; + private canonicalRoots: string[] = []; + private blobStoragePath: string | undefined; + private logger: IDextoLogger; + + constructor( + config: ValidatedFileSystemResourceConfig, + logger: IDextoLogger, + blobStoragePath?: string + ) { + this.config = config; + this.logger = logger.createChild(DextoLogComponent.RESOURCE); + this.blobStoragePath = blobStoragePath; + } + + getType(): string { + return 'filesystem'; + } + + async initialize(_services: InternalResourceServices): Promise { + // Config is set in constructor, just do async initialization + this.canonicalRoots = []; + for (const configPath of this.config.paths) { + try { + const canonicalRoot = await fs.realpath(path.resolve(configPath)); + this.canonicalRoots.push(canonicalRoot); + } catch (error) { + this.logger.warn( + `Failed to canonicalize root path '${configPath}': ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + await this.buildResourceCache(); + } + + async listResources(): Promise { + return Array.from(this.resourcesCache.values()); + } + + canHandle(uri: string): boolean { + return uri.startsWith('fs://'); + } + + private isPathAllowed(canonicalPath: string): boolean { + return this.canonicalRoots.some((root) => { + const normalizedPath = path.normalize(canonicalPath); + const normalizedRoot = path.normalize(root); + return ( + normalizedPath.startsWith(normalizedRoot + path.sep) || + normalizedPath === normalizedRoot + ); + }); + } + + /** + * Check if a path is a blob storage directory that should be excluded + * from filesystem resource scanning to avoid conflicts with BlobResourceHandler + */ + private isBlobStorageDirectory(canonicalPath: string): boolean { + if (!this.blobStoragePath) { + return false; + } + + // Check if this path is under the actual blob storage directory + const normalizedPath = path.normalize(canonicalPath); + const normalizedBlobPath = path.normalize(this.blobStoragePath); + + return ( + normalizedPath === normalizedBlobPath || + normalizedPath.startsWith(normalizedBlobPath + path.sep) + ); + } + + async readResource(uri: string): Promise { + if (!this.canHandle(uri)) { + throw ResourceError.noSuitableProvider(uri); + } + + const filePath = uri.replace('fs://', ''); + const resolvedPath = path.resolve(filePath); + + let canonicalPath: string; + try { + canonicalPath = await fs.realpath(resolvedPath); + } catch (_error) { + throw ResourceError.resourceNotFound(uri); + } + + if (!this.isPathAllowed(canonicalPath)) { + throw ResourceError.accessDenied(uri); + } + + try { + const stat = await fs.stat(canonicalPath); + if (stat.size > 10 * 1024 * 1024) { + throw ResourceError.readFailed(uri, `File too large (${stat.size} bytes)`); + } + + if (this.isBinaryFile(canonicalPath)) { + return { + contents: [ + { + uri, + mimeType: 'text/plain', + text: `[Binary file: ${path.basename(canonicalPath)} (${stat.size} bytes)]`, + }, + ], + _meta: { + isBinary: true, + size: stat.size, + originalMimeType: this.getMimeType(canonicalPath), + }, + }; + } + + const content = await fs.readFile(canonicalPath, 'utf-8'); + return { + contents: [ + { + uri, + mimeType: this.getMimeType(canonicalPath), + text: content, + }, + ], + _meta: { size: stat.size }, + }; + } catch (error) { + throw ResourceError.readFailed(uri, error); + } + } + + private isBinaryFile(filePath: string): boolean { + const ext = path.extname(filePath).toLowerCase(); + const binaryExtensions = [ + '.exe', + '.dll', + '.so', + '.dylib', + '.bin', + '.dat', + '.db', + '.sqlite', + '.jpg', + '.jpeg', + '.png', + '.gif', + '.bmp', + '.ico', + '.tiff', + '.webp', + '.mp3', + '.mp4', + '.avi', + '.mov', + '.wmv', + '.flv', + '.mkv', + '.webm', + '.pdf', + '.zip', + '.tar', + '.gz', + '.7z', + '.rar', + '.dmg', + '.iso', + '.woff', + '.woff2', + '.ttf', + '.otf', + '.eot', + '.class', + '.jar', + '.war', + '.ear', + '.o', + '.obj', + '.lib', + '.a', + ]; + return binaryExtensions.includes(ext); + } + + async refresh(): Promise { + await this.buildResourceCache(); + } + + private async buildResourceCache(): Promise { + if (!this.config) return; + + this.resourcesCache.clear(); + this.visitedPaths.clear(); + this.fileCount = 0; + + const { maxFiles, paths } = this.config; + + for (const configPath of paths) { + if (this.fileCount >= maxFiles) { + this.logger.warn(`Reached maximum file limit (${maxFiles}), stopping scan`); + break; + } + + try { + const root = await fs.realpath(path.resolve(configPath)); + await this.scanPath(root, 0, root); + } catch (error) { + this.logger.warn( + `Failed to scan path '${configPath}': ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + this.logger.debug( + `FileSystem resources cached: ${this.resourcesCache.size} resources (${this.fileCount} files scanned)` + ); + } + + private async scanPath( + targetPath: string, + currentDepth: number, + rootBase?: string + ): Promise { + const resolvedPath = path.resolve(targetPath); + let canonical: string; + try { + canonical = await fs.realpath(resolvedPath); + } catch { + return; + } + if (!this.isPathAllowed(canonical)) return; + + // Skip blob storage directories to avoid conflicts with BlobResourceHandler + if (this.isBlobStorageDirectory(canonical)) { + return; + } + + // Config has defaults already applied by schema validation + const { maxDepth, maxFiles, includeHidden, includeExtensions } = this.config; + + if (this.fileCount >= maxFiles) return; + if (currentDepth > maxDepth) { + // silly to avoid spamming the logs + this.logger.silly(`Skipping path due to depth limit (${maxDepth}): ${canonical}`); + return; + } + if (this.visitedPaths.has(canonical)) return; + this.visitedPaths.add(canonical); + + try { + const stat = await fs.stat(canonical); + if (stat.isFile()) { + if (!this.shouldIncludeFile(canonical, includeExtensions, includeHidden)) return; + + // Use absolute canonical path to ensure readResource resolves correctly + const uri = `fs://${canonical.replace(/\\/g, '/')}`; + this.resourcesCache.set(uri, { + uri, + name: this.generateCleanFileName(canonical), + description: 'Filesystem resource', + source: 'internal', + size: stat.size, + lastModified: stat.mtime, + }); + this.fileCount++; + return; + } + + if (stat.isDirectory()) { + const entries = await fs.readdir(canonical); + for (const entry of entries) { + const entryPath = path.join(canonical, entry); + await this.scanPath( + entryPath, + currentDepth + 1, + rootBase ?? this.canonicalRoots.find((r) => canonical.startsWith(r)) + ); + } + } + } catch (error) { + this.logger.debug( + `Skipping inaccessible path: ${canonical} - ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + private shouldIncludeFile( + filePath: string, + includeExtensions: string[], + includeHidden: boolean + ): boolean { + const basename = path.basename(filePath).toLowerCase(); + const ext = path.extname(filePath).toLowerCase(); + + if (basename.startsWith('.')) { + if (!includeHidden) { + const allowedDotfiles = [ + '.gitignore', + '.env', + '.env.example', + '.npmignore', + '.dockerignore', + '.editorconfig', + ]; + if (!allowedDotfiles.includes(basename)) return false; + } + if (basename === '.env' || basename.startsWith('.env.')) return true; + } + + if (!ext) { + const commonNoExtFiles = [ + 'dockerfile', + 'makefile', + 'readme', + 'license', + 'changelog', + 'contributing', + ]; + return commonNoExtFiles.some((common) => basename.includes(common)); + } + + if (basename === '.gitignore') return true; + + return includeExtensions.includes(ext); + } + + /** + * Generate a clean, user-friendly filename from a potentially messy path + */ + private generateCleanFileName(filePath: string): string { + const basename = path.basename(filePath); + + // For screenshot files with timestamps, clean them up + if (basename.startsWith('Screenshot ') && basename.includes(' at ')) { + // "Screenshot 2025-09-14 at 11.39.20 PM.png" -> "Screenshot 2025-09-14.png" + const match = basename.match(/^Screenshot (\d{4}-\d{2}-\d{2}).*?(\.[^.]+)$/); + if (match) { + return `Screenshot ${match[1]}${match[2]}`; + } + } + + // For other temp files, just use the basename as-is + // but remove any weird prefixes or temp markers + if (basename.length > 50) { + // If filename is too long, try to extract meaningful parts + const ext = path.extname(basename); + const nameWithoutExt = path.basename(basename, ext); + + // Look for recognizable patterns + const patterns = [ + /Screenshot.*(\d{4}-\d{2}-\d{2})/, + /([A-Za-z\s]+\d{4}-\d{2}-\d{2})/, + /(image|photo|file).*(\d+)/i, + ]; + + for (const pattern of patterns) { + const match = nameWithoutExt.match(pattern); + if (match) { + return `${match[1] || match[0]}${ext}`; + } + } + + // If no pattern matches, truncate intelligently + if (nameWithoutExt.length > 30) { + return `${nameWithoutExt.substring(0, 30)}...${ext}`; + } + } + + return basename; + } + + private getMimeType(filePath: string): string { + const ext = path.extname(filePath).toLowerCase(); + const mimeTypes: Record = { + '.txt': 'text/plain', + '.md': 'text/markdown', + '.markdown': 'text/markdown', + '.html': 'text/html', + '.htm': 'text/html', + '.css': 'text/css', + '.js': 'text/javascript', + '.mjs': 'text/javascript', + '.jsx': 'text/javascript', + '.ts': 'text/typescript', + '.tsx': 'text/typescript', + '.vue': 'text/x-vue', + '.json': 'application/json', + '.xml': 'text/xml', + '.yaml': 'text/yaml', + '.yml': 'text/yaml', + '.toml': 'text/toml', + '.ini': 'text/plain', + '.cfg': 'text/plain', + '.conf': 'text/plain', + '.py': 'text/x-python', + '.rb': 'text/x-ruby', + '.php': 'text/x-php', + '.java': 'text/x-java', + '.kt': 'text/x-kotlin', + '.swift': 'text/x-swift', + '.go': 'text/x-go', + '.rs': 'text/x-rust', + '.cpp': 'text/x-c++', + '.c': 'text/x-c', + '.h': 'text/x-c', + '.hpp': 'text/x-c++', + '.sh': 'text/x-shellscript', + '.bash': 'text/x-shellscript', + '.zsh': 'text/x-shellscript', + '.fish': 'text/x-shellscript', + '.sql': 'text/x-sql', + '.rst': 'text/x-rst', + '.tex': 'text/x-tex', + '.dockerfile': 'text/x-dockerfile', + }; + return mimeTypes[ext] || 'text/plain'; + } +} diff --git a/dexto/packages/core/src/resources/handlers/types.ts b/dexto/packages/core/src/resources/handlers/types.ts new file mode 100644 index 00000000..b494086f --- /dev/null +++ b/dexto/packages/core/src/resources/handlers/types.ts @@ -0,0 +1,16 @@ +import type { ResourceMetadata } from '../types.js'; +import type { ReadResourceResult } from '@modelcontextprotocol/sdk/types.js'; +import type { BlobStore } from '../../storage/blob/types.js'; + +export type InternalResourceServices = { + blobStore: BlobStore; +}; + +export interface InternalResourceHandler { + getType(): string; + initialize(services: InternalResourceServices): Promise; + listResources(): Promise; + readResource(uri: string): Promise; + canHandle(uri: string): boolean; + refresh?(): Promise; +} diff --git a/dexto/packages/core/src/resources/index.ts b/dexto/packages/core/src/resources/index.ts new file mode 100644 index 00000000..f39a8cd6 --- /dev/null +++ b/dexto/packages/core/src/resources/index.ts @@ -0,0 +1,66 @@ +/** + * Resource management module exports + * Organized into thematic sub-barrels for better tree-shaking and maintainability + */ + +// Core resource types and manager +export type { ResourceSource, ResourceMetadata, ResourceProvider, ResourceSet } from './types.js'; + +export type { + InternalResourcesConfig, + ValidatedInternalResourcesConfig, + ValidatedInternalResourceConfig, + ValidatedFileSystemResourceConfig, + ValidatedBlobResourceConfig, +} from './schemas.js'; + +export { ResourceManager } from './manager.js'; +export { ResourceError } from './errors.js'; +export { ResourceErrorCodes } from './error-codes.js'; + +// Internal resources provider and handlers +export type { InternalResourceHandler, InternalResourceServices } from './handlers/types.js'; +export { InternalResourcesProvider } from './internal-provider.js'; +export { + createInternalResourceHandler, + getInternalResourceHandlerTypes, +} from './handlers/factory.js'; + +// Resource reference parsing and expansion +/** + * Resource References + * + * Resource references allow you to include resource content in messages using @ syntax: + * + * Syntax: + * - Simple name: @filename.txt + * - URI with brackets: @ + * - Server-scoped: @servername:resource-identifier + * + * Important: @ symbols are ONLY treated as resource references if they: + * 1. Are at the start of the message, OR + * 2. Are preceded by whitespace (space, tab, newline) + * + * This means email addresses like "user@example.com" are automatically ignored + * without requiring any escape sequences. No special handling needed! + * + * Examples: + * - "@myfile.txt" → resource reference (at start) + * - "Check @myfile.txt" → resource reference (after space) + * - "user@example.com" → literal text (no leading space) + * - "See @file1.txt and email user@example.com" → only @file1.txt is a reference + */ +export type { ResourceReference, ResourceExpansionResult } from './reference-parser.js'; +export { + parseResourceReferences, + resolveResourceReferences, + expandMessageReferences, + formatResourceContent, +} from './reference-parser.js'; + +// Schemas and validation +export { + InternalResourceConfigSchema, + InternalResourcesSchema, + isInternalResourcesEnabled, +} from './schemas.js'; diff --git a/dexto/packages/core/src/resources/internal-provider.ts b/dexto/packages/core/src/resources/internal-provider.ts new file mode 100644 index 00000000..398557e8 --- /dev/null +++ b/dexto/packages/core/src/resources/internal-provider.ts @@ -0,0 +1,146 @@ +import { ResourceProvider, ResourceMetadata, ResourceSource } from './types.js'; +import { ReadResourceResult } from '@modelcontextprotocol/sdk/types.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import { createInternalResourceHandler } from './handlers/factory.js'; +import type { InternalResourceHandler, InternalResourceServices } from './handlers/types.js'; +import type { + ValidatedInternalResourcesConfig, + ValidatedInternalResourceConfig, +} from './schemas.js'; +import { InternalResourceConfigSchema } from './schemas.js'; +import { ResourceError } from './errors.js'; + +export class InternalResourcesProvider implements ResourceProvider { + private config: ValidatedInternalResourcesConfig; + private handlers: Map = new Map(); + private services: InternalResourceServices; + private logger: IDextoLogger; + + constructor( + config: ValidatedInternalResourcesConfig, + services: InternalResourceServices, + logger: IDextoLogger + ) { + this.config = config; + this.services = services; + this.logger = logger.createChild(DextoLogComponent.RESOURCE); + this.logger.debug( + `InternalResourcesProvider initialized with config: ${JSON.stringify(config)}` + ); + } + + async initialize(): Promise { + if (!this.config.enabled || this.config.resources.length === 0) { + this.logger.debug('Internal resources disabled or no resources configured'); + return; + } + + for (const resourceConfig of this.config.resources) { + try { + const parsedConfig = InternalResourceConfigSchema.parse(resourceConfig); + const handler = createInternalResourceHandler( + parsedConfig, + this.services, + this.logger + ); + await handler.initialize(this.services); + this.handlers.set(resourceConfig.type, handler); + this.logger.debug(`Initialized ${resourceConfig.type} resource handler`); + } catch (error) { + this.logger.error(`Failed to initialize ${resourceConfig.type} resource handler`, { + error: error instanceof Error ? error.message : String(error), + }); + } + } + + this.logger.debug( + `InternalResourcesProvider initialized with ${this.handlers.size} resource handlers` + ); + } + + getSource(): ResourceSource { + return 'internal'; + } + + async listResources(): Promise { + const allResources: ResourceMetadata[] = []; + for (const [type, handler] of this.handlers.entries()) { + try { + const resources = await handler.listResources(); + allResources.push(...resources); + } catch (error) { + this.logger.error( + `Failed to list resources from ${type} handler: ${error instanceof Error ? error.message : String(error)}`, + { error: error instanceof Error ? error.message : String(error) } + ); + } + } + return allResources; + } + + async hasResource(uri: string): Promise { + for (const handler of this.handlers.values()) { + if (handler.canHandle(uri)) return true; + } + return false; + } + + async readResource(uri: string): Promise { + for (const [type, handler] of this.handlers.entries()) { + if (handler.canHandle(uri)) { + try { + return await handler.readResource(uri); + } catch (error) { + this.logger.error( + `Failed to read resource ${uri} from ${type} handler: ${error instanceof Error ? error.message : String(error)}`, + { error: error instanceof Error ? error.message : String(error) } + ); + throw error; + } + } + } + throw ResourceError.noSuitableProvider(uri); + } + + async refresh(): Promise { + for (const [type, handler] of this.handlers.entries()) { + if (handler.refresh) { + try { + await handler.refresh(); + this.logger.debug(`Refreshed ${type} resource handler`); + } catch (error) { + this.logger.error( + `Failed to refresh ${type} resource handler: ${error instanceof Error ? error.message : String(error)}`, + { error: error instanceof Error ? error.message : String(error) } + ); + } + } + } + } + + async addResourceConfig(config: ValidatedInternalResourceConfig): Promise { + try { + const parsedConfig = InternalResourceConfigSchema.parse(config); + const handler = createInternalResourceHandler(parsedConfig, this.services, this.logger); + await handler.initialize(this.services); + this.handlers.set(config.type, handler); + this.config.resources.push(parsedConfig); + this.logger.info(`Added new ${config.type} resource handler`); + } catch (error) { + this.logger.error( + `Failed to add ${config.type} resource handler: ${error instanceof Error ? error.message : String(error)}`, + { error: error instanceof Error ? error.message : String(error) } + ); + throw error; + } + } + + async removeResourceHandler(type: string): Promise { + if (this.handlers.has(type)) { + this.handlers.delete(type); + this.config.resources = this.config.resources.filter((r) => r.type !== type); + this.logger.info(`Removed ${type} resource handler`); + } + } +} diff --git a/dexto/packages/core/src/resources/manager.ts b/dexto/packages/core/src/resources/manager.ts new file mode 100644 index 00000000..bf718987 --- /dev/null +++ b/dexto/packages/core/src/resources/manager.ts @@ -0,0 +1,248 @@ +import type { MCPManager } from '../mcp/manager.js'; +import type { ResourceSet, ResourceMetadata } from './types.js'; +import { InternalResourcesProvider } from './internal-provider.js'; +import type { ReadResourceResult } from '@modelcontextprotocol/sdk/types.js'; +import type { ValidatedInternalResourcesConfig } from './schemas.js'; +import type { InternalResourceServices } from './handlers/types.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import { ResourceError } from './errors.js'; +import { eventBus } from '../events/index.js'; +import type { BlobStore } from '../storage/blob/types.js'; + +export interface ResourceManagerOptions { + internalResourcesConfig: ValidatedInternalResourcesConfig; + blobStore: BlobStore; +} + +export class ResourceManager { + private readonly mcpManager: MCPManager; + private internalResourcesProvider?: InternalResourcesProvider; + private readonly blobStore: BlobStore; + private logger: IDextoLogger; + + constructor(mcpManager: MCPManager, options: ResourceManagerOptions, logger: IDextoLogger) { + this.mcpManager = mcpManager; + this.blobStore = options.blobStore; + this.logger = logger.createChild(DextoLogComponent.RESOURCE); + + const services: InternalResourceServices = { + blobStore: this.blobStore, + }; + + const config = options.internalResourcesConfig; + if (config.enabled || config.resources.length > 0) { + this.internalResourcesProvider = new InternalResourcesProvider( + config, + services, + this.logger + ); + } else { + // Always create provider to enable blob resources even if no other internal resources configured + this.internalResourcesProvider = new InternalResourcesProvider( + { enabled: true, resources: [] }, + services, + this.logger + ); + } + + // Listen for MCP resource notifications for real-time updates + this.setupNotificationListeners(); + + this.logger.debug('ResourceManager initialized'); + } + + async initialize(): Promise { + if (this.internalResourcesProvider) { + await this.internalResourcesProvider.initialize(); + } + this.logger.debug('ResourceManager initialization complete'); + } + + getBlobStore(): BlobStore { + return this.blobStore; + } + + private deriveName(uri: string): string { + const segments = uri.split(/[\\/]/).filter(Boolean); + const lastSegment = segments[segments.length - 1]; + return lastSegment ?? uri; + } + + async list(): Promise { + const resources: ResourceSet = {}; + + try { + const mcpResources = await this.mcpManager.listAllResources(); + for (const resource of mcpResources) { + const { + key, + serverName, + summary: { uri, name, description, mimeType }, + } = resource; + const metadata: ResourceMetadata = { + uri: key, + name: name ?? this.deriveName(uri), + description: description ?? `Resource from MCP server: ${serverName}`, + source: 'mcp', + serverName, + metadata: { + originalUri: uri, + serverName, + }, + }; + if (mimeType) { + metadata.mimeType = mimeType; + } + resources[key] = metadata; + } + if (mcpResources.length > 0) { + this.logger.debug( + `🗃️ Resource discovery (MCP): ${mcpResources.length} resources across ${ + new Set(mcpResources.map((r) => r.serverName)).size + } server(s)` + ); + } + } catch (error) { + this.logger.error( + `Failed to enumerate MCP resources: ${error instanceof Error ? error.message : String(error)}` + ); + } + + if (this.internalResourcesProvider) { + try { + const internalResources = await this.internalResourcesProvider.listResources(); + for (const resource of internalResources) { + resources[resource.uri] = resource; + } + if (internalResources.length > 0) { + this.logger.debug( + `🗃️ Resource discovery (internal): ${internalResources.length} resources` + ); + } + } catch (error) { + this.logger.error( + `Failed to enumerate internal resources: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + return resources; + } + + async has(uri: string): Promise { + if (uri.startsWith('mcp:')) { + return this.mcpManager.hasResource(uri); + } + // Always short-circuit blob: URIs to use blobStore directly + if (uri.startsWith('blob:')) { + try { + return await this.blobStore.exists(uri); + } catch (error) { + this.logger.warn( + `BlobService exists check failed for ${uri}: ${error instanceof Error ? error.message : String(error)}` + ); + return false; + } + } + if (!this.internalResourcesProvider) { + return false; + } + return await this.internalResourcesProvider.hasResource(uri); + } + + async read(uri: string): Promise { + this.logger.debug(`📖 Reading resource: ${uri}`); + try { + if (uri.startsWith('mcp:')) { + const result = await this.mcpManager.readResource(uri); + this.logger.debug(`✅ Successfully read MCP resource: ${uri}`); + return result; + } + + // Always short-circuit blob: URIs to use blobStore directly + if (uri.startsWith('blob:')) { + const blob = await this.blobStore.retrieve(uri, 'base64'); + return { + contents: [ + { + uri, + mimeType: blob.metadata.mimeType, + blob: blob.data as string, + }, + ], + _meta: { + size: blob.metadata.size, + createdAt: blob.metadata.createdAt, + originalName: blob.metadata.originalName, + source: blob.metadata.source, + }, + }; + } + + if (!this.internalResourcesProvider) { + throw ResourceError.providerNotInitialized('Internal', uri); + } + + const result = await this.internalResourcesProvider.readResource(uri); + this.logger.debug(`✅ Successfully read internal resource: ${uri}`); + return result; + } catch (error) { + this.logger.error( + `❌ Failed to read resource '${uri}': ${error instanceof Error ? error.message : String(error)}` + ); + throw error; + } + } + + async refresh(): Promise { + if (this.internalResourcesProvider) { + await this.internalResourcesProvider.refresh(); + } + this.logger.info('ResourceManager refreshed'); + } + + getInternalResourcesProvider(): InternalResourcesProvider | undefined { + return this.internalResourcesProvider; + } + + /** + * Set up listeners for MCP resource notifications to enable real-time updates + */ + private setupNotificationListeners(): void { + // Listen for MCP resource updates + eventBus.on('mcp:resource-updated', async (payload) => { + this.logger.debug( + `🔄 Resource updated notification: ${payload.resourceUri} from server '${payload.serverName}'` + ); + + // Emit a more specific event for components that need to refresh resource lists + eventBus.emit('resource:cache-invalidated', { + resourceUri: payload.resourceUri, + serverName: payload.serverName, + action: 'updated', + }); + }); + + // Listen for MCP server connection changes that affect resources + eventBus.on('mcp:server-connected', async (payload) => { + if (payload.success) { + this.logger.debug( + `🔄 Server connected, resources may have changed: ${payload.name}` + ); + eventBus.emit('resource:cache-invalidated', { + serverName: payload.name, + action: 'server_connected', + }); + } + }); + + eventBus.on('mcp:server-removed', async (payload) => { + this.logger.debug(`🔄 Server removed, resources invalidated: ${payload.serverName}`); + eventBus.emit('resource:cache-invalidated', { + serverName: payload.serverName, + action: 'server_removed', + }); + }); + } +} diff --git a/dexto/packages/core/src/resources/reference-parser.test.ts b/dexto/packages/core/src/resources/reference-parser.test.ts new file mode 100644 index 00000000..00cc294a --- /dev/null +++ b/dexto/packages/core/src/resources/reference-parser.test.ts @@ -0,0 +1,231 @@ +import { describe, it, expect } from 'vitest'; +import { parseResourceReferences, expandMessageReferences } from './reference-parser.js'; +import type { ResourceSet } from './types.js'; +import type { ReadResourceResult } from '@modelcontextprotocol/sdk/types.js'; + +describe('parseResourceReferences', () => { + it('should parse reference at start of message', () => { + const refs = parseResourceReferences('@myfile.txt is important'); + expect(refs).toHaveLength(1); + expect(refs[0]).toBeDefined(); + expect(refs[0]).toMatchObject({ + originalRef: '@myfile.txt', + type: 'name', + identifier: 'myfile.txt', + }); + }); + + it('should parse reference with leading whitespace', () => { + const refs = parseResourceReferences('Check @myfile.txt'); + expect(refs).toHaveLength(1); + expect(refs[0]).toBeDefined(); + expect(refs[0]).toMatchObject({ + originalRef: '@myfile.txt', + type: 'name', + identifier: 'myfile.txt', + }); + }); + + it('should parse URI reference with brackets', () => { + const refs = parseResourceReferences('Check @'); + expect(refs).toHaveLength(1); + expect(refs[0]).toBeDefined(); + expect(refs[0]).toMatchObject({ + originalRef: '@', + type: 'uri', + identifier: 'file:///path/to/file.txt', + }); + }); + + it('should parse server-scoped reference', () => { + const refs = parseResourceReferences('Check @filesystem:myfile.txt'); + expect(refs).toHaveLength(1); + expect(refs[0]).toBeDefined(); + expect(refs[0]).toMatchObject({ + originalRef: '@filesystem:myfile.txt', + type: 'server-scoped', + serverName: 'filesystem', + identifier: 'myfile.txt', + }); + }); + + it('should NOT parse @ in email addresses', () => { + const refs = parseResourceReferences('Email me at user@example.com'); + expect(refs).toHaveLength(0); + }); + + it('should parse real references but skip email addresses', () => { + const refs = parseResourceReferences('Check @myfile but email user@example.com'); + expect(refs).toHaveLength(1); + expect(refs[0]).toBeDefined(); + expect(refs[0]!.identifier).toBe('myfile'); + }); + + it('should handle multiple email addresses and references', () => { + const refs = parseResourceReferences( + 'Contact user@example.com or admin@example.com for @support.txt' + ); + expect(refs).toHaveLength(1); + expect(refs[0]).toBeDefined(); + expect(refs[0]!.identifier).toBe('support.txt'); + }); + + it('should NOT match @ without leading whitespace', () => { + const refs = parseResourceReferences('user@example.com has @file.txt'); + expect(refs).toHaveLength(1); + expect(refs[0]).toBeDefined(); + expect(refs[0]!.identifier).toBe('file.txt'); + }); + + it('should parse multiple references with whitespace', () => { + const refs = parseResourceReferences('Check @file1.txt and @file2.txt'); + expect(refs).toHaveLength(2); + expect(refs[0]).toBeDefined(); + expect(refs[1]).toBeDefined(); + expect(refs[0]!.identifier).toBe('file1.txt'); + expect(refs[1]!.identifier).toBe('file2.txt'); + }); + + it('should parse reference after newline', () => { + const refs = parseResourceReferences('First line\n@myfile.txt'); + expect(refs).toHaveLength(1); + expect(refs[0]).toBeDefined(); + expect(refs[0]!.identifier).toBe('myfile.txt'); + }); + + it('should NOT parse @ in middle of word', () => { + const refs = parseResourceReferences('test@something word @file.txt'); + expect(refs).toHaveLength(1); + expect(refs[0]).toBeDefined(); + expect(refs[0]!.identifier).toBe('file.txt'); + }); + + it('should handle @ at start with no space before', () => { + const refs = parseResourceReferences('@start then more@text and @end'); + expect(refs).toHaveLength(2); + expect(refs[0]).toBeDefined(); + expect(refs[1]).toBeDefined(); + expect(refs[0]!.identifier).toBe('start'); + expect(refs[1]!.identifier).toBe('end'); + }); +}); + +describe('expandMessageReferences', () => { + const mockResourceSet: ResourceSet = { + 'file:///test.txt': { + uri: 'file:///test.txt', + name: 'test.txt', + description: 'Test file', + source: 'internal', + }, + }; + + const mockResourceReader = async (uri: string): Promise => { + if (uri === 'file:///test.txt') { + return { + contents: [ + { + uri: 'file:///test.txt', + mimeType: 'text/plain', + text: 'File content here', + }, + ], + }; + } + throw new Error(`Resource not found: ${uri}`); + }; + + it('should expand resource reference', async () => { + const result = await expandMessageReferences( + 'Check @test.txt for info', + mockResourceSet, + mockResourceReader + ); + + expect(result.expandedReferences).toHaveLength(1); + expect(result.expandedMessage).toContain('File content here'); + expect(result.expandedMessage).toContain('test.txt'); + }); + + it('should NOT treat email addresses as references', async () => { + const result = await expandMessageReferences( + 'Email me at user@example.com', + mockResourceSet, + mockResourceReader + ); + + expect(result.expandedReferences).toHaveLength(0); + expect(result.expandedMessage).toBe('Email me at user@example.com'); + }); + + it('should handle mixed resource references and email addresses', async () => { + const result = await expandMessageReferences( + 'Check @test.txt and email user@example.com', + mockResourceSet, + mockResourceReader + ); + + expect(result.expandedReferences).toHaveLength(1); + expect(result.expandedMessage).toContain('File content here'); + expect(result.expandedMessage).toContain('user@example.com'); + }); + + it('should preserve multiple email addresses', async () => { + const result = await expandMessageReferences( + 'Contact user@example.com or admin@test.com', + mockResourceSet, + mockResourceReader + ); + + expect(result.expandedReferences).toHaveLength(0); + expect(result.expandedMessage).toBe('Contact user@example.com or admin@test.com'); + }); + + it('should preserve email addresses when resource expansion fails', async () => { + const result = await expandMessageReferences( + 'Check @nonexistent.txt and email user@example.com', + mockResourceSet, + mockResourceReader + ); + + expect(result.expandedReferences).toHaveLength(0); + expect(result.unresolvedReferences).toHaveLength(1); + expect(result.expandedMessage).toContain('user@example.com'); + }); + + it('should handle @ symbols in various contexts', async () => { + const result = await expandMessageReferences( + 'Before @test.txt middle more@text after', + mockResourceSet, + mockResourceReader + ); + + expect(result.expandedReferences).toHaveLength(1); + expect(result.expandedMessage).toContain('File content here'); + expect(result.expandedMessage).toContain('more@text'); + }); + + it('should handle message with no references', async () => { + const result = await expandMessageReferences( + 'Just email user@example.com', + mockResourceSet, + mockResourceReader + ); + + expect(result.expandedReferences).toHaveLength(0); + expect(result.unresolvedReferences).toHaveLength(0); + expect(result.expandedMessage).toBe('Just email user@example.com'); + }); + + it('should handle @ at start and in middle of text', async () => { + const result = await expandMessageReferences( + '@test.txt and contact@email.com', + mockResourceSet, + mockResourceReader + ); + + expect(result.expandedReferences).toHaveLength(1); + expect(result.expandedMessage).toContain('File content here'); + expect(result.expandedMessage).toContain('contact@email.com'); + }); +}); diff --git a/dexto/packages/core/src/resources/reference-parser.ts b/dexto/packages/core/src/resources/reference-parser.ts new file mode 100644 index 00000000..44eb2880 --- /dev/null +++ b/dexto/packages/core/src/resources/reference-parser.ts @@ -0,0 +1,297 @@ +import type { ResourceSet } from './types.js'; +import type { ReadResourceResult } from '@modelcontextprotocol/sdk/types.js'; + +// TODO: Implement Option A - pass logger as optional parameter for better observability +// when we refactor to injectable logger pattern (see CLAUDE.md note about future logger architecture) + +export interface ResourceReference { + originalRef: string; + resourceUri?: string; + type: 'name' | 'uri' | 'server-scoped'; + serverName?: string; + identifier: string; +} + +export interface ResourceExpansionResult { + expandedMessage: string; + expandedReferences: ResourceReference[]; + unresolvedReferences: ResourceReference[]; + extractedImages: Array<{ image: string; mimeType: string; name: string }>; +} + +function escapeRegExp(literal: string): string { + return literal.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); +} + +/** + * Parse resource references from a message. + * + * @ symbols are only treated as resource references if they: + * 1. Are at the start of the message, OR + * 2. Are preceded by whitespace + * + * This means email addresses like "user@example.com" are NOT treated as references. + */ +export function parseResourceReferences(message: string): ResourceReference[] { + const references: ResourceReference[] = []; + // Require whitespace before @ or start of string (^) + // This prevents matching @ in email addresses like user@example.com + const regex = + /(?:^|(?<=\s))@(?:(<[^>]+>)|([a-zA-Z0-9_-]+):([a-zA-Z0-9._/-]+)|([a-zA-Z0-9._/-]+))(?![a-zA-Z0-9@.])/g; + let match; + while ((match = regex.exec(message)) !== null) { + const [originalRef, uriWithBrackets, serverName, serverResource, simpleName] = match; + if (uriWithBrackets) { + references.push({ originalRef, type: 'uri', identifier: uriWithBrackets.slice(1, -1) }); + } else if (serverName && serverResource) { + references.push({ + originalRef, + type: 'server-scoped', + serverName, + identifier: serverResource, + }); + } else if (simpleName) { + references.push({ originalRef, type: 'name', identifier: simpleName }); + } + } + return references; +} + +export function resolveResourceReferences( + references: ResourceReference[], + availableResources: ResourceSet +): ResourceReference[] { + const resolvedRefs = references.map((ref) => ({ ...ref })); + for (const ref of resolvedRefs) { + switch (ref.type) { + case 'uri': { + // Try direct lookup first + if (availableResources[ref.identifier]) { + ref.resourceUri = ref.identifier; + } else { + // Fall back to searching by originalUri in metadata + const uriMatchUri = findResourceByOriginalUri( + availableResources, + ref.identifier + ); + if (uriMatchUri) ref.resourceUri = uriMatchUri; + } + break; + } + case 'server-scoped': { + const serverScopedUri = findResourceByServerAndName( + availableResources, + ref.serverName!, + ref.identifier + ); + if (serverScopedUri) ref.resourceUri = serverScopedUri; + break; + } + case 'name': { + const nameMatchUri = findResourceByName(availableResources, ref.identifier); + if (nameMatchUri) ref.resourceUri = nameMatchUri; + break; + } + } + } + return resolvedRefs; +} + +function findResourceByOriginalUri(resources: ResourceSet, uri: string): string | undefined { + const normalizedUri = uri.trim().toLowerCase(); + + // Look for exact match in originalUri metadata + for (const [resourceUri, resource] of Object.entries(resources)) { + const originalUri = + typeof resource.metadata?.originalUri === 'string' + ? resource.metadata.originalUri + : undefined; + if (originalUri && originalUri.toLowerCase() === normalizedUri) { + return resourceUri; + } + } + + // Fall back to partial match + for (const [resourceUri, resource] of Object.entries(resources)) { + const originalUri = + typeof resource.metadata?.originalUri === 'string' + ? resource.metadata.originalUri + : undefined; + if (originalUri && originalUri.toLowerCase().includes(normalizedUri)) { + return resourceUri; + } + } + + return undefined; +} + +function findResourceByServerAndName( + resources: ResourceSet, + serverName: string, + identifier: string +): string | undefined { + const normalizedIdentifier = identifier.trim().toLowerCase(); + const matchingResources = Object.entries(resources).filter( + ([, resource]) => resource.serverName === serverName + ); + + for (const [uri, resource] of matchingResources) { + if (!resource.name) continue; + const normalizedName = resource.name.trim().toLowerCase(); + if ( + normalizedName === normalizedIdentifier || + normalizedName.includes(normalizedIdentifier) + ) { + return uri; + } + } + + for (const [uri, resource] of matchingResources) { + const metadataUri = + typeof resource.metadata?.originalUri === 'string' + ? resource.metadata.originalUri + : undefined; + if ( + metadataUri?.toLowerCase().includes(normalizedIdentifier) || + uri.toLowerCase().includes(normalizedIdentifier) + ) { + return uri; + } + } + + return undefined; +} + +function findResourceByName(resources: ResourceSet, identifier: string): string | undefined { + const normalizedIdentifier = identifier.trim().toLowerCase(); + + for (const [uri, resource] of Object.entries(resources)) { + if (!resource.name) continue; + const normalizedName = resource.name.trim().toLowerCase(); + if ( + normalizedName === normalizedIdentifier || + normalizedName.includes(normalizedIdentifier) + ) { + return uri; + } + } + + for (const [uri, resource] of Object.entries(resources)) { + const originalUri = + typeof resource.metadata?.originalUri === 'string' + ? resource.metadata.originalUri + : undefined; + if ( + originalUri?.toLowerCase().includes(normalizedIdentifier) || + uri.toLowerCase().includes(normalizedIdentifier) + ) { + return uri; + } + } + + return undefined; +} + +export function formatResourceContent( + resourceUri: string, + resourceName: string, + content: ReadResourceResult +): string { + const contentParts: string[] = []; + contentParts.push(`\n--- Content from resource: ${resourceName} (${resourceUri}) ---`); + for (const item of content.contents) { + if ('text' in item && item.text && typeof item.text === 'string') { + contentParts.push(item.text); + } else if ('blob' in item && item.blob) { + const blobSize = typeof item.blob === 'string' ? item.blob.length : 'unknown'; + contentParts.push(`[Binary content: ${item.mimeType || 'unknown'}, ${blobSize} bytes]`); + } + } + contentParts.push('--- End of resource content ---\n'); + return contentParts.join('\n'); +} + +export async function expandMessageReferences( + message: string, + availableResources: ResourceSet, + resourceReader: (uri: string) => Promise +): Promise { + // Note: Logging removed to keep this function browser-safe + // TODO: Add logger as optional parameter when implementing Option A + + const parsedRefs = parseResourceReferences(message); + if (parsedRefs.length === 0) { + return { + expandedMessage: message, + expandedReferences: [], + unresolvedReferences: [], + extractedImages: [], + }; + } + + const resolvedRefs = resolveResourceReferences(parsedRefs, availableResources); + const expandedReferences = resolvedRefs.filter((ref) => ref.resourceUri); + const unresolvedReferences = resolvedRefs.filter((ref) => !ref.resourceUri); + + let expandedMessage = message; + const failedRefs: ResourceReference[] = []; + const extractedImages: Array<{ image: string; mimeType: string; name: string }> = []; + + for (const ref of expandedReferences) { + try { + const content = await resourceReader(ref.resourceUri!); + const resource = availableResources[ref.resourceUri!]; + + // Check if this is an image resource + let isImageResource = false; + for (const item of content.contents) { + if ( + 'blob' in item && + item.blob && + item.mimeType && + item.mimeType.startsWith('image/') && + typeof item.blob === 'string' + ) { + extractedImages.push({ + image: item.blob, + mimeType: item.mimeType, + name: resource?.name || ref.identifier, + }); + isImageResource = true; + break; + } + } + + if (isImageResource) { + // Remove the reference from the message for images + const pattern = new RegExp(escapeRegExp(ref.originalRef), 'g'); + expandedMessage = expandedMessage + .replace(pattern, ' ') + .replace(/\s{2,}/g, ' ') + .trim(); + } else { + // For non-image resources, expand them inline as before + const formattedContent = formatResourceContent( + ref.resourceUri!, + resource?.name || ref.identifier, + content + ); + const pattern = new RegExp(escapeRegExp(ref.originalRef), 'g'); + expandedMessage = expandedMessage.replace(pattern, formattedContent); + } + } catch (_error) { + failedRefs.push(ref); + } + } + + const failedRefSet = new Set(failedRefs); + const finalExpandedReferences = expandedReferences.filter((ref) => !failedRefSet.has(ref)); + unresolvedReferences.push(...failedRefs); + + return { + expandedMessage, + expandedReferences: finalExpandedReferences, + unresolvedReferences, + extractedImages, + }; +} diff --git a/dexto/packages/core/src/resources/schemas.ts b/dexto/packages/core/src/resources/schemas.ts new file mode 100644 index 00000000..1485254e --- /dev/null +++ b/dexto/packages/core/src/resources/schemas.ts @@ -0,0 +1,154 @@ +import { z } from 'zod'; + +/** + * Schema for validating file extensions (must start with a dot) + */ +const FileExtensionSchema = z + .string() + .regex( + /^\.[A-Za-z0-9][A-Za-z0-9._-]*$/, + 'Extensions must start with a dot and may include alphanumerics, dot, underscore, or hyphen (e.g., .d.ts, .tar.gz)' + ) + .describe('File extension pattern starting with a dot; supports multi-part extensions'); + +/** + * Schema for filesystem resource configuration + */ +const FileSystemResourceSchema = z + .object({ + type: z.literal('filesystem'), + paths: z + .array(z.string()) + .min(1) + .describe('File paths or directories to expose as resources (at least one required)'), + maxDepth: z + .number() + .min(1) + .max(10) + .default(3) + .describe('Maximum directory depth to traverse (default: 3)'), + maxFiles: z + .number() + .min(1) + .max(10000) + .default(1000) + .describe('Maximum number of files to include (default: 1000)'), + includeHidden: z + .boolean() + .default(false) + .describe('Include hidden files and directories (default: false)'), + includeExtensions: z + .array(FileExtensionSchema) + .default([ + '.txt', + '.md', + '.js', + '.ts', + '.json', + '.html', + '.css', + '.py', + '.yaml', + '.yml', + '.xml', + '.jsx', + '.tsx', + '.vue', + '.php', + '.rb', + '.go', + '.rs', + '.java', + '.kt', + '.swift', + '.sql', + '.sh', + '.bash', + '.zsh', + ]) + .describe('File extensions to include (default: common text files)'), + }) + .strict(); + +/** + * Validated filesystem resource configuration type + */ +export type ValidatedFileSystemResourceConfig = z.output; + +/** + * Schema for blob storage resource configuration + * + * NOTE: This only enables the blob resource provider. + * Actual blob storage settings (size limits, backend, cleanup) are configured + * in the 'storage.blob' section of the agent config. + */ +const BlobResourceSchema = z + .object({ + type: z.literal('blob').describe('Enable blob storage resource provider'), + }) + .strict() + .describe( + 'Blob resource provider configuration - actual storage settings are in storage.blob section' + ); + +/** + * Validated blob resource configuration type + */ +export type ValidatedBlobResourceConfig = z.output; + +/** + * Union schema for all internal resource types (composed from individual schemas) + */ +export const InternalResourceConfigSchema = z.discriminatedUnion('type', [ + FileSystemResourceSchema, + BlobResourceSchema, +]); + +/** + * Validated union type for all internal resource configurations + */ +export type ValidatedInternalResourceConfig = z.output; + +/** + * Schema for internal resources configuration with smart auto-enable logic + * + * Design principles: + * - Clean input format: just specify resources array or object + * - Auto-enable when resources are specified + * - Backward compatibility with explicit enabled field + * - Empty/omitted = disabled + */ +export const InternalResourcesSchema = z + .union([ + z.array(InternalResourceConfigSchema), // array-only form + z + .object({ + enabled: z + .boolean() + .optional() + .describe('Explicit toggle; auto-enabled when resources are non-empty'), + resources: z + .array(InternalResourceConfigSchema) + .default([]) + .describe('List of internal resource configurations'), + }) + .strict(), + ]) + .default([]) + .describe( + 'Internal resource configuration. Can be an array of resources (auto-enabled) or object with enabled field' + ) + .transform((input) => { + if (Array.isArray(input)) { + return { enabled: input.length > 0, resources: input }; + } + const enabled = input.enabled !== undefined ? input.enabled : input.resources.length > 0; + return { enabled, resources: input.resources }; + }); + +export type InternalResourcesConfig = z.input; +export type ValidatedInternalResourcesConfig = z.output; + +export function isInternalResourcesEnabled(config: ValidatedInternalResourcesConfig): boolean { + return config.enabled && config.resources.length > 0; +} diff --git a/dexto/packages/core/src/resources/types.ts b/dexto/packages/core/src/resources/types.ts new file mode 100644 index 00000000..cddf3261 --- /dev/null +++ b/dexto/packages/core/src/resources/types.ts @@ -0,0 +1,64 @@ +/** + * Core resource types and interfaces for the ResourceManager + */ + +import type { ReadResourceResult } from '@modelcontextprotocol/sdk/types.js'; + +/** + * Supported resource sources + */ +export type ResourceSource = 'mcp' | 'internal'; + +/** + * Resource metadata information + */ +export interface ResourceMetadata { + /** Unique URI/identifier for the resource */ + uri: string; + /** Human-readable name for the resource */ + name?: string; + /** Description of what this resource contains */ + description?: string; + /** MIME type of the resource content */ + mimeType?: string; + /** Source system that provides this resource */ + source: ResourceSource; + /** Original server/provider name (for MCP resources) */ + serverName?: string; + /** Size of the resource in bytes (if known) */ + size?: number; + /** Last modified timestamp (ISO string or Date) */ + lastModified?: string | Date; + /** Additional metadata specific to the resource type */ + metadata?: Record; +} + +/** + * Resource provider interface - implemented by sources that can provide resources + */ +export interface ResourceProvider { + /** + * List all available resources from this provider + */ + listResources(): Promise; + + /** + * Read the content of a specific resource + */ + readResource(uri: string): Promise; + + /** + * Check if a resource exists + */ + hasResource(uri: string): Promise; + + /** + * Get the source type of this provider + */ + getSource(): ResourceSource; +} + +/** + * Resource set mapping URIs to resource metadata + */ +export type ResourceSet = Record; diff --git a/dexto/packages/core/src/search/index.ts b/dexto/packages/core/src/search/index.ts new file mode 100644 index 00000000..2c39f5b1 --- /dev/null +++ b/dexto/packages/core/src/search/index.ts @@ -0,0 +1,8 @@ +export { SearchService } from './search-service.js'; +export type { + SearchOptions, + SearchResult, + SessionSearchResult, + SearchResponse, + SessionSearchResponse, +} from './types.js'; diff --git a/dexto/packages/core/src/search/search-service.ts b/dexto/packages/core/src/search/search-service.ts new file mode 100644 index 00000000..0a4483b0 --- /dev/null +++ b/dexto/packages/core/src/search/search-service.ts @@ -0,0 +1,317 @@ +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import type { Database } from '@core/storage/types.js'; +import type { InternalMessage } from '../context/types.js'; +import type { + SearchOptions, + SearchResult, + SessionSearchResult, + SearchResponse, + SessionSearchResponse, +} from './types.js'; + +/** + * Service for searching through conversation history + * TODO: remove duplicate stuff related to session manager instead of directly using DB + */ +export class SearchService { + private logger: IDextoLogger; + + constructor( + private database: Database, + logger: IDextoLogger + ) { + this.logger = logger.createChild(DextoLogComponent.SESSION); + } + + /** + * Search for messages across all sessions or within a specific session + */ + async searchMessages(query: string, options: SearchOptions = {}): Promise { + const { sessionId, role, limit = 20, offset = 0 } = options; + + if (!query.trim()) { + return { + results: [], + total: 0, + hasMore: false, + query, + options, + }; + } + + try { + this.logger.debug(`Searching messages for query: "${query}"`, { + sessionId, + role, + limit, + offset, + }); + + const allResults: SearchResult[] = []; + const sessionIds = sessionId ? [sessionId] : await this.getSessionIds(); + + // Search through each session + for (const sId of sessionIds) { + const sessionResults = await this.searchInSession(query, sId, role); + allResults.push(...sessionResults); + } + + // Sort results by relevance (exact matches first, then by session activity) + const sortedResults = this.sortResults(allResults, query); + + // Apply pagination + const total = sortedResults.length; + const paginatedResults = sortedResults.slice(offset, offset + limit); + const hasMore = offset + limit < total; + + return { + results: paginatedResults, + total, + hasMore, + query, + options, + }; + } catch (error) { + this.logger.error( + `Error searching messages: ${error instanceof Error ? error.message : String(error)}` + ); + return { + results: [], + total: 0, + hasMore: false, + query, + options, + }; + } + } + + /** + * Search for sessions that contain the query + */ + async searchSessions(query: string): Promise { + if (!query.trim()) { + return { + results: [], + total: 0, + hasMore: false, + query, + }; + } + + try { + this.logger.debug(`Searching sessions for query: "${query}"`); + + const sessionResults: SessionSearchResult[] = []; + const sessionIds = await this.getSessionIds(); + + // Search through each session and collect session-level results + for (const sessionId of sessionIds) { + const messageResults = await this.searchInSession(query, sessionId); + + if (messageResults.length > 0) { + const sessionMetadata = await this.getSessionMetadata(sessionId); + if (sessionMetadata) { + const firstMatch = messageResults[0]; + if (firstMatch) { + sessionResults.push({ + sessionId, + matchCount: messageResults.length, + firstMatch, + metadata: sessionMetadata, + }); + } + } + } + } + + // Sort sessions by match count and recent activity + const sortedResults = sessionResults.sort((a, b) => { + // First by match count + if (a.matchCount !== b.matchCount) { + return b.matchCount - a.matchCount; + } + // Then by recent activity + return b.metadata.lastActivity - a.metadata.lastActivity; + }); + + return { + results: sortedResults, + total: sortedResults.length, + hasMore: false, + query, + }; + } catch (error) { + this.logger.error( + `Error searching sessions: ${error instanceof Error ? error.message : String(error)}` + ); + return { + results: [], + total: 0, + hasMore: false, + query, + }; + } + } + + /** + * Search for messages within a specific session + */ + private async searchInSession( + query: string, + sessionId: string, + role?: string + ): Promise { + const messagesKey = `messages:${sessionId}`; + // TODO: Consider implementing pagination or using database search capabilities + const MAX_MESSAGES_PER_SEARCH = 10000; // Configurable limit + const messages = await this.database.getRange( + messagesKey, + 0, + MAX_MESSAGES_PER_SEARCH + ); + + const results: SearchResult[] = []; + const lowerQuery = query.toLowerCase(); + + for (let i = 0; i < messages.length; i++) { + const message = messages[i]; + if (!message) { + continue; + } + + // Skip if role filter doesn't match + if (role && message.role !== role) { + continue; + } + + // Extract searchable text from message content + const searchableText = this.extractSearchableText(message); + if (!searchableText) { + continue; + } + + const lowerText = searchableText.toLowerCase(); + const matchIndex = lowerText.indexOf(lowerQuery); + + if (matchIndex !== -1) { + const matchedText = searchableText.substring(matchIndex, matchIndex + query.length); + const context = this.getContext(searchableText, matchIndex, query.length); + + results.push({ + sessionId, + message, + matchedText, + context, + messageIndex: i, + }); + } + } + + return results; + } + + /** + * Extract searchable text from a message + */ + private extractSearchableText(message: InternalMessage): string | null { + if (!message.content) { + return null; + } + + if (typeof message.content === 'string') { + return message.content; + } + + if (Array.isArray(message.content)) { + return message.content + .filter((part) => part.type === 'text') + .map((part) => ('text' in part ? part.text : '')) + .join(' '); + } + + return null; + } + + /** + * Get context around a match for preview + */ + private getContext( + text: string, + matchIndex: number, + matchLength: number, + contextLength = 50 + ): string { + const start = Math.max(0, matchIndex - contextLength); + const end = Math.min(text.length, matchIndex + matchLength + contextLength); + + let context = text.substring(start, end); + + // Add ellipsis if we truncated + if (start > 0) { + context = '...' + context; + } + if (end < text.length) { + context = context + '...'; + } + + return context; + } + + /** + * Sort search results by relevance + */ + private sortResults(results: SearchResult[], query: string): SearchResult[] { + const lowerQuery = query.toLowerCase(); + + return results.sort((a, b) => { + const aText = this.extractSearchableText(a.message)?.toLowerCase() || ''; + const bText = this.extractSearchableText(b.message)?.toLowerCase() || ''; + + // Exact word matches score higher + const aExactMatch = + aText.includes(` ${lowerQuery} `) || + aText.startsWith(lowerQuery) || + aText.endsWith(lowerQuery); + const bExactMatch = + bText.includes(` ${lowerQuery} `) || + bText.startsWith(lowerQuery) || + bText.endsWith(lowerQuery); + + if (aExactMatch && !bExactMatch) return -1; + if (!aExactMatch && bExactMatch) return 1; + + // Then by message index (more recent messages first) + return b.messageIndex - a.messageIndex; + }); + } + + /** + * Get all session IDs + */ + private async getSessionIds(): Promise { + const sessionKeys = await this.database.list('session:'); + return sessionKeys.map((key: string) => key.replace('session:', '')); + } + + /** + * Get session metadata + */ + private async getSessionMetadata(sessionId: string): Promise<{ + createdAt: number; + lastActivity: number; + messageCount: number; + }> { + const sessionKey = `session:${sessionId}`; + const sessionData = await this.database.get<{ + createdAt: number; + lastActivity: number; + messageCount: number; + }>(sessionKey); + + if (!sessionData) { + throw new Error(`Session metadata not found: ${sessionId}`); + } + return sessionData; + } +} diff --git a/dexto/packages/core/src/search/types.ts b/dexto/packages/core/src/search/types.ts new file mode 100644 index 00000000..15cd9fd6 --- /dev/null +++ b/dexto/packages/core/src/search/types.ts @@ -0,0 +1,79 @@ +import type { InternalMessage } from '../context/types.js'; + +/** + * Options for searching messages + */ +export interface SearchOptions { + /** Limit search to a specific session */ + sessionId?: string; + /** Filter by message role */ + role?: 'user' | 'assistant' | 'system' | 'tool'; + /** Maximum number of results to return */ + limit?: number; + /** Offset for pagination */ + offset?: number; +} + +/** + * Result of a message search + */ +export interface SearchResult { + /** Session ID where the message was found */ + sessionId: string; + /** The message that matched the search */ + message: InternalMessage; + /** The specific text that matched the search query */ + matchedText: string; + /** Context around the match for preview */ + context: string; + /** Index of the message within the session */ + messageIndex: number; +} + +/** + * Result of a session search + */ +export interface SessionSearchResult { + /** Session ID */ + sessionId: string; + /** Number of messages that matched in this session */ + matchCount: number; + /** Preview of the first matching message */ + firstMatch: SearchResult; + /** Session metadata */ + metadata: { + createdAt: number; + lastActivity: number; + messageCount: number; + }; +} + +/** + * Response format for search API + */ +export interface SearchResponse { + /** Array of search results */ + results: SearchResult[]; + /** Total number of results available */ + total: number; + /** Whether there are more results beyond the current page */ + hasMore: boolean; + /** Query that was searched */ + query: string; + /** Options used for the search */ + options: SearchOptions; +} + +/** + * Response format for session search API + */ +export interface SessionSearchResponse { + /** Array of session search results */ + results: SessionSearchResult[]; + /** Total number of sessions with matches */ + total: number; + /** Whether there are more results beyond the current page */ + hasMore: boolean; + /** Query that was searched */ + query: string; +} diff --git a/dexto/packages/core/src/session/chat-session.test.ts b/dexto/packages/core/src/session/chat-session.test.ts new file mode 100644 index 00000000..3d47c287 --- /dev/null +++ b/dexto/packages/core/src/session/chat-session.test.ts @@ -0,0 +1,458 @@ +import { describe, test, expect, vi, beforeEach, afterEach } from 'vitest'; +import { ChatSession } from './chat-session.js'; +import { type ValidatedLLMConfig } from '@core/llm/schemas.js'; +import { LLMConfigSchema } from '@core/llm/schemas.js'; + +// Mock all dependencies +vi.mock('./history/factory.js', () => ({ + createDatabaseHistoryProvider: vi.fn(), +})); +vi.mock('../llm/services/factory.js', () => ({ + createLLMService: vi.fn(), + createVercelModel: vi.fn(), +})); +vi.mock('../context/compaction/index.js', () => ({ + createCompactionStrategy: vi.fn(), + compactionRegistry: { + register: vi.fn(), + get: vi.fn(), + has: vi.fn(), + getTypes: vi.fn(), + getAll: vi.fn(), + clear: vi.fn(), + }, +})); +vi.mock('../llm/registry.js', async (importOriginal) => { + const actual = (await importOriginal()) as typeof import('../llm/registry.js'); + return { + ...actual, + getEffectiveMaxInputTokens: vi.fn(), + }; +}); +vi.mock('../logger/index.js', () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + silly: vi.fn(), + }, +})); + +import { createDatabaseHistoryProvider } from './history/factory.js'; +import { createLLMService, createVercelModel } from '../llm/services/factory.js'; +import { createCompactionStrategy } from '../context/compaction/index.js'; +import { getEffectiveMaxInputTokens } from '../llm/registry.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; + +const mockCreateDatabaseHistoryProvider = vi.mocked(createDatabaseHistoryProvider); +const mockCreateLLMService = vi.mocked(createLLMService); +const mockCreateVercelModel = vi.mocked(createVercelModel); +const mockCreateCompactionStrategy = vi.mocked(createCompactionStrategy); +const mockGetEffectiveMaxInputTokens = vi.mocked(getEffectiveMaxInputTokens); + +describe('ChatSession', () => { + let chatSession: ChatSession; + let mockServices: any; + let mockHistoryProvider: any; + let mockLLMService: any; + let mockCache: any; + let mockDatabase: any; + let mockBlobStore: any; + let mockContextManager: any; + const mockLogger = createMockLogger(); + + const sessionId = 'test-session-123'; + const mockLLMConfig = LLMConfigSchema.parse({ + provider: 'openai', + model: 'gpt-5', + apiKey: 'test-key', + maxIterations: 50, + maxInputTokens: 128000, + }); + + beforeEach(() => { + vi.resetAllMocks(); + + // Mock history provider + mockHistoryProvider = { + addMessage: vi.fn().mockResolvedValue(undefined), + getMessages: vi.fn().mockResolvedValue([]), + clearHistory: vi.fn().mockResolvedValue(undefined), + getMessageCount: vi.fn().mockResolvedValue(0), + }; + + // Mock LLM service + mockContextManager = { + resetConversation: vi.fn().mockResolvedValue(undefined), + }; + mockLLMService = { + stream: vi.fn().mockResolvedValue('Mock response'), + switchLLM: vi.fn().mockResolvedValue(undefined), + getContextManager: vi.fn().mockReturnValue(mockContextManager), + eventBus: { + emit: vi.fn(), + on: vi.fn(), + off: vi.fn(), + }, + }; + + // Mock storage manager with proper getter structure + mockCache = { + get: vi.fn().mockResolvedValue(null), + set: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(true), + list: vi.fn().mockResolvedValue([]), + clear: vi.fn().mockResolvedValue(undefined), + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + isConnected: vi.fn().mockReturnValue(true), + getBackendType: vi.fn().mockReturnValue('memory'), + }; + + mockDatabase = { + get: vi.fn().mockResolvedValue(null), + set: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(true), + list: vi.fn().mockResolvedValue([]), + clear: vi.fn().mockResolvedValue(undefined), + append: vi.fn().mockResolvedValue(undefined), + getRange: vi.fn().mockResolvedValue([]), + getLength: vi.fn().mockResolvedValue(0), + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + isConnected: vi.fn().mockReturnValue(true), + getBackendType: vi.fn().mockReturnValue('memory'), + }; + + mockBlobStore = { + store: vi.fn().mockResolvedValue({ id: 'test', uri: 'blob:test' }), + retrieve: vi.fn().mockResolvedValue({ data: '', metadata: {} }), + exists: vi.fn().mockResolvedValue(false), + delete: vi.fn().mockResolvedValue(undefined), + cleanup: vi.fn().mockResolvedValue(0), + getStats: vi.fn().mockResolvedValue({ count: 0, totalSize: 0, backendType: 'local' }), + listBlobs: vi.fn().mockResolvedValue([]), + getStoragePath: vi.fn().mockReturnValue(undefined), + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + isConnected: vi.fn().mockReturnValue(true), + getStoreType: vi.fn().mockReturnValue('local'), + }; + + const mockStorageManager = { + getCache: vi.fn().mockReturnValue(mockCache), + getDatabase: vi.fn().mockReturnValue(mockDatabase), + getBlobStore: vi.fn().mockReturnValue(mockBlobStore), + disconnect: vi.fn().mockResolvedValue(undefined), + }; + + // Mock services + mockServices = { + stateManager: { + getLLMConfig: vi.fn().mockReturnValue(mockLLMConfig), + getRuntimeConfig: vi.fn().mockReturnValue({ + llm: mockLLMConfig, + compression: { type: 'noop', enabled: true }, + }), + updateLLM: vi.fn().mockReturnValue({ isValid: true, errors: [], warnings: [] }), + }, + systemPromptManager: { + getSystemPrompt: vi.fn().mockReturnValue('System prompt'), + }, + mcpManager: { + getAllTools: vi.fn().mockResolvedValue({}), + }, + agentEventBus: { + emit: vi.fn(), + on: vi.fn(), + off: vi.fn(), + }, + storageManager: mockStorageManager, + resourceManager: { + getBlobStore: vi.fn(), + readResource: vi.fn(), + listResources: vi.fn(), + }, + toolManager: { + getAllTools: vi.fn().mockReturnValue([]), + }, + pluginManager: { + executePlugins: vi.fn().mockImplementation(async (_point, payload) => payload), + cleanup: vi.fn(), + }, + sessionManager: { + // Add sessionManager mock if needed + }, + }; + + // Set up factory mocks + mockCreateDatabaseHistoryProvider.mockReturnValue(mockHistoryProvider); + mockCreateLLMService.mockReturnValue(mockLLMService); + mockCreateVercelModel.mockReturnValue('mock-model' as any); + mockCreateCompactionStrategy.mockResolvedValue(null); // No compaction for tests + mockGetEffectiveMaxInputTokens.mockReturnValue(128000); + + // Create ChatSession instance + chatSession = new ChatSession(mockServices, sessionId, mockLogger); + }); + + afterEach(() => { + // Clean up any resources + if (chatSession) { + chatSession.dispose(); + } + }); + + describe('Session Identity and Lifecycle', () => { + test('should maintain session identity throughout lifecycle', () => { + expect(chatSession.id).toBe(sessionId); + expect(chatSession.eventBus).toBeDefined(); + }); + + test('should initialize with unified storage system', async () => { + await chatSession.init(); + + // Verify createDatabaseHistoryProvider is called with the database backend, sessionId, and logger + expect(mockCreateDatabaseHistoryProvider).toHaveBeenCalledWith( + mockDatabase, + sessionId, + expect.any(Object) // Logger object + ); + }); + + test('should properly dispose resources to prevent memory leaks', () => { + const eventSpy = vi.spyOn(chatSession.eventBus, 'off'); + + chatSession.dispose(); + chatSession.dispose(); // Should not throw on multiple calls + + expect(eventSpy).toHaveBeenCalled(); + }); + }); + + describe('Event System Integration', () => { + test('should forward all session events to agent bus with session context', async () => { + await chatSession.init(); + + // Emit a session event + chatSession.eventBus.emit('llm:thinking'); + + expect(mockServices.agentEventBus.emit).toHaveBeenCalledWith( + 'llm:thinking', + expect.objectContaining({ + sessionId, + }) + ); + }); + + test('should handle events with no payload by adding session context', async () => { + await chatSession.init(); + + // Emit event without payload (using llm:thinking as example) + chatSession.eventBus.emit('llm:thinking'); + + expect(mockServices.agentEventBus.emit).toHaveBeenCalledWith('llm:thinking', { + sessionId, + }); + }); + + test('should emit dexto:conversationReset event when conversation is reset', async () => { + await chatSession.init(); + + await chatSession.reset(); + + // Should reset conversation via ContextManager + expect(mockContextManager.resetConversation).toHaveBeenCalled(); + + // Should emit dexto:conversationReset event with session context + expect(mockServices.agentEventBus.emit).toHaveBeenCalledWith('session:reset', { + sessionId, + }); + }); + }); + + describe('LLM Configuration Management', () => { + beforeEach(async () => { + await chatSession.init(); + }); + + test('should create new LLM service when configuration changes', async () => { + const newConfig: ValidatedLLMConfig = { + ...mockLLMConfig, + maxInputTokens: 256000, // Change maxInputTokens + }; + + // Clear previous calls + mockCreateLLMService.mockClear(); + + await chatSession.switchLLM(newConfig); + + // Should create a new LLM service with updated config + expect(mockCreateLLMService).toHaveBeenCalledWith( + newConfig, + mockServices.toolManager, + mockServices.systemPromptManager, + mockHistoryProvider, + chatSession.eventBus, + sessionId, + mockServices.resourceManager, + mockLogger, + null, // compaction strategy + undefined // compaction config + ); + }); + + test('should create new LLM service during LLM switch', async () => { + const newConfig: ValidatedLLMConfig = { + ...mockLLMConfig, + provider: 'anthropic', + model: 'claude-4-opus-20250514', + }; + + // Clear previous calls to createLLMService + mockCreateLLMService.mockClear(); + + await chatSession.switchLLM(newConfig); + + // Should create a new LLM service with the new config + expect(mockCreateLLMService).toHaveBeenCalledWith( + newConfig, + mockServices.toolManager, + mockServices.systemPromptManager, + mockHistoryProvider, + chatSession.eventBus, + sessionId, + mockServices.resourceManager, + mockLogger, + null, // compaction strategy + undefined // compaction config + ); + }); + + test('should emit LLM switched event with correct metadata', async () => { + const newConfig: ValidatedLLMConfig = { + ...mockLLMConfig, + provider: 'anthropic', + model: 'claude-4-opus-20250514', + }; + + const eventSpy = vi.spyOn(chatSession.eventBus, 'emit'); + + await chatSession.switchLLM(newConfig); + + expect(eventSpy).toHaveBeenCalledWith( + 'llm:switched', + expect.objectContaining({ + newConfig, + historyRetained: true, + }) + ); + }); + }); + + describe('Error Handling and Resilience', () => { + test('should handle storage initialization failures gracefully', async () => { + mockCreateDatabaseHistoryProvider.mockImplementation(() => { + throw new Error('Storage initialization failed'); + }); + + // The init method should throw the error since it doesn't catch it + await expect(chatSession.init()).rejects.toThrow('Storage initialization failed'); + }); + + test('should handle LLM service creation failures', async () => { + mockCreateLLMService.mockImplementation(() => { + throw new Error('LLM service creation failed'); + }); + + await expect(chatSession.init()).rejects.toThrow('LLM service creation failed'); + }); + + test('should handle LLM switch failures and propagate errors', async () => { + await chatSession.init(); + + const newConfig: ValidatedLLMConfig = { + ...mockLLMConfig, + provider: 'invalid-provider' as any, + }; + + mockCreateLLMService.mockImplementation(() => { + throw new Error('Invalid provider'); + }); + + await expect(chatSession.switchLLM(newConfig)).rejects.toThrow('Invalid provider'); + }); + + test('should handle conversation errors from LLM service', async () => { + await chatSession.init(); + + mockLLMService.stream.mockRejectedValue(new Error('LLM service error')); + + await expect(chatSession.stream('test message')).rejects.toThrow('LLM service error'); + }); + }); + + describe('Service Integration Points', () => { + beforeEach(async () => { + await chatSession.init(); + }); + + test('should delegate conversation operations to LLM service', async () => { + const userMessage = 'Hello, world!'; + const expectedResponse = 'Hello! How can I help you?'; + + mockLLMService.stream.mockResolvedValue({ text: expectedResponse }); + + const response = await chatSession.stream(userMessage); + + expect(response).toEqual({ text: expectedResponse }); + expect(mockLLMService.stream).toHaveBeenCalledWith( + [{ type: 'text', text: userMessage }], + expect.objectContaining({ signal: expect.any(AbortSignal) }) + ); + }); + + test('should delegate history operations to history provider', async () => { + const mockHistory = [ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ]; + + mockHistoryProvider.getHistory = vi.fn().mockResolvedValue(mockHistory); + + await chatSession.init(); + const history = await chatSession.getHistory(); + + expect(history).toEqual(mockHistory); + expect(mockHistoryProvider.getHistory).toHaveBeenCalled(); + }); + }); + + describe('Session Isolation', () => { + test('should create session-specific services with proper isolation', async () => { + await chatSession.init(); + + // Verify session-specific LLM service creation with new signature + expect(mockCreateLLMService).toHaveBeenCalledWith( + mockLLMConfig, + mockServices.toolManager, + mockServices.systemPromptManager, + mockHistoryProvider, + chatSession.eventBus, // Session-specific event bus + sessionId, + mockServices.resourceManager, // ResourceManager parameter + mockLogger, // Logger parameter + null, // compaction strategy + undefined // compaction config + ); + + // Verify session-specific history provider creation + expect(mockCreateDatabaseHistoryProvider).toHaveBeenCalledWith( + mockDatabase, + sessionId, + expect.any(Object) // Logger object + ); + }); + }); +}); diff --git a/dexto/packages/core/src/session/chat-session.ts b/dexto/packages/core/src/session/chat-session.ts new file mode 100644 index 00000000..f37388ef --- /dev/null +++ b/dexto/packages/core/src/session/chat-session.ts @@ -0,0 +1,793 @@ +import { createDatabaseHistoryProvider } from './history/factory.js'; +import { createLLMService, createVercelModel } from '../llm/services/factory.js'; +import { createCompactionStrategy } from '../context/compaction/index.js'; +import type { ContextManager } from '@core/context/index.js'; +import type { IConversationHistoryProvider } from './history/types.js'; +import type { VercelLLMService } from '../llm/services/vercel.js'; +import type { SystemPromptManager } from '../systemPrompt/manager.js'; +import type { ToolManager } from '../tools/tool-manager.js'; +import type { ValidatedLLMConfig } from '@core/llm/schemas.js'; +import type { AgentStateManager } from '../agent/state-manager.js'; +import type { StorageManager } from '../storage/index.js'; +import type { PluginManager } from '../plugins/manager.js'; +import type { MCPManager } from '../mcp/manager.js'; +import type { BeforeLLMRequestPayload, BeforeResponsePayload } from '../plugins/types.js'; +import { + SessionEventBus, + AgentEventBus, + SessionEventNames, + SessionEventName, + SessionEventMap, +} from '../events/index.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import { DextoRuntimeError, ErrorScope, ErrorType } from '../errors/index.js'; +import { PluginErrorCode } from '../plugins/error-codes.js'; +import type { InternalMessage, ContentPart } from '../context/types.js'; +import type { UserMessageInput } from './message-queue.js'; +import type { ContentInput } from '../agent/types.js'; +import { getModelPricing, calculateCost } from '../llm/registry.js'; + +/** + * Represents an isolated conversation session within a Dexto agent. + * + * ChatSession provides session-level isolation for conversations, allowing multiple + * independent chat contexts to exist within a single DextoAgent instance. Each session + * maintains its own conversation history, message management, and event handling. + * + * ## Architecture + * + * The ChatSession acts as a lightweight wrapper around core Dexto services, providing + * session-specific instances of: + * - **ContextManager**: Handles conversation history and message formatting + * - **LLMService**: Manages AI model interactions and tool execution + * - **TypedEventEmitter**: Provides session-scoped event handling + * + * ## Event Handling + * + * Each session has its own event bus that emits standard Dexto events: + * - `llm:*` events (thinking, toolCall, response, etc.) + * + * Session events are forwarded to the global agent event bus with session prefixes. + * + * ## Usage Example + * + * ```typescript + * // Create a new session + * const session = agent.createSession('user-123'); + * + * // Listen for session events + * session.eventBus.on('llm:response', (payload) => { + * console.log('Session response:', payload.content); + * }); + * + * // Run a conversation turn + * const response = await session.run('Hello, how are you?'); + * + * // Reset session history + * await session.reset(); + * ``` + * + * @see {@link SessionManager} for session lifecycle management + * @see {@link ContextManager} for conversation history management + * @see {@link VercelLLMService} for AI model interaction + */ +export class ChatSession { + /** + * Session-scoped event emitter for handling conversation events. + * + * This is a session-local SessionEventBus instance that forwards events + * to the global agent event bus. + * + * Events emitted include: + * - `llm:thinking` - AI model is processing + * - `llm:tool-call` - Tool execution requested + * - `llm:response` - Final response generated + */ + public readonly eventBus: SessionEventBus; + + /** + * History provider that persists conversation messages. + * Shared across LLM switches to maintain conversation continuity. + */ + private historyProvider!: IConversationHistoryProvider; + + /** + * Handles AI model interactions, tool execution, and response generation for this session. + * + * Each session has its own LLMService instance that uses the session's + * ContextManager and event bus. + */ + private llmService!: VercelLLMService; + + /** + * Map of event forwarder functions for cleanup. + * Stores the bound functions so they can be removed from the event bus. + */ + private forwarders: Map void> = new Map(); + + /** + * Token accumulator listener for cleanup. + */ + private tokenAccumulatorListener: ((payload: SessionEventMap['llm:response']) => void) | null = + null; + + /** + * AbortController for the currently running turn, if any. + * Calling cancel() aborts the in-flight LLM request and tool execution checks. + */ + private currentRunController: AbortController | null = null; + + private logger: IDextoLogger; + + /** + * Creates a new ChatSession instance. + * + * Each session creates its own isolated services: + * - ConversationHistoryProvider (with session-specific storage, shared across LLM switches) + * - LLMService (creates its own properly-typed ContextManager internally) + * - SessionEventBus (session-local event handling with forwarding) + * + * @param services - The shared services from the agent (state manager, prompt, client managers, etc.) + * @param id - Unique identifier for this session + * @param logger - Logger instance for dependency injection + */ + constructor( + private services: { + stateManager: AgentStateManager; + systemPromptManager: SystemPromptManager; + toolManager: ToolManager; + agentEventBus: AgentEventBus; + storageManager: StorageManager; + resourceManager: import('../resources/index.js').ResourceManager; + pluginManager: PluginManager; + mcpManager: MCPManager; + sessionManager: import('./session-manager.js').SessionManager; + }, + public readonly id: string, + logger: IDextoLogger + ) { + this.logger = logger.createChild(DextoLogComponent.SESSION); + // Create session-specific event bus + this.eventBus = new SessionEventBus(); + + // Set up event forwarding to agent's global bus + this.setupEventForwarding(); + + // Services will be initialized in init() method due to async requirements + this.logger.debug(`ChatSession ${this.id}: Created, awaiting initialization`); + } + + /** + * Initialize the session services asynchronously. + * This must be called after construction to set up the storage-backed services. + */ + public async init(): Promise { + await this.initializeServices(); + } + + /** + * Sets up event forwarding from session bus to global agent bus. + * + * All session events are automatically forwarded to the global bus with the same + * event names, but with session context added to the payload. This allows the app + * layer to continue listening to standard events while having access to session + * information when needed. + */ + private setupEventForwarding(): void { + // Forward each session event type to the agent bus with session context + SessionEventNames.forEach((eventName) => { + const forwarder = (payload?: any) => { + // Create payload with sessionId - handle both void and object payloads + const payloadWithSession = + payload && typeof payload === 'object' + ? { ...payload, sessionId: this.id } + : { sessionId: this.id }; + // Forward to agent bus with session context + this.services.agentEventBus.emit(eventName as any, payloadWithSession); + }; + + // Store the forwarder function for later cleanup + this.forwarders.set(eventName, forwarder); + + // Attach the forwarder to the session event bus + this.eventBus.on(eventName, forwarder); + }); + + // Set up token usage accumulation on llm:response + this.setupTokenAccumulation(); + + this.logger.debug( + `[setupEventForwarding] Event forwarding setup complete for session=${this.id}` + ); + } + + /** + * Sets up token usage accumulation by listening to llm:response events. + * Accumulates token usage and cost to session metadata for /stats tracking. + */ + private setupTokenAccumulation(): void { + this.tokenAccumulatorListener = (payload: SessionEventMap['llm:response']) => { + if (payload.tokenUsage) { + // Calculate cost if pricing is available + let cost: number | undefined; + const llmConfig = this.services.stateManager.getLLMConfig(this.id); + const pricing = getModelPricing(llmConfig.provider, llmConfig.model); + if (pricing) { + cost = calculateCost(payload.tokenUsage, pricing); + } + + // Fire and forget - don't block the event flow + this.services.sessionManager + .accumulateTokenUsage(this.id, payload.tokenUsage, cost) + .catch((err) => { + this.logger.warn( + `Failed to accumulate token usage: ${err instanceof Error ? err.message : String(err)}` + ); + }); + } + }; + + this.eventBus.on('llm:response', this.tokenAccumulatorListener); + } + + /** + * Initializes session-specific services. + */ + private async initializeServices(): Promise { + // Get current effective configuration for this session from state manager + const runtimeConfig = this.services.stateManager.getRuntimeConfig(this.id); + const llmConfig = runtimeConfig.llm; + + // Create session-specific history provider directly with database backend + // This persists across LLM switches to maintain conversation history + this.historyProvider = createDatabaseHistoryProvider( + this.services.storageManager.getDatabase(), + this.id, + this.logger + ); + + // Create model and compaction strategy from config + const model = createVercelModel(llmConfig); + const compactionStrategy = await createCompactionStrategy(runtimeConfig.compaction, { + logger: this.logger, + model, + }); + + // Create session-specific LLM service + // The service will create its own properly-typed ContextManager internally + this.llmService = createLLMService( + llmConfig, + this.services.toolManager, + this.services.systemPromptManager, + this.historyProvider, // Pass history provider for service to use + this.eventBus, // Use session event bus + this.id, + this.services.resourceManager, // Pass ResourceManager for blob storage + this.logger, // Pass logger for dependency injection + compactionStrategy, // Pass compaction strategy + runtimeConfig.compaction // Pass compaction config for threshold settings + ); + + this.logger.debug(`ChatSession ${this.id}: Services initialized with storage`); + } + + /** + * Saves a blocked interaction to history when a plugin blocks execution. + * This ensures that even when a plugin blocks execution (e.g., due to abusive language), + * the user's message and the error response are preserved in the conversation history. + * + * @param userInput - The user's text input that was blocked + * @param errorMessage - The error message explaining why execution was blocked + * @param imageData - Optional image data that was part of the blocked message + * @param fileData - Optional file data that was part of the blocked message + * @private + */ + private async saveBlockedInteraction( + userInput: string, + errorMessage: string, + _imageData?: { image: string; mimeType: string }, + _fileData?: { data: string; mimeType: string; filename?: string } + ): Promise { + // Create redacted user message (do not persist sensitive content or attachments) + // When content is blocked by policy (abusive language, inappropriate content, etc.), + // we shouldn't store the original content to comply with data minimization principles + const userMessage: InternalMessage = { + role: 'user', + content: [{ type: 'text', text: '[Blocked by content policy: input redacted]' }], + }; + + // Create assistant error message + const errorContent = `Error: ${errorMessage}`; + const assistantMessage: InternalMessage = { + role: 'assistant', + content: [{ type: 'text', text: errorContent }], + }; + + // Add both messages to history + await this.historyProvider.saveMessage(userMessage); + await this.historyProvider.saveMessage(assistantMessage); + + // Emit response event so UI updates immediately on blocked interactions + // This ensures listeners relying on llm:response know a response was added + // Note: sessionId is automatically added by event forwarding layer + const llmConfig = this.services.stateManager.getLLMConfig(this.id); + this.eventBus.emit('llm:response', { + content: errorContent, + provider: llmConfig.provider, + model: llmConfig.model, + }); + } + + /** + * Stream a response for the given content. + * Primary method for running conversations with multi-image support. + * + * @param content - String or ContentPart[] (text, images, files) + * @param options - { signal?: AbortSignal } + * @returns Promise that resolves to object with text response + * + * @example + * ```typescript + * // Text only + * const { text } = await session.stream('What is the weather?'); + * + * // Multiple images + * const { text } = await session.stream([ + * { type: 'text', text: 'Compare these images' }, + * { type: 'image', image: base64Data1, mimeType: 'image/png' }, + * { type: 'image', image: base64Data2, mimeType: 'image/png' } + * ]); + * ``` + */ + public async stream( + content: ContentInput, + options?: { signal?: AbortSignal } + ): Promise<{ text: string }> { + // Normalize content to ContentPart[] + const parts: ContentPart[] = + typeof content === 'string' ? [{ type: 'text', text: content }] : content; + + // Extract text for logging (no sensitive content) + const textParts = parts.filter( + (p): p is { type: 'text'; text: string } => p.type === 'text' + ); + const imageParts = parts.filter((p) => p.type === 'image'); + const fileParts = parts.filter((p) => p.type === 'file'); + + this.logger.debug( + `Streaming session ${this.id} | textParts=${textParts.length} | images=${imageParts.length} | files=${fileParts.length}` + ); + + // Create an AbortController for this run and expose for cancellation + this.currentRunController = new AbortController(); + const signal = options?.signal + ? this.combineSignals(options.signal, this.currentRunController.signal) + : this.currentRunController.signal; + + try { + // Execute beforeLLMRequest plugins + // For backward compatibility, extract first image/file for plugin payload + const textContent = textParts.map((p) => p.text).join('\n'); + const firstImage = imageParts[0] as + | { type: 'image'; image: string; mimeType?: string } + | undefined; + const firstFile = fileParts[0] as + | { type: 'file'; data: string; mimeType: string; filename?: string } + | undefined; + + const beforeLLMPayload: BeforeLLMRequestPayload = { + text: textContent, + ...(firstImage && { + imageData: { + image: typeof firstImage.image === 'string' ? firstImage.image : '[binary]', + mimeType: firstImage.mimeType || 'image/jpeg', + }, + }), + ...(firstFile && { + fileData: { + data: typeof firstFile.data === 'string' ? firstFile.data : '[binary]', + mimeType: firstFile.mimeType, + ...(firstFile.filename && { filename: firstFile.filename }), + }, + }), + sessionId: this.id, + }; + + const modifiedBeforePayload = await this.services.pluginManager.executePlugins( + 'beforeLLMRequest', + beforeLLMPayload, + { + sessionManager: this.services.sessionManager, + mcpManager: this.services.mcpManager, + toolManager: this.services.toolManager, + stateManager: this.services.stateManager, + sessionId: this.id, + abortSignal: signal, + } + ); + + // Apply plugin text modifications to the first text part + let modifiedParts = [...parts]; + if (modifiedBeforePayload.text !== textContent && textParts.length > 0) { + // Replace text parts with modified text + modifiedParts = modifiedParts.filter((p) => p.type !== 'text'); + modifiedParts.unshift({ type: 'text', text: modifiedBeforePayload.text }); + } + + // Call LLM service stream + const streamResult = await this.llmService.stream(modifiedParts, { signal }); + + // Execute beforeResponse plugins + const llmConfig = this.services.stateManager.getLLMConfig(this.id); + const beforeResponsePayload: BeforeResponsePayload = { + content: streamResult.text, + provider: llmConfig.provider, + model: llmConfig.model, + sessionId: this.id, + }; + + const modifiedResponsePayload = await this.services.pluginManager.executePlugins( + 'beforeResponse', + beforeResponsePayload, + { + sessionManager: this.services.sessionManager, + mcpManager: this.services.mcpManager, + toolManager: this.services.toolManager, + stateManager: this.services.stateManager, + sessionId: this.id, + abortSignal: signal, + } + ); + + return { + text: modifiedResponsePayload.content, + }; + } catch (error) { + // If this was an intentional cancellation, return partial response from history + const aborted = + (error instanceof Error && error.name === 'AbortError') || + (typeof error === 'object' && error !== null && (error as any).aborted === true); + if (aborted) { + this.eventBus.emit('llm:error', { + error: new Error('Run cancelled'), + context: 'user_cancelled', + recoverable: true, + }); + + // Return partial content that was persisted during streaming + try { + const history = await this.getHistory(); + const lastAssistant = history.filter((m) => m.role === 'assistant').pop(); + if (lastAssistant) { + if (typeof lastAssistant.content === 'string') { + return { text: lastAssistant.content }; + } + // Handle multimodal content (ContentPart[]) - extract text parts + if (Array.isArray(lastAssistant.content)) { + const text = lastAssistant.content + .filter( + (part): part is { type: 'text'; text: string } => + part.type === 'text' + ) + .map((part) => part.text) + .join(''); + if (text) { + return { text }; + } + } + } + } catch { + this.logger.debug('Failed to retrieve partial response from history on cancel'); + } + return { text: '' }; + } + + // Check if this is a plugin blocking error + if ( + error instanceof DextoRuntimeError && + error.code === PluginErrorCode.PLUGIN_BLOCKED_EXECUTION && + error.scope === ErrorScope.PLUGIN && + error.type === ErrorType.FORBIDDEN + ) { + // Save the blocked interaction to history + const textContent = parts + .filter((p): p is { type: 'text'; text: string } => p.type === 'text') + .map((p) => p.text) + .join('\n'); + try { + await this.saveBlockedInteraction(textContent, error.message); + this.logger.debug( + `ChatSession ${this.id}: Saved blocked interaction to history` + ); + } catch (saveError) { + this.logger.warn( + `Failed to save blocked interaction to history: ${ + saveError instanceof Error ? saveError.message : String(saveError) + }` + ); + } + + return { text: error.message }; + } + + this.logger.error( + `Error in ChatSession.stream: ${error instanceof Error ? error.message : String(error)}` + ); + throw error; + } finally { + this.currentRunController = null; + } + } + + /** + * Combine multiple abort signals into one. + */ + private combineSignals(signal1: AbortSignal, signal2: AbortSignal): AbortSignal { + const controller = new AbortController(); + + const onAbort = () => controller.abort(); + + signal1.addEventListener('abort', onAbort); + signal2.addEventListener('abort', onAbort); + + if (signal1.aborted || signal2.aborted) { + controller.abort(); + } + + return controller.signal; + } + + /** + * Retrieves the complete conversation history for this session. + * + * Returns a read-only copy of all messages in the conversation, including: + * - User messages + * - Assistant responses + * - Tool call results + * - System messages + * + * The history is formatted as internal messages and may include multimodal + * content (text and images). + * + * @returns Promise that resolves to a read-only array of conversation messages in chronological order + * + * @example + * ```typescript + * const history = await session.getHistory(); + * console.log(`Conversation has ${history.length} messages`); + * history.forEach(msg => console.log(`${msg.role}: ${msg.content}`)); + * ``` + */ + public async getHistory() { + return await this.historyProvider.getHistory(); + } + + /** + * Reset the conversation history for this session. + * + * This method: + * 1. Clears all messages from the session's conversation history + * 2. Removes persisted history from the storage provider + * 3. Emits a `session:reset` event with session context + * + * The system prompt and session configuration remain unchanged. + * Only the conversation messages are cleared. + * + * @returns Promise that resolves when the reset is complete + * + * @example + * ```typescript + * await session.reset(); + * console.log('Conversation history cleared'); + * ``` + * + * @see {@link ContextManager.resetConversation} for the underlying implementation + */ + public async reset(): Promise { + await this.llmService.getContextManager().resetConversation(); + + // Emit agent-level event with session context + this.services.agentEventBus.emit('session:reset', { + sessionId: this.id, + }); + } + + /** + * Gets the session's ContextManager instance. + * + * @returns The ContextManager for this session + */ + public getContextManager(): ContextManager { + return this.llmService.getContextManager(); + } + + /** + * Gets the session's LLMService instance. + * + * @returns The LLMService for this session + */ + public getLLMService(): VercelLLMService { + return this.llmService; + } + + /** + * Switches the LLM service for this session while preserving conversation history. + * + * This method creates a new LLM service with the specified configuration, + * while maintaining the existing ContextManager and conversation history. This allows + * users to change AI models mid-conversation without losing context. + * + * @param newLLMConfig The new LLM configuration to use + * + * @example + * ```typescript + * // Switch from Claude to GPT-5 while keeping conversation history + * session.switchLLM({ + * provider: 'openai', + * model: 'gpt-5', + * apiKey: process.env.OPENAI_API_KEY, + * }); + * ``` + */ + public async switchLLM(newLLMConfig: ValidatedLLMConfig): Promise { + try { + // Get compression config for this session + const runtimeConfig = this.services.stateManager.getRuntimeConfig(this.id); + + // Create model and compaction strategy from config + const model = createVercelModel(newLLMConfig); + const compactionStrategy = await createCompactionStrategy(runtimeConfig.compaction, { + logger: this.logger, + model, + }); + + // Create new LLM service with new config but SAME history provider + // The service will create its own new ContextManager internally + const newLLMService = createLLMService( + newLLMConfig, + this.services.toolManager, + this.services.systemPromptManager, + this.historyProvider, // Pass the SAME history provider - preserves conversation! + this.eventBus, // Use session event bus + this.id, + this.services.resourceManager, + this.logger, + compactionStrategy, // Pass compaction strategy + runtimeConfig.compaction // Pass compaction config for threshold settings + ); + + // Replace the LLM service + this.llmService = newLLMService; + + this.logger.info( + `ChatSession ${this.id}: LLM switched to ${newLLMConfig.provider}/${newLLMConfig.model}` + ); + + // Emit session-level event + this.eventBus.emit('llm:switched', { + newConfig: newLLMConfig, + historyRetained: true, + }); + } catch (error) { + this.logger.error( + `Error during ChatSession.switchLLM for session ${this.id}: ${error instanceof Error ? error.message : String(error)}` + ); + throw error; + } + } + + /** + * Cleanup the session and its in-memory resources. + * This method should be called when the session is being removed from memory. + * Chat history is preserved in storage and can be restored later. + */ + public async cleanup(): Promise { + try { + // Only dispose of event listeners and in-memory resources + // Do NOT reset conversation - that would delete chat history! + this.dispose(); + + this.logger.debug( + `ChatSession ${this.id}: Memory cleanup completed (chat history preserved)` + ); + } catch (error) { + this.logger.error( + `Error during ChatSession cleanup for session ${this.id}: ${error instanceof Error ? error.message : String(error)}` + ); + throw error; + } + } + + /** + * Cleans up listeners and other resources to prevent memory leaks. + * + * This method should be called when the session is being discarded to ensure + * that event listeners are properly removed from the global event bus. + * Without this cleanup, sessions would remain in memory due to listener references. + */ + public dispose(): void { + this.logger.debug(`Disposing session ${this.id} - cleaning up event listeners`); + + // Remove all event forwarders from the session event bus + this.forwarders.forEach((forwarder, eventName) => { + this.eventBus.off(eventName, forwarder); + }); + + // Clear the forwarders map + this.forwarders.clear(); + + // Remove token accumulator listener + if (this.tokenAccumulatorListener) { + this.eventBus.off('llm:response', this.tokenAccumulatorListener); + this.tokenAccumulatorListener = null; + } + + this.logger.debug(`Session ${this.id} disposed successfully`); + } + + /** + * Check if this session is currently processing a message. + * Returns true if a run is in progress and has not been aborted. + */ + public isBusy(): boolean { + return this.currentRunController !== null && !this.currentRunController.signal.aborted; + } + + /** + * Queue a message for processing when the session is busy. + * The message will be injected into the conversation when the current turn completes. + * + * @param message The user message to queue + * @returns Queue position and message ID + */ + public queueMessage(message: UserMessageInput): { queued: true; position: number; id: string } { + return this.llmService.getMessageQueue().enqueue(message); + } + + /** + * Get all messages currently in the queue. + * @returns Array of queued messages + */ + public getQueuedMessages(): import('./types.js').QueuedMessage[] { + return this.llmService.getMessageQueue().getAll(); + } + + /** + * Remove a queued message. + * @param id Message ID to remove + * @returns true if message was found and removed; false otherwise + */ + public removeQueuedMessage(id: string): boolean { + return this.llmService.getMessageQueue().remove(id); + } + + /** + * Clear all queued messages. + * @returns Number of messages that were cleared + */ + public clearMessageQueue(): number { + const queue = this.llmService.getMessageQueue(); + const count = queue.pendingCount(); + queue.clear(); + return count; + } + + /** + * Cancel the currently running turn for this session, if any. + * Returns true if a run was in progress and was signaled to abort. + */ + public cancel(): boolean { + const controller = this.currentRunController; + if (!controller || controller.signal.aborted) { + return false; + } + try { + controller.abort(); + return true; + } catch { + // Already aborted or abort failed + return false; + } + } +} diff --git a/dexto/packages/core/src/session/error-codes.ts b/dexto/packages/core/src/session/error-codes.ts new file mode 100644 index 00000000..b0b48e45 --- /dev/null +++ b/dexto/packages/core/src/session/error-codes.ts @@ -0,0 +1,16 @@ +/** + * Session-specific error codes + * Includes session lifecycle, management, and state errors + */ +export enum SessionErrorCode { + // Session lifecycle + SESSION_NOT_FOUND = 'session_not_found', + SESSION_INITIALIZATION_FAILED = 'session_initialization_failed', + SESSION_MAX_SESSIONS_EXCEEDED = 'session_max_sessions_exceeded', + + // Session storage + SESSION_STORAGE_FAILED = 'session_storage_failed', + + // Session operations + SESSION_RESET_FAILED = 'session_reset_failed', +} diff --git a/dexto/packages/core/src/session/errors.ts b/dexto/packages/core/src/session/errors.ts new file mode 100644 index 00000000..274ccfb5 --- /dev/null +++ b/dexto/packages/core/src/session/errors.ts @@ -0,0 +1,75 @@ +import { DextoRuntimeError } from '@core/errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { SessionErrorCode } from './error-codes.js'; + +/** + * Session error factory with typed methods for creating session-specific errors + * Each method creates a properly typed DextoError with SESSION scope + */ +export class SessionError { + /** + * Session not found + */ + static notFound(sessionId: string) { + return new DextoRuntimeError( + SessionErrorCode.SESSION_NOT_FOUND, + ErrorScope.SESSION, + ErrorType.NOT_FOUND, + `Session ${sessionId} not found`, + { sessionId } + ); + } + + /** + * Session initialization failed + */ + static initializationFailed(sessionId: string, reason: string) { + return new DextoRuntimeError( + SessionErrorCode.SESSION_INITIALIZATION_FAILED, + ErrorScope.SESSION, + ErrorType.SYSTEM, + `Failed to initialize session '${sessionId}': ${reason}`, + { sessionId, reason } + ); + } + + /** + * Maximum number of sessions exceeded + */ + static maxSessionsExceeded(currentCount: number, maxSessions: number) { + return new DextoRuntimeError( + SessionErrorCode.SESSION_MAX_SESSIONS_EXCEEDED, + ErrorScope.SESSION, + ErrorType.USER, + `Maximum sessions (${maxSessions}) reached`, + { currentCount, maxSessions }, + 'Delete unused sessions or increase maxSessions limit in configuration' + ); + } + + /** + * Session storage failed + */ + static storageFailed(sessionId: string, operation: string, reason: string) { + return new DextoRuntimeError( + SessionErrorCode.SESSION_STORAGE_FAILED, + ErrorScope.SESSION, + ErrorType.SYSTEM, + `Failed to ${operation} session '${sessionId}': ${reason}`, + { sessionId, operation, reason } + ); + } + + /** + * Session reset failed + */ + static resetFailed(sessionId: string, reason: string) { + return new DextoRuntimeError( + SessionErrorCode.SESSION_RESET_FAILED, + ErrorScope.SESSION, + ErrorType.SYSTEM, + `Failed to reset session '${sessionId}': ${reason}`, + { sessionId, reason } + ); + } +} diff --git a/dexto/packages/core/src/session/history/database.test.ts b/dexto/packages/core/src/session/history/database.test.ts new file mode 100644 index 00000000..ae969e11 --- /dev/null +++ b/dexto/packages/core/src/session/history/database.test.ts @@ -0,0 +1,64 @@ +import { describe, test, expect, vi, beforeEach, type Mocked } from 'vitest'; +import { DatabaseHistoryProvider } from './database.js'; +import type { Database } from '@core/storage/types.js'; +import { SessionErrorCode } from '../error-codes.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { createMockLogger } from '@core/logger/v2/test-utils.js'; + +describe('DatabaseHistoryProvider error mapping', () => { + let db: Mocked; + let provider: DatabaseHistoryProvider; + const sessionId = 's-1'; + const mockLogger = createMockLogger(); + + beforeEach(() => { + db = { + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + list: vi.fn(), + clear: vi.fn(), + append: vi.fn(), + getRange: vi.fn(), + getLength: vi.fn(), + connect: vi.fn(), + disconnect: vi.fn(), + isConnected: vi.fn().mockReturnValue(true), + getStoreType: vi.fn().mockReturnValue('memory'), + } as any; + + provider = new DatabaseHistoryProvider(sessionId, db, mockLogger); + }); + + test('saveMessage maps backend error to SessionError.storageFailed', async () => { + db.append.mockRejectedValue(new Error('append failed')); + await expect( + provider.saveMessage({ role: 'user', content: 'hi' } as any) + ).rejects.toMatchObject({ + code: SessionErrorCode.SESSION_STORAGE_FAILED, + scope: ErrorScope.SESSION, + type: ErrorType.SYSTEM, + context: expect.objectContaining({ sessionId }), + }); + }); + + test('clearHistory maps backend error to SessionError.resetFailed', async () => { + db.delete.mockRejectedValue(new Error('delete failed')); + await expect(provider.clearHistory()).rejects.toMatchObject({ + code: SessionErrorCode.SESSION_RESET_FAILED, + scope: ErrorScope.SESSION, + type: ErrorType.SYSTEM, + context: expect.objectContaining({ sessionId }), + }); + }); + + test('getHistory maps backend error to SessionError.storageFailed', async () => { + db.getRange.mockRejectedValue(new Error('getRange failed')); + await expect(provider.getHistory()).rejects.toMatchObject({ + code: SessionErrorCode.SESSION_STORAGE_FAILED, + scope: ErrorScope.SESSION, + type: ErrorType.SYSTEM, + context: expect.objectContaining({ sessionId }), + }); + }); +}); diff --git a/dexto/packages/core/src/session/history/database.ts b/dexto/packages/core/src/session/history/database.ts new file mode 100644 index 00000000..2e9fe04f --- /dev/null +++ b/dexto/packages/core/src/session/history/database.ts @@ -0,0 +1,313 @@ +import type { IDextoLogger } from '@core/logger/v2/types.js'; +import { DextoLogComponent } from '@core/logger/v2/types.js'; +import type { Database } from '@core/storage/types.js'; +import { SessionError } from '../errors.js'; +import type { InternalMessage } from '@core/context/types.js'; +import type { IConversationHistoryProvider } from './types.js'; + +/** + * History provider that works directly with DatabaseBackend. + * Uses write-through caching for read performance while maintaining durability. + * + * Caching strategy: + * - getHistory(): Returns cached messages after first load (eliminates repeated DB reads) + * - saveMessage(): Updates cache AND writes to DB immediately (new messages are critical) + * - updateMessage(): Updates cache immediately, debounces DB writes (batches rapid updates) + * - flush(): Forces all pending updates to DB (called at turn boundaries) + * - clearHistory(): Clears cache and DB immediately + * + * Durability guarantees: + * - New messages (saveMessage) are always immediately durable + * - Updates (updateMessage) are durable within flush interval or on explicit flush() + * - Worst case on crash: lose updates from last flush interval (typically <100ms) + */ +export class DatabaseHistoryProvider implements IConversationHistoryProvider { + private logger: IDextoLogger; + + // Cache state + private cache: InternalMessage[] | null = null; + private dirty = false; + private flushTimer: ReturnType | null = null; + private flushPromise: Promise | null = null; + + // Flush configuration + private static readonly FLUSH_DELAY_MS = 100; // Debounce window for batching updates + + constructor( + private sessionId: string, + private database: Database, + logger: IDextoLogger + ) { + this.logger = logger.createChild(DextoLogComponent.SESSION); + } + + async getHistory(): Promise { + // Load from DB on first access + if (this.cache === null) { + const key = this.getMessagesKey(); + try { + const limit = 10000; + const rawMessages = await this.database.getRange(key, 0, limit); + + if (rawMessages.length === limit) { + this.logger.warn( + `DatabaseHistoryProvider: Session ${this.sessionId} hit message limit (${limit}), history may be truncated` + ); + } + + // Deduplicate messages by ID (keep first occurrence to preserve order) + const seen = new Set(); + this.cache = []; + let duplicateCount = 0; + + for (const msg of rawMessages) { + if (msg.id && seen.has(msg.id)) { + duplicateCount++; + continue; // Skip duplicate + } + if (msg.id) { + seen.add(msg.id); + } + this.cache.push(msg); + } + + // Log and self-heal if duplicates found (indicates prior data corruption) + if (duplicateCount > 0) { + this.logger.warn( + `DatabaseHistoryProvider: Found ${duplicateCount} duplicate messages for session ${this.sessionId}, deduped to ${this.cache.length}` + ); + // Mark dirty to rewrite clean data on next flush + this.dirty = true; + this.scheduleFlush(); + } else { + this.logger.debug( + `DatabaseHistoryProvider: Loaded ${this.cache.length} messages for session ${this.sessionId}` + ); + } + } catch (error) { + this.logger.error( + `DatabaseHistoryProvider: Error loading messages for session ${this.sessionId}: ${error instanceof Error ? error.message : String(error)}` + ); + throw SessionError.storageFailed( + this.sessionId, + 'load history', + error instanceof Error ? error.message : String(error) + ); + } + } + + // Return a copy to prevent external mutation + return [...this.cache]; + } + + async saveMessage(message: InternalMessage): Promise { + const key = this.getMessagesKey(); + + // Ensure cache is initialized + if (this.cache === null) { + await this.getHistory(); + } + + // Check if message already exists in cache (prevent duplicates) + if (message.id && this.cache!.some((m) => m.id === message.id)) { + this.logger.debug( + `DatabaseHistoryProvider: Message ${message.id} already exists, skipping` + ); + return; + } + + // Update cache + this.cache!.push(message); + + // Write to DB immediately - new messages must be durable + try { + await this.database.append(key, message); + + this.logger.debug( + `DatabaseHistoryProvider: Saved message ${message.id} (${message.role}) for session ${this.sessionId}` + ); + } catch (error) { + // Remove from cache on failure to keep in sync + this.cache!.pop(); + this.logger.error( + `DatabaseHistoryProvider: Error saving message for session ${this.sessionId}: ${error instanceof Error ? error.message : String(error)}` + ); + throw SessionError.storageFailed( + this.sessionId, + 'save message', + error instanceof Error ? error.message : String(error) + ); + } + } + + async updateMessage(message: InternalMessage): Promise { + // Guard against undefined id + if (!message.id) { + this.logger.warn( + `DatabaseHistoryProvider: Ignoring update for message without id in session ${this.sessionId}` + ); + return; + } + + // Ensure cache is initialized + if (this.cache === null) { + await this.getHistory(); + } + + // Update cache immediately (fast, in-memory) + const index = this.cache!.findIndex((m) => m.id === message.id); + if (index !== -1) { + this.cache![index] = message; + this.dirty = true; + + // Schedule debounced flush + this.scheduleFlush(); + + this.logger.debug( + `DatabaseHistoryProvider: Updated message ${message.id} in cache for session ${this.sessionId}` + ); + } else { + this.logger.warn( + `DatabaseHistoryProvider: Message ${message.id} not found for update in session ${this.sessionId}` + ); + } + } + + async clearHistory(): Promise { + // Cancel any pending flush + this.cancelPendingFlush(); + + // Clear cache + this.cache = []; + this.dirty = false; + + // Clear DB + const key = this.getMessagesKey(); + try { + await this.database.delete(key); + this.logger.debug( + `DatabaseHistoryProvider: Cleared history for session ${this.sessionId}` + ); + } catch (error) { + this.logger.error( + `DatabaseHistoryProvider: Error clearing session ${this.sessionId}: ${error instanceof Error ? error.message : String(error)}` + ); + throw SessionError.resetFailed( + this.sessionId, + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Flush any pending updates to the database. + * Should be called at turn boundaries to ensure durability. + */ + async flush(): Promise { + // If a flush is already in progress, wait for it + if (this.flushPromise) { + await this.flushPromise; + return; + } + + // Cancel any scheduled flush since we're flushing now + this.cancelPendingFlush(); + + // Nothing to flush + if (!this.dirty || !this.cache) { + return; + } + + // Perform the flush + this.flushPromise = this.doFlush(); + try { + await this.flushPromise; + } finally { + this.flushPromise = null; + } + } + + /** + * Internal flush implementation. + * Writes entire cache to DB (delete + re-append all). + */ + private async doFlush(): Promise { + if (!this.dirty || !this.cache) { + return; + } + + const key = this.getMessagesKey(); + + // Take a snapshot of cache to avoid race conditions with concurrent saveMessage() calls. + // If saveMessage() is called during flush, it will append to the live cache AND write to DB. + // By iterating over a snapshot, we avoid re-appending messages that were already written. + const snapshot = [...this.cache]; + const messageCount = snapshot.length; + + this.logger.debug( + `DatabaseHistoryProvider: FLUSH START key=${key} snapshotSize=${messageCount} ids=[${snapshot.map((m) => m.id).join(',')}]` + ); + + try { + // Atomic replace: delete all + re-append from snapshot + await this.database.delete(key); + this.logger.debug(`DatabaseHistoryProvider: FLUSH DELETED key=${key}`); + + for (const msg of snapshot) { + await this.database.append(key, msg); + } + this.logger.debug( + `DatabaseHistoryProvider: FLUSH REAPPENDED key=${key} count=${messageCount}` + ); + + // Only clear dirty if no new updates were scheduled during flush. + // If flushTimer exists, updateMessage() was called during the flush, + // so keep dirty=true to ensure the scheduled flush persists those updates. + if (!this.flushTimer) { + this.dirty = false; + } + } catch (error) { + this.logger.error( + `DatabaseHistoryProvider: Error flushing messages for session ${this.sessionId}: ${error instanceof Error ? error.message : String(error)}` + ); + throw SessionError.storageFailed( + this.sessionId, + 'flush messages', + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Schedule a debounced flush. + * Batches rapid updateMessage() calls into a single DB write. + */ + private scheduleFlush(): void { + // Already scheduled + if (this.flushTimer) { + return; + } + + this.flushTimer = setTimeout(() => { + this.flushTimer = null; + // Use flush() instead of doFlush() to respect flushPromise concurrency guard + this.flush().catch(() => { + // Error already logged in doFlush + }); + }, DatabaseHistoryProvider.FLUSH_DELAY_MS); + } + + /** + * Cancel any pending scheduled flush. + */ + private cancelPendingFlush(): void { + if (this.flushTimer) { + clearTimeout(this.flushTimer); + this.flushTimer = null; + } + } + + private getMessagesKey(): string { + return `messages:${this.sessionId}`; + } +} diff --git a/dexto/packages/core/src/session/history/factory.ts b/dexto/packages/core/src/session/history/factory.ts new file mode 100644 index 00000000..59b34b0e --- /dev/null +++ b/dexto/packages/core/src/session/history/factory.ts @@ -0,0 +1,18 @@ +import type { IConversationHistoryProvider } from './types.js'; +import type { Database } from '@core/storage/types.js'; +import type { IDextoLogger } from '@core/logger/v2/types.js'; +import { DatabaseHistoryProvider } from './database.js'; + +/** + * Create a history provider directly with database backend + * @param database Database instance + * @param sessionId Session ID + * @param logger Logger instance for logging + */ +export function createDatabaseHistoryProvider( + database: Database, + sessionId: string, + logger: IDextoLogger +): IConversationHistoryProvider { + return new DatabaseHistoryProvider(sessionId, database, logger); +} diff --git a/dexto/packages/core/src/session/history/memory.ts b/dexto/packages/core/src/session/history/memory.ts new file mode 100644 index 00000000..171785e1 --- /dev/null +++ b/dexto/packages/core/src/session/history/memory.ts @@ -0,0 +1,46 @@ +import type { InternalMessage } from '@core/context/types.js'; +import type { IConversationHistoryProvider } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; + +/** + * Lightweight in-memory history provider for ephemeral, isolated LLM calls. + * Used to run background tasks (e.g., title generation) without touching + * the real session history or emitting history-related side effects. + */ +export class MemoryHistoryProvider implements IConversationHistoryProvider { + private messages: InternalMessage[] = []; + + constructor(private logger: IDextoLogger) {} + + async getHistory(): Promise { + // Return a shallow copy to prevent external mutation + return [...this.messages]; + } + + async saveMessage(message: InternalMessage): Promise { + this.messages.push(message); + } + + async updateMessage(message: InternalMessage): Promise { + // Guard against undefined id - could match another message with undefined id + if (!message.id) { + this.logger.warn('MemoryHistoryProvider: Ignoring update for message without id'); + return; + } + const index = this.messages.findIndex((m) => m.id === message.id); + if (index !== -1) { + this.messages[index] = message; + } + } + + async clearHistory(): Promise { + this.messages = []; + } + + /** + * No-op for in-memory provider - all operations are already "flushed". + */ + async flush(): Promise { + // Nothing to flush - memory provider is always in sync + } +} diff --git a/dexto/packages/core/src/session/history/types.ts b/dexto/packages/core/src/session/history/types.ts new file mode 100644 index 00000000..f01973a6 --- /dev/null +++ b/dexto/packages/core/src/session/history/types.ts @@ -0,0 +1,30 @@ +import type { InternalMessage } from '@core/context/types.js'; + +/** + * Session-scoped conversation history provider. + * Each instance is tied to a specific session and manages only that session's messages. + * + * Implementations may use caching to optimize read performance. If caching is used, + * the flush() method should be called at turn boundaries to ensure all updates + * are persisted to durable storage. + */ +export interface IConversationHistoryProvider { + /** Load the full message history for this session */ + getHistory(): Promise; + + /** Append a message to this session's history (must be durable immediately) */ + saveMessage(message: InternalMessage): Promise; + + /** Update an existing message in this session's history (may be cached/batched) */ + updateMessage(message: InternalMessage): Promise; + + /** Clear all messages for this session */ + clearHistory(): Promise; + + /** + * Flush any pending updates to durable storage. + * Called at turn boundaries to ensure data durability. + * Implementations without caching can make this a no-op. + */ + flush(): Promise; +} diff --git a/dexto/packages/core/src/session/index.ts b/dexto/packages/core/src/session/index.ts new file mode 100644 index 00000000..28aa430f --- /dev/null +++ b/dexto/packages/core/src/session/index.ts @@ -0,0 +1,8 @@ +export { ChatSession } from './chat-session.js'; +export { SessionManager } from './session-manager.js'; +export type { SessionMetadata } from './session-manager.js'; +export { SessionErrorCode } from './error-codes.js'; +export { SessionError } from './errors.js'; +export { MessageQueueService } from './message-queue.js'; +export type { UserMessageInput } from './message-queue.js'; +export type { QueuedMessage, CoalescedMessage } from './types.js'; diff --git a/dexto/packages/core/src/session/message-queue.test.ts b/dexto/packages/core/src/session/message-queue.test.ts new file mode 100644 index 00000000..2620a1b0 --- /dev/null +++ b/dexto/packages/core/src/session/message-queue.test.ts @@ -0,0 +1,378 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { MessageQueueService } from './message-queue.js'; +import type { SessionEventBus } from '../events/index.js'; +import type { ContentPart } from '../context/types.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; + +// Create a mock SessionEventBus +function createMockEventBus(): SessionEventBus { + return { + emit: vi.fn(), + on: vi.fn(), + off: vi.fn(), + once: vi.fn(), + removeAllListeners: vi.fn(), + } as unknown as SessionEventBus; +} + +describe('MessageQueueService', () => { + let eventBus: SessionEventBus; + let logger: IDextoLogger; + let queue: MessageQueueService; + + beforeEach(() => { + eventBus = createMockEventBus(); + logger = createMockLogger(); + queue = new MessageQueueService(eventBus, logger); + }); + + describe('enqueue()', () => { + it('should add a message to the queue and return position and id', () => { + const content: ContentPart[] = [{ type: 'text', text: 'hello' }]; + + const result = queue.enqueue({ content }); + + expect(result.queued).toBe(true); + expect(result.position).toBe(1); + expect(result.id).toMatch(/^msg_\d+_[a-z0-9]+$/); + }); + + it('should increment position for multiple enqueued messages', () => { + const content: ContentPart[] = [{ type: 'text', text: 'hello' }]; + + const result1 = queue.enqueue({ content }); + const result2 = queue.enqueue({ content }); + const result3 = queue.enqueue({ content }); + + expect(result1.position).toBe(1); + expect(result2.position).toBe(2); + expect(result3.position).toBe(3); + }); + + it('should emit message:queued event with correct data', () => { + const content: ContentPart[] = [{ type: 'text', text: 'hello' }]; + + const result = queue.enqueue({ content }); + + expect(eventBus.emit).toHaveBeenCalledWith('message:queued', { + position: 1, + id: result.id, + }); + }); + + it('should include metadata when provided', () => { + const content: ContentPart[] = [{ type: 'text', text: 'hello' }]; + const metadata = { source: 'api', priority: 'high' }; + + queue.enqueue({ content, metadata }); + const coalesced = queue.dequeueAll(); + + expect(coalesced).not.toBeNull(); + const firstMessage = coalesced?.messages[0]; + expect(firstMessage?.metadata).toEqual(metadata); + }); + + it('should not include metadata field when not provided', () => { + const content: ContentPart[] = [{ type: 'text', text: 'hello' }]; + + queue.enqueue({ content }); + const coalesced = queue.dequeueAll(); + + expect(coalesced).not.toBeNull(); + const firstMessage = coalesced?.messages[0]; + expect(firstMessage?.metadata).toBeUndefined(); + }); + }); + + describe('dequeueAll()', () => { + it('should return null when queue is empty', () => { + const result = queue.dequeueAll(); + expect(result).toBeNull(); + }); + + it('should return CoalescedMessage with single message', () => { + const content: ContentPart[] = [{ type: 'text', text: 'hello' }]; + queue.enqueue({ content }); + + const result = queue.dequeueAll(); + + expect(result).not.toBeNull(); + expect(result?.messages).toHaveLength(1); + expect(result?.combinedContent).toEqual(content); + }); + + it('should clear the queue after dequeue', () => { + queue.enqueue({ content: [{ type: 'text', text: 'hello' }] }); + queue.dequeueAll(); + + expect(queue.hasPending()).toBe(false); + expect(queue.pendingCount()).toBe(0); + }); + + it('should emit message:dequeued event with correct data', () => { + queue.enqueue({ content: [{ type: 'text', text: 'msg1' }] }); + queue.enqueue({ content: [{ type: 'text', text: 'msg2' }] }); + + queue.dequeueAll(); + + expect(eventBus.emit).toHaveBeenCalledWith('message:dequeued', { + count: 2, + ids: expect.arrayContaining([expect.stringMatching(/^msg_/)]), + coalesced: true, + content: expect.any(Array), + }); + }); + + it('should set coalesced to false for single message', () => { + queue.enqueue({ content: [{ type: 'text', text: 'solo' }] }); + + queue.dequeueAll(); + + expect(eventBus.emit).toHaveBeenCalledWith('message:dequeued', { + count: 1, + ids: expect.any(Array), + coalesced: false, + content: [{ type: 'text', text: 'solo' }], + }); + }); + }); + + describe('coalescing', () => { + it('should return single message content as-is', () => { + const content: ContentPart[] = [ + { type: 'text', text: 'hello world' }, + { type: 'image', image: 'base64data', mimeType: 'image/png' }, + ]; + queue.enqueue({ content }); + + const result = queue.dequeueAll(); + + expect(result?.combinedContent).toEqual(content); + }); + + it('should prefix two messages with First and Also', () => { + queue.enqueue({ content: [{ type: 'text', text: 'stop' }] }); + queue.enqueue({ content: [{ type: 'text', text: 'try another way' }] }); + + const result = queue.dequeueAll(); + + expect(result?.combinedContent).toHaveLength(3); // First + separator + Also + expect(result?.combinedContent[0]).toEqual({ type: 'text', text: 'First: stop' }); + expect(result?.combinedContent[1]).toEqual({ type: 'text', text: '\n\n' }); + expect(result?.combinedContent[2]).toEqual({ + type: 'text', + text: 'Also: try another way', + }); + }); + + it('should number three or more messages', () => { + queue.enqueue({ content: [{ type: 'text', text: 'one' }] }); + queue.enqueue({ content: [{ type: 'text', text: 'two' }] }); + queue.enqueue({ content: [{ type: 'text', text: 'three' }] }); + + const result = queue.dequeueAll(); + + expect(result?.combinedContent).toHaveLength(5); // 3 messages + 2 separators + expect(result?.combinedContent[0]).toEqual({ type: 'text', text: '[1]: one' }); + expect(result?.combinedContent[2]).toEqual({ type: 'text', text: '[2]: two' }); + expect(result?.combinedContent[4]).toEqual({ type: 'text', text: '[3]: three' }); + }); + + it('should preserve multimodal content (text + images)', () => { + queue.enqueue({ + content: [ + { type: 'text', text: 'look at this' }, + { type: 'image', image: 'base64img1', mimeType: 'image/png' }, + ], + }); + queue.enqueue({ + content: [{ type: 'image', image: 'base64img2', mimeType: 'image/jpeg' }], + }); + + const result = queue.dequeueAll(); + + // Should have: "First: look at this", image1, separator, "Also: ", image2 + expect(result?.combinedContent).toHaveLength(5); + expect(result?.combinedContent[0]).toEqual({ + type: 'text', + text: 'First: look at this', + }); + expect(result?.combinedContent[1]).toEqual({ + type: 'image', + image: 'base64img1', + mimeType: 'image/png', + }); + expect(result?.combinedContent[3]).toEqual({ type: 'text', text: 'Also: ' }); + expect(result?.combinedContent[4]).toEqual({ + type: 'image', + image: 'base64img2', + mimeType: 'image/jpeg', + }); + }); + + it('should handle empty message content with placeholder', () => { + queue.enqueue({ content: [{ type: 'text', text: 'first' }] }); + queue.enqueue({ content: [] }); + + const result = queue.dequeueAll(); + + expect(result?.combinedContent).toContainEqual({ + type: 'text', + text: 'Also: [empty message]', + }); + }); + + it('should set correct firstQueuedAt and lastQueuedAt timestamps', async () => { + queue.enqueue({ content: [{ type: 'text', text: 'first' }] }); + + // Small delay to ensure different timestamps + await new Promise((resolve) => setTimeout(resolve, 10)); + + queue.enqueue({ content: [{ type: 'text', text: 'second' }] }); + + const result = queue.dequeueAll(); + + expect(result?.firstQueuedAt).toBeLessThan(result?.lastQueuedAt ?? 0); + }); + }); + + describe('hasPending() and pendingCount()', () => { + it('should return false and 0 for empty queue', () => { + expect(queue.hasPending()).toBe(false); + expect(queue.pendingCount()).toBe(0); + }); + + it('should return true and correct count for non-empty queue', () => { + queue.enqueue({ content: [{ type: 'text', text: 'msg1' }] }); + queue.enqueue({ content: [{ type: 'text', text: 'msg2' }] }); + + expect(queue.hasPending()).toBe(true); + expect(queue.pendingCount()).toBe(2); + }); + + it('should update after dequeue', () => { + queue.enqueue({ content: [{ type: 'text', text: 'msg1' }] }); + expect(queue.pendingCount()).toBe(1); + + queue.dequeueAll(); + + expect(queue.hasPending()).toBe(false); + expect(queue.pendingCount()).toBe(0); + }); + }); + + describe('clear()', () => { + it('should empty the queue', () => { + queue.enqueue({ content: [{ type: 'text', text: 'msg1' }] }); + queue.enqueue({ content: [{ type: 'text', text: 'msg2' }] }); + + queue.clear(); + + expect(queue.hasPending()).toBe(false); + expect(queue.pendingCount()).toBe(0); + expect(queue.dequeueAll()).toBeNull(); + }); + }); + + describe('getAll()', () => { + it('should return empty array when queue is empty', () => { + expect(queue.getAll()).toEqual([]); + }); + + it('should return shallow copy of queued messages', () => { + const result1 = queue.enqueue({ content: [{ type: 'text', text: 'msg1' }] }); + const result2 = queue.enqueue({ content: [{ type: 'text', text: 'msg2' }] }); + + const all = queue.getAll(); + + expect(all).toHaveLength(2); + expect(all[0]?.id).toBe(result1.id); + expect(all[1]?.id).toBe(result2.id); + }); + + it('should not allow external mutation of queue', () => { + queue.enqueue({ content: [{ type: 'text', text: 'msg1' }] }); + + const all = queue.getAll(); + all.push({ + id: 'fake', + content: [{ type: 'text', text: 'fake' }], + queuedAt: Date.now(), + }); + + expect(queue.getAll()).toHaveLength(1); + }); + }); + + describe('get()', () => { + it('should return undefined for non-existent id', () => { + expect(queue.get('non-existent')).toBeUndefined(); + }); + + it('should return message by id', () => { + const content: ContentPart[] = [{ type: 'text', text: 'hello' }]; + const result = queue.enqueue({ content }); + + const msg = queue.get(result.id); + + expect(msg).toBeDefined(); + expect(msg?.id).toBe(result.id); + expect(msg?.content).toEqual(content); + }); + }); + + describe('remove()', () => { + it('should return false for non-existent id', () => { + const result = queue.remove('non-existent'); + + expect(result).toBe(false); + expect(logger.debug).toHaveBeenCalledWith( + 'Remove failed: message non-existent not found in queue' + ); + }); + + it('should remove message and return true', () => { + const result = queue.enqueue({ content: [{ type: 'text', text: 'to remove' }] }); + + const removed = queue.remove(result.id); + + expect(removed).toBe(true); + expect(queue.get(result.id)).toBeUndefined(); + expect(queue.pendingCount()).toBe(0); + }); + + it('should emit message:removed event', () => { + const result = queue.enqueue({ content: [{ type: 'text', text: 'to remove' }] }); + + queue.remove(result.id); + + expect(eventBus.emit).toHaveBeenCalledWith('message:removed', { + id: result.id, + }); + }); + + it('should log debug message on successful removal', () => { + const result = queue.enqueue({ content: [{ type: 'text', text: 'to remove' }] }); + + queue.remove(result.id); + + expect(logger.debug).toHaveBeenCalledWith( + `Message removed: ${result.id}, remaining: 0` + ); + }); + + it('should maintain order of remaining messages', () => { + const r1 = queue.enqueue({ content: [{ type: 'text', text: 'first' }] }); + const r2 = queue.enqueue({ content: [{ type: 'text', text: 'second' }] }); + const r3 = queue.enqueue({ content: [{ type: 'text', text: 'third' }] }); + + queue.remove(r2.id); + + const all = queue.getAll(); + expect(all).toHaveLength(2); + expect(all[0]?.id).toBe(r1.id); + expect(all[1]?.id).toBe(r3.id); + }); + }); +}); diff --git a/dexto/packages/core/src/session/message-queue.ts b/dexto/packages/core/src/session/message-queue.ts new file mode 100644 index 00000000..030fb8f6 --- /dev/null +++ b/dexto/packages/core/src/session/message-queue.ts @@ -0,0 +1,264 @@ +import type { SessionEventBus } from '../events/index.js'; +import type { QueuedMessage, CoalescedMessage } from './types.js'; +import type { ContentPart } from '../context/types.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; + +/** + * Generates a unique ID for queued messages. + */ +function generateId(): string { + return `msg_${Date.now()}_${Math.random().toString(36).slice(2, 11)}`; +} + +/** + * Input for enqueuing a user message to the queue. + * (Not to be confused with UserMessage from context/types.ts which represents + * a message in conversation history) + */ +export interface UserMessageInput { + /** Multimodal content array (text, images, files, etc.) */ + content: ContentPart[]; + /** Optional metadata to attach to the message */ + metadata?: Record; +} + +/** + * MessageQueueService handles queuing of user messages during agent execution. + * + * Key features: + * - Accepts messages when the agent is busy executing tools + * - Coalesces multiple queued messages into a single injection + * - Supports multimodal content (text, images, files) + * + * This enables user guidance where users can send + * mid-task instructions like "stop" or "try a different approach". + * + * @example + * ```typescript + * // In API handler - queue message if agent is busy + * if (agent.isBusy()) { + * return messageQueue.enqueue({ content: [{ type: 'text', text: 'stop' }] }); + * } + * + * // In TurnExecutor - check for queued messages + * const coalesced = messageQueue.dequeueAll(); + * if (coalesced) { + * await contextManager.addMessage({ + * role: 'user', + * content: coalesced.combinedContent, + * }); + * } + * ``` + */ +export class MessageQueueService { + private queue: QueuedMessage[] = []; + + constructor( + private eventBus: SessionEventBus, + private logger: IDextoLogger + ) {} + + /** + * Add a message to the queue. + * Called by API endpoint - returns immediately with queue position. + * + * @param message The user message to queue + * @returns Queue position and message ID + */ + enqueue(message: UserMessageInput): { queued: true; position: number; id: string } { + const queuedMsg: QueuedMessage = { + id: generateId(), + content: message.content, + queuedAt: Date.now(), + ...(message.metadata !== undefined && { metadata: message.metadata }), + }; + + this.queue.push(queuedMsg); + + this.logger.debug(`Message queued: ${queuedMsg.id}, position: ${this.queue.length}`); + + this.eventBus.emit('message:queued', { + position: this.queue.length, + id: queuedMsg.id, + }); + + return { + queued: true, + position: this.queue.length, + id: queuedMsg.id, + }; + } + + /** + * Dequeue ALL pending messages and coalesce into single injection. + * Called by executor between steps. + * + * Multiple queued messages become ONE combined message to the LLM. + * + * @example + * If 3 messages are queued: "stop", "try X instead", "also check Y" + * They become: + * ``` + * [1]: stop + * + * [2]: try X instead + * + * [3]: also check Y + * ``` + * + * @returns Coalesced message or null if queue is empty + */ + dequeueAll(): CoalescedMessage | null { + if (this.queue.length === 0) return null; + + const messages = [...this.queue]; + this.queue = []; + + const combined = this.coalesce(messages); + + this.logger.debug( + `Dequeued ${messages.length} message(s): ${messages.map((m) => m.id).join(', ')}` + ); + + this.eventBus.emit('message:dequeued', { + count: messages.length, + ids: messages.map((m) => m.id), + coalesced: messages.length > 1, + content: combined.combinedContent, + }); + + return combined; + } + + /** + * Coalesce multiple messages into one (multimodal-aware). + * Strategy: Combine with numbered separators, preserve all media. + */ + private coalesce(messages: QueuedMessage[]): CoalescedMessage { + // Single message - return as-is + if (messages.length === 1) { + const firstMsg = messages[0]; + if (!firstMsg) { + // This should never happen since we check length === 1, but satisfies TypeScript + throw new Error('Unexpected empty messages array'); + } + return { + messages, + combinedContent: firstMsg.content, + firstQueuedAt: firstMsg.queuedAt, + lastQueuedAt: firstMsg.queuedAt, + }; + } + + // Multiple messages - combine with numbered prefixes + const combinedContent: ContentPart[] = []; + + for (const [i, msg] of messages.entries()) { + // Add prefix based on message count + const prefix = messages.length === 2 ? (i === 0 ? 'First' : 'Also') : `[${i + 1}]`; + + // Start with prefix text + let prefixText = `${prefix}: `; + + // Process content parts + for (const part of msg.content) { + if (part.type === 'text') { + // Combine prefix with first text part for cleaner output + if (prefixText) { + combinedContent.push({ type: 'text', text: prefixText + part.text }); + prefixText = ''; + } else { + combinedContent.push(part); + } + } else { + // If we haven't added prefix yet (message started with media), add it first + if (prefixText) { + combinedContent.push({ type: 'text', text: prefixText }); + prefixText = ''; + } + // Images, files, and other media are added as-is + combinedContent.push(part); + } + } + + // If the message only had media (no text), prefix was already added + // If message was empty, add just the prefix + if (prefixText && msg.content.length === 0) { + combinedContent.push({ type: 'text', text: prefixText + '[empty message]' }); + } + + // Add separator between messages (not after last one) + if (i < messages.length - 1) { + combinedContent.push({ type: 'text', text: '\n\n' }); + } + } + + // Get first and last messages - safe because we checked length > 1 above + const firstMessage = messages[0]; + const lastMessage = messages[messages.length - 1]; + if (!firstMessage || !lastMessage) { + throw new Error('Unexpected undefined message in non-empty array'); + } + + return { + messages, + combinedContent, + firstQueuedAt: firstMessage.queuedAt, + lastQueuedAt: lastMessage.queuedAt, + }; + } + + /** + * Check if there are pending messages in the queue. + */ + hasPending(): boolean { + return this.queue.length > 0; + } + + /** + * Get the number of pending messages. + */ + pendingCount(): number { + return this.queue.length; + } + + /** + * Clear all pending messages without processing. + * Used during cleanup/abort. + */ + clear(): void { + this.queue = []; + } + + /** + * Get all queued messages (for UI display). + * Returns a shallow copy to prevent external mutation. + */ + getAll(): QueuedMessage[] { + return [...this.queue]; + } + + /** + * Get a single queued message by ID. + */ + get(id: string): QueuedMessage | undefined { + return this.queue.find((m) => m.id === id); + } + + /** + * Remove a single queued message by ID. + * @returns true if message was found and removed; false otherwise + */ + remove(id: string): boolean { + const index = this.queue.findIndex((m) => m.id === id); + if (index === -1) { + this.logger.debug(`Remove failed: message ${id} not found in queue`); + return false; + } + + this.queue.splice(index, 1); + this.logger.debug(`Message removed: ${id}, remaining: ${this.queue.length}`); + this.eventBus.emit('message:removed', { id }); + return true; + } +} diff --git a/dexto/packages/core/src/session/schemas.test.ts b/dexto/packages/core/src/session/schemas.test.ts new file mode 100644 index 00000000..96872ef9 --- /dev/null +++ b/dexto/packages/core/src/session/schemas.test.ts @@ -0,0 +1,204 @@ +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { SessionConfigSchema, type SessionConfig, type ValidatedSessionConfig } from './schemas.js'; + +describe('SessionConfigSchema', () => { + describe('Field Validation', () => { + it('should validate maxSessions as positive integer', () => { + // Negative should fail + let result = SessionConfigSchema.safeParse({ maxSessions: -1 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['maxSessions']); + + // Zero should fail + result = SessionConfigSchema.safeParse({ maxSessions: 0 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['maxSessions']); + + // Float should fail + result = SessionConfigSchema.safeParse({ maxSessions: 1.5 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['maxSessions']); + + // Valid values should pass + const valid1 = SessionConfigSchema.parse({ maxSessions: 1 }); + expect(valid1.maxSessions).toBe(1); + + const valid2 = SessionConfigSchema.parse({ maxSessions: 100 }); + expect(valid2.maxSessions).toBe(100); + }); + + it('should validate sessionTTL as positive integer', () => { + // Negative should fail + let result = SessionConfigSchema.safeParse({ sessionTTL: -1 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['sessionTTL']); + + // Zero should fail + result = SessionConfigSchema.safeParse({ sessionTTL: 0 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['sessionTTL']); + + // Float should fail + result = SessionConfigSchema.safeParse({ sessionTTL: 1.5 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['sessionTTL']); + + // Valid values should pass + const valid1 = SessionConfigSchema.parse({ sessionTTL: 1000 }); + expect(valid1.sessionTTL).toBe(1000); + + const valid2 = SessionConfigSchema.parse({ sessionTTL: 3600000 }); + expect(valid2.sessionTTL).toBe(3600000); + }); + + it('should reject string inputs without coercion', () => { + const result = SessionConfigSchema.safeParse({ + maxSessions: '50', + sessionTTL: '1800000', + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['maxSessions']); + }); + }); + + describe('Default Values', () => { + it('should apply field defaults for empty object', () => { + const result = SessionConfigSchema.parse({}); + + expect(result).toEqual({ + maxSessions: 100, + sessionTTL: 3600000, + }); + }); + + it('should apply field defaults for partial config', () => { + const result1 = SessionConfigSchema.parse({ maxSessions: 50 }); + expect(result1).toEqual({ + maxSessions: 50, + sessionTTL: 3600000, + }); + + const result2 = SessionConfigSchema.parse({ sessionTTL: 1800000 }); + expect(result2).toEqual({ + maxSessions: 100, + sessionTTL: 1800000, + }); + }); + + it('should override defaults when values provided', () => { + const config = { + maxSessions: 200, + sessionTTL: 7200000, + }; + + const result = SessionConfigSchema.parse(config); + expect(result).toEqual(config); + }); + }); + + describe('Edge Cases', () => { + it('should handle boundary values', () => { + // Very small valid values + const small = SessionConfigSchema.parse({ + maxSessions: 1, + sessionTTL: 1, + }); + expect(small.maxSessions).toBe(1); + expect(small.sessionTTL).toBe(1); + + // Large values + const large = SessionConfigSchema.parse({ + maxSessions: 10000, + sessionTTL: 86400000, // 24 hours + }); + expect(large.maxSessions).toBe(10000); + expect(large.sessionTTL).toBe(86400000); + }); + + it('should reject non-numeric types', () => { + // String should fail + let result = SessionConfigSchema.safeParse({ maxSessions: 'abc' }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['maxSessions']); + + // Null should fail + result = SessionConfigSchema.safeParse({ sessionTTL: null }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['sessionTTL']); + + // Object should fail + result = SessionConfigSchema.safeParse({ maxSessions: {} }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['maxSessions']); + }); + + it('should reject extra fields with strict validation', () => { + const configWithExtra = { + maxSessions: 100, + sessionTTL: 3600000, + unknownField: 'should fail', + }; + + const result = SessionConfigSchema.safeParse(configWithExtra); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + + describe('Type Safety', () => { + it('should have correct input and output types', () => { + // Input type allows optional fields (due to defaults) + const input: SessionConfig = {}; + const inputPartial: SessionConfig = { maxSessions: 50 }; + const inputFull: SessionConfig = { maxSessions: 100, sessionTTL: 3600000 }; + + // All should be valid inputs + expect(() => SessionConfigSchema.parse(input)).not.toThrow(); + expect(() => SessionConfigSchema.parse(inputPartial)).not.toThrow(); + expect(() => SessionConfigSchema.parse(inputFull)).not.toThrow(); + }); + + it('should produce validated output type', () => { + const result: ValidatedSessionConfig = SessionConfigSchema.parse({}); + + // Output type guarantees all fields are present + expect(typeof result.maxSessions).toBe('number'); + expect(typeof result.sessionTTL).toBe('number'); + expect(result.maxSessions).toBeGreaterThan(0); + expect(result.sessionTTL).toBeGreaterThan(0); + }); + }); + + describe('Real-world Scenarios', () => { + it('should handle typical production config', () => { + const prodConfig = { + maxSessions: 1000, + sessionTTL: 7200000, // 2 hours + }; + + const result = SessionConfigSchema.parse(prodConfig); + expect(result).toEqual(prodConfig); + }); + + it('should handle development config with shorter TTL', () => { + const devConfig = { + maxSessions: 10, + sessionTTL: 300000, // 5 minutes + }; + + const result = SessionConfigSchema.parse(devConfig); + expect(result).toEqual(devConfig); + }); + }); +}); diff --git a/dexto/packages/core/src/session/schemas.ts b/dexto/packages/core/src/session/schemas.ts new file mode 100644 index 00000000..209058ea --- /dev/null +++ b/dexto/packages/core/src/session/schemas.ts @@ -0,0 +1,22 @@ +import { z } from 'zod'; + +export const SessionConfigSchema = z + .object({ + maxSessions: z + .number() + .int() + .positive() + .default(100) + .describe('Maximum number of concurrent sessions allowed, defaults to 100'), + sessionTTL: z + .number() + .int() + .positive() + .default(3600000) + .describe('Session time-to-live in milliseconds, defaults to 3600000ms (1 hour)'), + }) + .strict() + .describe('Session management configuration'); + +export type SessionConfig = z.input; +export type ValidatedSessionConfig = z.output; diff --git a/dexto/packages/core/src/session/session-manager.integration.test.ts b/dexto/packages/core/src/session/session-manager.integration.test.ts new file mode 100644 index 00000000..aa040bb9 --- /dev/null +++ b/dexto/packages/core/src/session/session-manager.integration.test.ts @@ -0,0 +1,219 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest'; +import { DextoAgent } from '../agent/DextoAgent.js'; +import type { AgentConfig } from '@core/agent/schemas.js'; +import type { SessionData } from './session-manager.js'; + +/** + * Full end-to-end integration tests for chat history preservation. + * Tests the complete flow from DextoAgent -> SessionManager -> ChatSession -> Storage + */ +describe('Session Integration: Chat History Preservation', () => { + let agent: DextoAgent; + + const testConfig: AgentConfig = { + systemPrompt: 'You are a helpful assistant.', + llm: { + provider: 'openai', + model: 'gpt-5-mini', + apiKey: 'test-key-123', + }, + mcpServers: {}, + sessions: { + maxSessions: 10, + sessionTTL: 100, // 100ms for fast testing + }, + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: false, + timeout: 120000, + }, + }; + + beforeEach(async () => { + agent = new DextoAgent(testConfig); + await agent.start(); + }); + + afterEach(async () => { + if (agent.isStarted()) { + await agent.stop(); + } + }); + + test('full integration: chat history survives session expiry through DextoAgent', async () => { + const sessionId = 'integration-test-session'; + + // Step 1: Create session through DextoAgent + const session = await agent.createSession(sessionId); + expect(session.id).toBe(sessionId); + + // Step 2: Simulate adding messages to the session + // In a real scenario, this would happen through agent.run() calls + // For testing, we'll access the underlying storage directly + const storage = agent.services.storageManager; + const messagesKey = `messages:${sessionId}`; + const chatHistory = [ + { role: 'user', content: 'What is 2+2?' }, + { role: 'assistant', content: '2+2 equals 4.' }, + { role: 'user', content: 'Thank you!' }, + { + role: 'assistant', + content: "You're welcome! Is there anything else I can help you with?", + }, + ]; + await storage.getDatabase().set(messagesKey, chatHistory); + + // Step 3: Verify session exists and has history + const activeSession = await agent.getSession(sessionId); + expect(activeSession).toBeDefined(); + expect(activeSession!.id).toBe(sessionId); + + const storedHistory = await storage.getDatabase().get(messagesKey); + expect(storedHistory).toEqual(chatHistory); + + // Step 4: Force session expiry by manipulating lastActivity timestamp + await new Promise((resolve) => setTimeout(resolve, 150)); // Wait > TTL + + const sessionKey = `session:${sessionId}`; + const sessionData = await storage.getDatabase().get(sessionKey); + if (sessionData) { + sessionData.lastActivity = Date.now() - 200; // Mark as expired + await storage.getDatabase().set(sessionKey, sessionData); + } + + // Access private method to manually trigger cleanup for testing session expiry behavior + const sessionManager = agent.sessionManager; + await (sessionManager as any).cleanupExpiredSessions(); + + // Step 5: Verify session is removed from memory but preserved in storage + const sessionsMap = (sessionManager as any).sessions; + expect(sessionsMap.has(sessionId)).toBe(false); + + // But storage should still have both session metadata and chat history + expect(await storage.getDatabase().get(sessionKey)).toBeDefined(); + expect(await storage.getDatabase().get(messagesKey)).toEqual(chatHistory); + + // Step 6: Access session again through DextoAgent - should restore seamlessly + const restoredSession = await agent.getSession(sessionId); + expect(restoredSession).toBeDefined(); + expect(restoredSession!.id).toBe(sessionId); + + // Session should be back in memory + expect(sessionsMap.has(sessionId)).toBe(true); + + // Chat history should still be intact + const restoredHistory = await storage.getDatabase().get(messagesKey); + expect(restoredHistory).toEqual(chatHistory); + + // Step 7: Verify we can continue the conversation + const newMessage = { role: 'user', content: 'One more question: what is 3+3?' }; + await storage.getDatabase().set(messagesKey, [...chatHistory, newMessage]); + + const finalHistory = await storage.getDatabase().get(messagesKey); + expect(finalHistory).toBeDefined(); + expect(finalHistory!).toHaveLength(5); + expect(finalHistory![4]).toEqual(newMessage); + }); + + test('full integration: explicit session deletion removes everything', async () => { + const sessionId = 'deletion-test-session'; + + // Create session and add history + await agent.createSession(sessionId); + + const storage = agent.services.storageManager; + const messagesKey = `messages:${sessionId}`; + const sessionKey = `session:${sessionId}`; + const history = [{ role: 'user', content: 'Hello!' }]; + + await storage.getDatabase().set(messagesKey, history); + + // Verify everything exists + expect(await agent.getSession(sessionId)).toBeDefined(); + expect(await storage.getDatabase().get(sessionKey)).toBeDefined(); + expect(await storage.getDatabase().get(messagesKey)).toEqual(history); + + // Delete session through DextoAgent + await agent.deleteSession(sessionId); + + // Everything should be gone including chat history + const deletedSession = await agent.getSession(sessionId); + expect(deletedSession).toBeUndefined(); + expect(await storage.getDatabase().get(sessionKey)).toBeUndefined(); + expect(await storage.getDatabase().get(messagesKey)).toBeUndefined(); + }); + + test('full integration: multiple concurrent sessions with independent histories', async () => { + const sessionIds = ['concurrent-1', 'concurrent-2', 'concurrent-3']; + const histories = sessionIds.map((_, index) => [ + { role: 'user', content: `Message from session ${index + 1}` }, + { role: 'assistant', content: `Response to session ${index + 1}` }, + ]); + + // Create multiple sessions with different histories + const storage = agent.services.storageManager; + for (let i = 0; i < sessionIds.length; i++) { + await agent.createSession(sessionIds[i]); + await storage.getDatabase().set(`messages:${sessionIds[i]}`, histories[i]); + } + + // Verify all sessions exist and have correct histories + for (let i = 0; i < sessionIds.length; i++) { + const sessionId = sessionIds[i]!; + const session = await agent.getSession(sessionId); + expect(session).toBeDefined(); + expect(session!.id).toBe(sessionId); + + const history = await storage.getDatabase().get(`messages:${sessionId}`); + expect(history).toEqual(histories[i]); + } + + // Force expiry and cleanup for all sessions + await new Promise((resolve) => setTimeout(resolve, 150)); + for (const sessionId of sessionIds) { + const sessionData = await storage + .getDatabase() + .get(`session:${sessionId}`); + if (sessionData) { + sessionData.lastActivity = Date.now() - 200; + await storage.getDatabase().set(`session:${sessionId}`, sessionData); + } + } + + const sessionManager = agent.sessionManager; + await (sessionManager as any).cleanupExpiredSessions(); + + // All should be removed from memory + const sessionsMap = (sessionManager as any).sessions; + sessionIds.forEach((id) => { + expect(sessionsMap.has(id)).toBe(false); + }); + + // But histories should be preserved in storage + for (let i = 0; i < sessionIds.length; i++) { + const history = await storage.getDatabase().get(`messages:${sessionIds[i]}`); + expect(history).toEqual(histories[i]); + } + + // Restore sessions one by one and verify independent operation + for (let i = 0; i < sessionIds.length; i++) { + const sessionId = sessionIds[i]!; + const restoredSession = await agent.getSession(sessionId); + expect(restoredSession).toBeDefined(); + expect(restoredSession!.id).toBe(sessionId); + + // Verify the session is back in memory + expect(sessionsMap.has(sessionId)).toBe(true); + + // Verify history is still intact and independent + const history = await storage.getDatabase().get(`messages:${sessionId}`); + expect(history).toEqual(histories[i]); + } + }); + + // Note: Activity-based expiry prevention test removed due to timing complexities + // The core functionality (chat history preservation) is thoroughly tested above +}); diff --git a/dexto/packages/core/src/session/session-manager.test.ts b/dexto/packages/core/src/session/session-manager.test.ts new file mode 100644 index 00000000..6f004549 --- /dev/null +++ b/dexto/packages/core/src/session/session-manager.test.ts @@ -0,0 +1,1201 @@ +import { describe, test, expect, vi, beforeEach, afterEach } from 'vitest'; +import { SessionManager } from './session-manager.js'; +import { ChatSession } from './chat-session.js'; +import { type ValidatedLLMConfig } from '@core/llm/schemas.js'; +import { LLMConfigSchema } from '@core/llm/schemas.js'; +import { StorageSchema } from '@core/storage/schemas.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { SessionErrorCode } from './error-codes.js'; +import { createMockLogger } from '@core/logger/v2/test-utils.js'; + +// Mock dependencies +vi.mock('./chat-session.js'); +vi.mock('../logger/index.js', () => ({ + logger: { + error: vi.fn(), + warn: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + }, +})); +vi.mock('crypto', () => ({ + randomUUID: vi.fn(() => 'mock-uuid-123'), +})); + +const MockChatSession = vi.mocked(ChatSession); + +describe('SessionManager', () => { + let sessionManager: SessionManager; + let mockServices: any; + let mockStorageManager: any; + let mockLLMConfig: ValidatedLLMConfig; + const mockLogger = createMockLogger(); + + const mockSessionData = { + id: 'test-session', + createdAt: new Date('2024-01-01T00:00:00Z').getTime(), + lastActivity: new Date('2024-01-01T01:00:00Z').getTime(), + messageCount: 5, + }; + + beforeEach(() => { + vi.resetAllMocks(); + + // Mock storage manager with proper getter structure + const mockCache = { + get: vi.fn().mockResolvedValue(null), + set: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(true), + list: vi.fn().mockResolvedValue([]), + clear: vi.fn().mockResolvedValue(undefined), + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + isConnected: vi.fn().mockReturnValue(true), + getBackendType: vi.fn().mockReturnValue('memory'), + }; + + const mockDatabase = { + get: vi.fn().mockResolvedValue(null), + set: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(true), + list: vi.fn().mockResolvedValue([]), + clear: vi.fn().mockResolvedValue(undefined), + append: vi.fn().mockResolvedValue(undefined), + getRange: vi.fn().mockResolvedValue([]), + getLength: vi.fn().mockResolvedValue(0), + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + isConnected: vi.fn().mockReturnValue(true), + getBackendType: vi.fn().mockReturnValue('memory'), + }; + + const mockBlobStore = { + store: vi.fn().mockResolvedValue({ id: 'test', uri: 'blob:test' }), + retrieve: vi.fn().mockResolvedValue({ data: '', metadata: {} }), + exists: vi.fn().mockResolvedValue(false), + delete: vi.fn().mockResolvedValue(undefined), + cleanup: vi.fn().mockResolvedValue(0), + getStats: vi.fn().mockResolvedValue({ count: 0, totalSize: 0, backendType: 'local' }), + listBlobs: vi.fn().mockResolvedValue([]), + getStoragePath: vi.fn().mockReturnValue(undefined), + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + isConnected: vi.fn().mockReturnValue(true), + getStoreType: vi.fn().mockReturnValue('local'), + }; + + mockStorageManager = { + getCache: vi.fn().mockReturnValue(mockCache), + getDatabase: vi.fn().mockReturnValue(mockDatabase), + getBlobStore: vi.fn().mockReturnValue(mockBlobStore), + disconnect: vi.fn().mockResolvedValue(undefined), + // Also expose direct references for test assertions + cache: mockCache, + database: mockDatabase, + blobStore: mockBlobStore, + }; + + // Mock services + mockServices = { + stateManager: { + getLLMConfig: vi.fn().mockReturnValue(mockLLMConfig), + updateLLM: vi.fn().mockReturnValue({ isValid: true, errors: [], warnings: [] }), + }, + systemPromptManager: { + getSystemPrompt: vi.fn().mockReturnValue('System prompt'), + }, + mcpManager: { + getAllTools: vi.fn().mockResolvedValue({}), + }, + agentEventBus: { + emit: vi.fn(), + }, + storageManager: mockStorageManager, + resourceManager: { + getBlobStore: vi.fn(), + readResource: vi.fn(), + listResources: vi.fn(), + }, + toolManager: { + getAllTools: vi.fn().mockReturnValue([]), + }, + pluginManager: { + executePlugins: vi.fn().mockImplementation(async (_point, payload) => payload), + cleanup: vi.fn(), + }, + }; + + // Parse LLM config now that mocks are set up + mockLLMConfig = LLMConfigSchema.parse({ + provider: 'openai', + model: 'gpt-5', + apiKey: 'test-key', + maxIterations: 50, + maxInputTokens: 128000, + }); + + // Create SessionManager instance + sessionManager = new SessionManager( + mockServices, + { + maxSessions: 10, + sessionTTL: 1800000, // 30 minutes + }, + mockLogger + ); + + // Mock ChatSession constructor and methods + MockChatSession.mockImplementation((services, id, _logger) => { + const mockSession = { + id, + init: vi.fn().mockResolvedValue(undefined), + run: vi.fn().mockResolvedValue('Mock response'), + reset: vi.fn().mockResolvedValue(undefined), + dispose: vi.fn(), + cleanup: vi.fn().mockImplementation(async () => { + // Simulate the new cleanup behavior - only call dispose, not reset + mockSession.dispose(); + }), + switchLLM: vi.fn().mockResolvedValue(undefined), + getHistory: vi.fn().mockResolvedValue([]), + getContextManager: vi.fn(), + getLLMService: vi.fn(), + eventBus: { + emit: vi.fn(), + on: vi.fn(), + off: vi.fn(), + }, + }; + return mockSession as any; + }); + }); + + afterEach(async () => { + if (sessionManager) { + await sessionManager.cleanup(); + } + }); + + describe('Session Lifecycle Management', () => { + test('should support flexible initialization options', () => { + const defaultManager = new SessionManager(mockServices, {}, mockLogger); + const customManager = new SessionManager( + mockServices, + { + maxSessions: 50, + sessionTTL: 7200000, // 2 hours + }, + mockLogger + ); + + expect(defaultManager).toBeDefined(); + expect(customManager).toBeDefined(); + }); + + test('should initialize storage layer on first use', async () => { + await sessionManager.init(); + + // Verify database.list is called to find existing sessions + expect(mockStorageManager.database.list).toHaveBeenCalledWith('session:'); + }); + + test('should prevent duplicate initialization', async () => { + await sessionManager.init(); + await sessionManager.init(); // Second call + + // Should only call database.list once since it's already initialized + expect(mockStorageManager.database.list).toHaveBeenCalledTimes(1); + }); + + test('should restore valid sessions from persistent storage on startup', async () => { + const existingSessionIds = ['session-1', 'session-2']; + const existingSessionKeys = ['session:session-1', 'session:session-2']; + const validMetadata = { + ...mockSessionData, + lastActivity: new Date().getTime(), // Recent activity + }; + + mockStorageManager.database.list.mockResolvedValue(existingSessionKeys); + mockStorageManager.database.get.mockResolvedValue(validMetadata); + + await sessionManager.init(); + + expect(mockStorageManager.database.list).toHaveBeenCalledWith('session:'); + expect(mockStorageManager.database.get).toHaveBeenCalledTimes( + existingSessionIds.length + ); + }); + + test('should clean up expired sessions during startup restoration', async () => { + const existingSessionKeys = ['session:expired-session']; + const expiredMetadata = { + ...mockSessionData, + lastActivity: new Date(Date.now() - 7200000).getTime(), // 2 hours ago + }; + + mockStorageManager.database.list.mockResolvedValue(existingSessionKeys); + mockStorageManager.database.get.mockResolvedValue(expiredMetadata); + + await sessionManager.init(); + + expect(mockStorageManager.database.delete).toHaveBeenCalledWith( + 'session:expired-session' + ); + }); + }); + + describe('Session Creation and Identity Management', () => { + beforeEach(async () => { + await sessionManager.init(); + }); + + test('should create sessions with auto-generated IDs when none provided', async () => { + const session = await sessionManager.createSession(); + + expect(session).toBeDefined(); + expect(session.id).toBe('mock-uuid-123'); + expect(MockChatSession).toHaveBeenCalledWith( + expect.objectContaining({ ...mockServices, sessionManager: expect.anything() }), + 'mock-uuid-123', + mockLogger + ); + }); + + test('should create sessions with custom IDs when provided', async () => { + const customId = 'custom-session-id'; + const session = await sessionManager.createSession(customId); + + expect(session.id).toBe(customId); + expect(MockChatSession).toHaveBeenCalledWith( + expect.objectContaining({ ...mockServices, sessionManager: expect.anything() }), + customId, + mockLogger + ); + }); + + test('should return existing session instance for duplicate creation requests', async () => { + const sessionId = 'existing-session'; + + const session1 = await sessionManager.createSession(sessionId); + const session2 = await sessionManager.createSession(sessionId); + + // Both should have the same ID (testing the actual behavior) + expect(session1.id).toBe(session2.id); + expect(session1.id).toBe(sessionId); + expect(session2.id).toBe(sessionId); + }); + + test('should restore sessions from storage when not in memory', async () => { + const sessionId = 'stored-session'; + + mockStorageManager.database.get.mockResolvedValue(mockSessionData); + + const session = await sessionManager.createSession(sessionId); + + expect(session.id).toBe(sessionId); + expect(mockStorageManager.database.get).toHaveBeenCalledWith(`session:${sessionId}`); + }); + }); + + describe('Session Limits and Resource Management', () => { + test('should enforce maximum session limits', async () => { + const maxSessions = 2; + const limitedManager = new SessionManager(mockServices, { maxSessions }, mockLogger); + await limitedManager.init(); + + // Mock that we already have max sessions + mockStorageManager.database.list.mockResolvedValue([ + 'session:session-1', + 'session:session-2', + ]); + + await expect(limitedManager.createSession()).rejects.toMatchObject({ + code: SessionErrorCode.SESSION_MAX_SESSIONS_EXCEEDED, + scope: ErrorScope.SESSION, + type: ErrorType.USER, + }); + }); + + test('should clean up expired sessions before enforcing limits', async () => { + await sessionManager.init(); + + // Mock expired session that should be cleaned up + mockStorageManager.database.list.mockResolvedValue(['session:expired-session']); + + const session = await sessionManager.createSession(); + + expect(session).toBeDefined(); + // Should not throw max sessions error because expired session was cleaned up + }); + + test('should provide session statistics for monitoring', async () => { + await sessionManager.init(); + + const activeSessionIds = ['session:session-1', 'session:session-2']; + mockStorageManager.database.list.mockResolvedValue(activeSessionIds); + + // Create one session in memory + await sessionManager.createSession('session-1'); + + const stats = await sessionManager.getSessionStats(); + + expect(stats).toEqual({ + totalSessions: 2, + inMemorySessions: 1, + maxSessions: 10, + sessionTTL: 1800000, + }); + }); + }); + + describe('Session Retrieval and Access Patterns', () => { + beforeEach(async () => { + await sessionManager.init(); + }); + + test('should retrieve existing sessions from memory efficiently', async () => { + const sessionId = 'test-session'; + + // Create session first + await sessionManager.createSession(sessionId); + + // Get session + const session = await sessionManager.getSession(sessionId); + + expect(session).toBeDefined(); + expect(session!.id).toBe(sessionId); + }); + + test('should restore sessions from storage when not in memory', async () => { + const sessionId = 'stored-session'; + + mockStorageManager.database.get.mockResolvedValue(mockSessionData); + + const session = await sessionManager.getSession(sessionId); + + expect(session).toBeDefined(); + expect(session!.id).toBe(sessionId); + expect(mockStorageManager.database.get).toHaveBeenCalledWith(`session:${sessionId}`); + }); + + test('should return undefined for non-existent sessions', async () => { + const result = await sessionManager.getSession('non-existent'); + expect(result).toBeUndefined(); + }); + + test('should update session activity timestamps on access', async () => { + const sessionId = 'test-session'; + + // Create the session first so it exists in storage + await sessionManager.createSession(sessionId); + + // Reset the mock to clear the creation calls + mockStorageManager.database.set.mockClear(); + mockStorageManager.database.get.mockResolvedValue(mockSessionData); + + // Call incrementMessageCount which should update activity + await sessionManager.incrementMessageCount(sessionId); + + expect(mockStorageManager.database.set).toHaveBeenCalledWith( + `session:${sessionId}`, + expect.objectContaining({ + lastActivity: expect.any(Number), + }) + ); + }); + }); + + describe('Session Metadata and Activity Tracking', () => { + beforeEach(async () => { + await sessionManager.init(); + }); + + test('should track and persist session metadata', async () => { + const session = await sessionManager.createSession(); + + expect(mockStorageManager.database.set).toHaveBeenCalledWith( + `session:${session.id}`, + expect.objectContaining({ + createdAt: expect.any(Number), + lastActivity: expect.any(Number), + messageCount: 0, + }) + ); + }); + + test('should increment message counts and update activity', async () => { + const sessionId = 'test-session'; + + mockStorageManager.database.get.mockResolvedValue({ ...mockSessionData }); + + await sessionManager.incrementMessageCount(sessionId); + + expect(mockStorageManager.database.set).toHaveBeenCalledWith( + `session:${sessionId}`, + expect.objectContaining({ + messageCount: mockSessionData.messageCount + 1, + lastActivity: expect.any(Number), + }) + ); + }); + + test('should provide access to session metadata', async () => { + const sessionId = 'test-session'; + + mockStorageManager.database.get.mockResolvedValue(mockSessionData); + + const metadata = await sessionManager.getSessionMetadata(sessionId); + + expect(metadata).toEqual({ + createdAt: mockSessionData.createdAt, + lastActivity: mockSessionData.lastActivity, + messageCount: mockSessionData.messageCount, + }); + expect(mockStorageManager.database.get).toHaveBeenCalledWith(`session:${sessionId}`); + }); + + test('should provide access to global configuration', async () => { + const config = sessionManager.getConfig(); + + expect(config).toEqual({ + maxSessions: 10, + sessionTTL: 1800000, + }); + }); + + test('should list all active sessions', async () => { + const activeSessionKeys = [ + 'session:session-1', + 'session:session-2', + 'session:session-3', + ]; + const expectedSessionIds = ['session-1', 'session-2', 'session-3']; + mockStorageManager.database.list.mockResolvedValue(activeSessionKeys); + + const sessions = await sessionManager.listSessions(); + + expect(sessions).toEqual(expectedSessionIds); + expect(mockStorageManager.database.list).toHaveBeenCalledWith('session:'); + }); + }); + + describe('Session Termination and Cleanup', () => { + beforeEach(async () => { + await sessionManager.init(); + }); + + test('should properly end sessions and clean up resources', async () => { + const sessionId = 'test-session'; + + // Create session first + const session = await sessionManager.createSession(sessionId); + + // Delete session + await sessionManager.deleteSession(sessionId); + + expect(session.cleanup).toHaveBeenCalled(); + expect(mockStorageManager.database.delete).toHaveBeenCalledWith(`session:${sessionId}`); + }); + + test('should handle deleting non-existent sessions gracefully', async () => { + await expect(sessionManager.deleteSession('non-existent')).resolves.not.toThrow(); + expect(mockStorageManager.database.delete).toHaveBeenCalledWith('session:non-existent'); + }); + + test('should cleanup all sessions during shutdown', async () => { + // Create multiple sessions + const sessions = [ + await sessionManager.createSession('session-1'), + await sessionManager.createSession('session-2'), + await sessionManager.createSession('session-3'), + ]; + + // Cleanup all sessions + await sessionManager.cleanup(); + + // Verify all sessions were cleaned up + for (const session of sessions) { + expect(session.cleanup).toHaveBeenCalled(); + } + }); + + test('should handle cleanup errors gracefully', async () => { + const sessionId = 'error-session'; + + // Create session + const session = await sessionManager.createSession(sessionId); + + // Mock error during cleanup (SessionManager.cleanup -> endSession -> session.cleanup) + (session.cleanup as any).mockRejectedValue(new Error('Cleanup error')); + + await expect(sessionManager.cleanup()).resolves.not.toThrow(); + }); + }); + + describe('LLM Configuration Management Across Sessions', () => { + beforeEach(async () => { + await sessionManager.init(); + }); + + test('should switch LLM for specific session', async () => { + const sessionId = 'test-session'; + const newLLMConfig: ValidatedLLMConfig = { + ...mockLLMConfig, + provider: 'anthropic', + model: 'claude-4-opus-20250514', + }; + + // Create session first + await sessionManager.createSession(sessionId); + + const result = await sessionManager.switchLLMForSpecificSession( + newLLMConfig, + sessionId + ); + + expect(result.message).toContain( + `Successfully switched to anthropic/claude-4-opus-20250514 for session ${sessionId}` + ); + expect(result.warnings).toEqual([]); + }); + + test('should handle LLM switch for non-existent session', async () => { + const newLLMConfig: ValidatedLLMConfig = { + ...mockLLMConfig, + provider: 'anthropic', + model: 'claude-4-opus-20250514', + }; + + await expect( + sessionManager.switchLLMForSpecificSession(newLLMConfig, 'non-existent') + ).rejects.toMatchObject({ + code: SessionErrorCode.SESSION_NOT_FOUND, + scope: ErrorScope.SESSION, + type: ErrorType.NOT_FOUND, + }); + }); + + test('should handle partial failures when switching LLM for all sessions', async () => { + const sessionIds = ['session:session-1', 'session:session-2']; + const newLLMConfig: ValidatedLLMConfig = { + ...mockLLMConfig, + provider: 'anthropic', + model: 'claude-4-opus-20250514', + }; + + mockStorageManager.database.list.mockResolvedValue(sessionIds); + + // Mock runtime failure for one session (e.g., session corruption, disposal, etc.) + mockServices.stateManager.updateLLM.mockImplementation( + (config: any, sessionId: string) => { + if (sessionId === 'session-2') { + throw new Error('Session state corruption detected'); + } + // Normal case - returns void (no return needed) + } + ); + + // Create sessions + for (const sessionKey of sessionIds) { + const sessionId = sessionKey.replace('session:', ''); + await sessionManager.createSession(sessionId); + } + + const result = await sessionManager.switchLLMForAllSessions(newLLMConfig); + + expect(result.message).toContain('1 sessions failed'); + expect(result.warnings).toContain('Failed to switch LLM for sessions: session-2'); + }); + }); + + describe('Error Handling and Resilience', () => { + test('should handle storage initialization failures', async () => { + mockStorageManager.database.list.mockRejectedValue( + new Error('Storage initialization failed') + ); + + // init() should not throw - it catches and logs errors + await expect(sessionManager.init()).resolves.toBeUndefined(); + }); + + test('should handle storage operation failures gracefully', async () => { + await sessionManager.init(); + + mockStorageManager.database.set.mockRejectedValue(new Error('Storage write failed')); + + // Should still create session in memory despite storage failure + await expect(sessionManager.createSession()).rejects.toThrow('Storage write failed'); + }); + + test('should handle session restoration errors during startup', async () => { + mockStorageManager.database.list.mockResolvedValue(['session:session-1']); + mockStorageManager.database.get.mockRejectedValue(new Error('Storage read failed')); + + // Should not throw during initialization + await expect(sessionManager.init()).resolves.not.toThrow(); + }); + + test('should handle cleanup errors during expired session removal', async () => { + await sessionManager.init(); + + const sessionId = 'test-session'; + const session = await sessionManager.createSession(sessionId); + + // Mock cleanup error + (session.reset as any).mockRejectedValue(new Error('Reset failed')); + + // Mock that session is expired + mockStorageManager.database.list.mockResolvedValue([]); + + // Should handle error gracefully during cleanup + await expect(sessionManager.createSession('new-session')).resolves.toBeDefined(); + }); + }); + + describe('Concurrency and Edge Cases', () => { + beforeEach(async () => { + await sessionManager.init(); + }); + + test('should handle concurrent session creation requests safely', async () => { + const sessionId = 'concurrent-session'; + + // Create multiple promises for the same session + const promises = [ + sessionManager.createSession(sessionId), + sessionManager.createSession(sessionId), + sessionManager.createSession(sessionId), + ]; + + const sessions = await Promise.all(promises); + + // All should have the same session ID + expect(sessions[0]?.id).toBe(sessions[1]?.id); + expect(sessions[1]?.id).toBe(sessions[2]?.id); + expect(sessions[0]?.id).toBe(sessionId); + + // Verify all sessions are properly created + expect(sessions[0]).toBeDefined(); + expect(sessions[1]).toBeDefined(); + expect(sessions[2]).toBeDefined(); + }); + + test('should prevent race conditions when creating multiple different sessions concurrently', async () => { + // Set a low session limit to test the race condition prevention + const limitedSessionManager = new SessionManager( + mockServices, + { + maxSessions: 2, + sessionTTL: 1800000, // 30 minutes + }, + mockLogger + ); + await limitedSessionManager.init(); + + // Create 2 sessions sequentially first to reach the limit + await limitedSessionManager.createSession('existing-1'); + await limitedSessionManager.createSession('existing-2'); + + // Mock the database.list to return the existing sessions + mockStorageManager.database.list.mockResolvedValue([ + 'session:existing-1', + 'session:existing-2', + ]); + + // Now try to create 4 more sessions concurrently - all should fail + const promises = [ + limitedSessionManager.createSession('session-1'), + limitedSessionManager.createSession('session-2'), + limitedSessionManager.createSession('session-3'), + limitedSessionManager.createSession('session-4'), + ]; + + const results = await Promise.allSettled(promises); + + // Count successful and failed creations + const successes = results.filter((result) => result.status === 'fulfilled'); + const failures = results.filter((result) => result.status === 'rejected'); + + // All should fail due to the limit since we're at capacity + expect(failures.length).toBe(4); + expect(successes.length).toBe(0); + + // All failures should be due to session limit + failures.forEach((failure) => { + const err = (failure as PromiseRejectedResult).reason as any; + expect(err.code).toBe(SessionErrorCode.SESSION_MAX_SESSIONS_EXCEEDED); + expect(err.scope).toBe(ErrorScope.SESSION); + expect(err.type).toBe(ErrorType.USER); + }); + + // Clean up + await limitedSessionManager.cleanup(); + }); + + test('should handle legacy session metadata without TTL fields', async () => { + const sessionId = 'legacy-session'; + const legacyMetadata = { + createdAt: new Date().getTime(), + lastActivity: new Date().getTime(), + messageCount: 0, + // Missing maxSessions and sessionTTL + }; + + mockStorageManager.database.get.mockResolvedValue(legacyMetadata); + + const session = await sessionManager.getSession(sessionId); + + expect(session).toBeDefined(); + expect(session!.id).toBe(sessionId); + }); + + test('should handle empty session lists gracefully', async () => { + mockStorageManager.database.list.mockResolvedValue([]); + + const sessions = await sessionManager.listSessions(); + + expect(sessions).toEqual([]); + }); + + test('should continue operating when storage is temporarily unavailable', async () => { + mockStorageManager.database.set.mockRejectedValue(new Error('Storage unavailable')); + + // Should throw error since storage is required for session persistence + await expect(sessionManager.createSession()).rejects.toThrow('Storage unavailable'); + }); + + test('should clean up expired in-memory sessions automatically', async () => { + const sessionId = 'test-session'; + + // Create session + await sessionManager.createSession(sessionId); + + // Mock that session is no longer in storage (expired) + mockStorageManager.database.list.mockResolvedValue([]); + + // Trigger cleanup by creating another session + await sessionManager.createSession('new-session'); + + // The expired session should have been cleaned up from memory + // This is tested indirectly through the cleanup process + }); + }); + + describe('Periodic Cleanup', () => { + test('should start periodic cleanup timer during initialization', async () => { + const setIntervalSpy = vi.spyOn(global, 'setInterval'); + + await sessionManager.init(); + + expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), expect.any(Number)); + + // Verify the interval is calculated correctly (min of TTL/4 or 15 minutes) + // Test setup uses 30 minutes TTL, so TTL/4 = 7.5 minutes = 450000ms + const expectedInterval = Math.min(1800000 / 4, 15 * 60 * 1000); // 7.5 minutes since TTL/4 < 15 minutes + expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), expectedInterval); + + setIntervalSpy.mockRestore(); + }); + + test('should stop periodic cleanup timer during cleanup', async () => { + const clearIntervalSpy = vi.spyOn(global, 'clearInterval'); + + await sessionManager.init(); + await sessionManager.cleanup(); + + expect(clearIntervalSpy).toHaveBeenCalled(); + clearIntervalSpy.mockRestore(); + }); + + test('should handle periodic cleanup errors gracefully', async () => { + const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const setIntervalSpy = vi.spyOn(global, 'setInterval'); + mockStorageManager.database.list.mockRejectedValue(new Error('Storage error')); + + await sessionManager.init(); + + // Access the interval function and call it directly to test error handling + const setIntervalCall = setIntervalSpy.mock.calls[0]; + const cleanupFunction = setIntervalCall?.[0] as () => Promise; + + // This should not throw + await expect(cleanupFunction()).resolves.toBeUndefined(); + + consoleSpy.mockRestore(); + setIntervalSpy.mockRestore(); + }); + + test('should use correct cleanup interval based on session TTL', async () => { + // Create SessionManager with different TTL + const shortTTLSessionManager = new SessionManager( + mockServices, + { + sessionTTL: 60000, // 1 minute + }, + mockLogger + ); + + const setIntervalSpy = vi.spyOn(global, 'setInterval'); + + await shortTTLSessionManager.init(); + + // Should use TTL/4 (15 seconds) since it's less than 15 minutes + const expectedInterval = Math.min(60000 / 4, 15 * 60 * 1000); + expect(expectedInterval).toBe(15000); // 15 seconds + expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), 15000); + + setIntervalSpy.mockRestore(); + await shortTTLSessionManager.cleanup(); + }); + }); + + describe('Chat History Preservation', () => { + beforeEach(async () => { + await sessionManager.init(); + }); + + test('should preserve chat history when session expires from memory', async () => { + const sessionId = 'test-session-history'; + + // Create session and simulate it has messages + const session = await sessionManager.createSession(sessionId); + expect(session).toBeDefined(); + + // Mock that session has been inactive for longer than TTL + const sessionKey = `session:${sessionId}`; + const expiredSessionData = { + id: sessionId, + createdAt: Date.now() - 7200000, // 2 hours ago + lastActivity: Date.now() - 7200000, // 2 hours ago (expired) + messageCount: 5, + }; + mockStorageManager.database.get.mockResolvedValue(expiredSessionData); + + // Trigger cleanup - should remove from memory but preserve storage + await sessionManager['cleanupExpiredSessions'](); + + // Session should be removed from memory + expect(sessionManager['sessions'].has(sessionId)).toBe(false); + + // But session should still exist in storage (not deleted) + expect(mockStorageManager.database.delete).not.toHaveBeenCalledWith(sessionKey); + + // Chat history should still be accessible through DatabaseHistoryProvider + // (The actual history is stored separately from session metadata) + }); + + test('should restore session from storage with chat history intact', async () => { + const sessionId = 'restored-session'; + + // Mock that session exists in storage but not in memory + const storedSessionData = { + id: sessionId, + createdAt: Date.now() - 3600000, // 1 hour ago + lastActivity: Date.now() - 1800000, // 30 minutes ago + messageCount: 10, + }; + mockStorageManager.database.get.mockResolvedValue(storedSessionData); + + // Get session - should restore from storage + const restoredSession = await sessionManager.getSession(sessionId); + + expect(restoredSession).toBeDefined(); + expect(restoredSession!.id).toBe(sessionId); + expect(MockChatSession).toHaveBeenCalledWith( + expect.objectContaining({ ...mockServices, sessionManager: expect.anything() }), + sessionId, + mockLogger + ); + + // Session should now be in memory + expect(sessionManager['sessions'].has(sessionId)).toBe(true); + }); + + test('should only call dispose() during cleanup, not reset()', async () => { + const sessionId = 'memory-only-cleanup'; + const session = await sessionManager.createSession(sessionId); + + // Spy on session methods + const disposeSpy = vi.spyOn(session, 'dispose'); + const resetSpy = vi.spyOn(session, 'reset'); + + // Manually call cleanup (simulating what happens during expiry) + await session.cleanup(); + + // Should only dispose memory resources, NOT reset conversation + expect(disposeSpy).toHaveBeenCalled(); + expect(resetSpy).not.toHaveBeenCalled(); + }); + + test('should preserve session metadata and history after memory cleanup', async () => { + const sessionId = 'persistent-session'; + + // Create session + await sessionManager.createSession(sessionId); + + // Mock expired session data + const expiredSessionData = { + id: sessionId, + createdAt: Date.now() - 7200000, + lastActivity: Date.now() - 7200000, // Expired + messageCount: 15, + }; + mockStorageManager.database.get.mockResolvedValue(expiredSessionData); + + // Trigger cleanup + await sessionManager['cleanupExpiredSessions'](); + + // Session metadata should still exist in storage + expect(mockStorageManager.database.delete).not.toHaveBeenCalledWith( + `session:${sessionId}` + ); + + // Session should be able to be restored later + const restoredSession = await sessionManager.getSession(sessionId); + expect(restoredSession).toBeDefined(); + expect(restoredSession!.id).toBe(sessionId); + }); + + test('should delete conversation history and session metadata on explicit deletion', async () => { + const sessionId = 'explicit-delete-test'; + const session = await sessionManager.createSession(sessionId); + + // Explicit deletion should remove everything including conversation history + await sessionManager.deleteSession(sessionId); + + // Should call cleanup to dispose memory resources + expect(session.cleanup).toHaveBeenCalled(); + + // Should remove session metadata from storage completely + expect(mockStorageManager.database.delete).toHaveBeenCalledWith(`session:${sessionId}`); + expect(mockStorageManager.cache.delete).toHaveBeenCalledWith(`session:${sessionId}`); + + // Should delete conversation messages directly from storage + expect(mockStorageManager.database.delete).toHaveBeenCalledWith( + `messages:${sessionId}` + ); + }); + + test('should handle multiple expired sessions without affecting storage', async () => { + const sessionIds = ['expired-1', 'expired-2', 'expired-3']; + + // Create multiple sessions + for (const sessionId of sessionIds) { + await sessionManager.createSession(sessionId); + } + + // Mock all as expired + mockStorageManager.database.get.mockImplementation((key: string) => { + const sessionId = key.replace('session:', ''); + if (sessionIds.includes(sessionId)) { + return Promise.resolve({ + id: sessionId, + createdAt: Date.now() - 7200000, + lastActivity: Date.now() - 7200000, // All expired + messageCount: 5, + }); + } + return Promise.resolve(null); + }); + + // Trigger cleanup + await sessionManager['cleanupExpiredSessions'](); + + // All sessions should be removed from memory + sessionIds.forEach((sessionId) => { + expect(sessionManager['sessions'].has(sessionId)).toBe(false); + }); + + // But none should be deleted from storage + sessionIds.forEach((sessionId) => { + expect(mockStorageManager.database.delete).not.toHaveBeenCalledWith( + `session:${sessionId}` + ); + }); + }); + }); + + describe('End-to-End Chat History Preservation', () => { + let realStorageBackends: any; + let realSessionManager: SessionManager; + + beforeEach(async () => { + // Create real storage manager for end-to-end testing + const { createStorageManager } = await import('../storage/index.js'); + + const storageConfig = StorageSchema.parse({ + cache: { type: 'in-memory' as const }, + database: { type: 'in-memory' as const }, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }); + + realStorageBackends = await createStorageManager(storageConfig, mockLogger); + + // Create SessionManager with real storage and short TTL for faster testing + realSessionManager = new SessionManager( + { + ...mockServices, + storageManager: realStorageBackends, + }, + { + maxSessions: 10, + sessionTTL: 100, // 100ms for fast testing + }, + mockLogger + ); + + await realSessionManager.init(); + }); + + afterEach(async () => { + if (realSessionManager) { + await realSessionManager.cleanup(); + } + if (realStorageBackends) { + await realStorageBackends.database.disconnect(); + await realStorageBackends.cache.disconnect(); + } + }); + + test('end-to-end: chat history survives session expiry and is restored on access', async () => { + const sessionId = 'e2e-test-session'; + + // Step 1: Create session and simulate adding chat history + const originalSession = await realSessionManager.createSession(sessionId); + expect(originalSession).toBeDefined(); + + // Simulate chat history by storing messages directly (since we're mocking ChatSession) + const messagesKey = `messages:${sessionId}`; + const mockChatHistory = [ + { role: 'user', content: 'Hello!' }, + { role: 'assistant', content: 'Hi there!' }, + { role: 'user', content: 'How are you?' }, + { role: 'assistant', content: 'I am doing well, thank you!' }, + ]; + await realStorageBackends.database.set(messagesKey, mockChatHistory); + + // Verify session exists in memory + expect(realSessionManager['sessions'].has(sessionId)).toBe(true); + + // Step 2: Wait for session to expire, then trigger cleanup + await new Promise((resolve) => setTimeout(resolve, 150)); // Wait > TTL (100ms) + + // Update session metadata to mark it as expired + const sessionKey = `session:${sessionId}`; + const sessionData = await realStorageBackends.database.get(sessionKey); + if (sessionData) { + sessionData.lastActivity = Date.now() - 200; // Mark as expired + await realStorageBackends.database.set(sessionKey, sessionData); + } + + // Trigger cleanup manually (simulating periodic cleanup) + await realSessionManager['cleanupExpiredSessions'](); + + // Step 3: Verify session removed from memory but preserved in storage + expect(realSessionManager['sessions'].has(sessionId)).toBe(false); + + // Session metadata should still exist + const preservedSessionData = await realStorageBackends.database.get(sessionKey); + expect(preservedSessionData).toBeDefined(); + + // Chat history should still exist + const preservedHistory = await realStorageBackends.database.get(messagesKey); + expect(preservedHistory).toEqual(mockChatHistory); + + // Step 4: Access session again - should restore from storage + const restoredSession = await realSessionManager.getSession(sessionId); + expect(restoredSession).toBeDefined(); + expect(restoredSession!.id).toBe(sessionId); + + // Session should be back in memory + expect(realSessionManager['sessions'].has(sessionId)).toBe(true); + + // Chat history should still be accessible + const finalHistory = await realStorageBackends.database.get(messagesKey); + expect(finalHistory).toEqual(mockChatHistory); + + // Step 5: Verify new messages can be added to restored session + await realStorageBackends.database.set(messagesKey, [ + ...mockChatHistory, + { role: 'user', content: 'Still here!' }, + ]); + + const updatedHistory = await realStorageBackends.database.get(messagesKey); + expect(updatedHistory).toHaveLength(5); + expect(updatedHistory[4]).toEqual({ role: 'user', content: 'Still here!' }); + }); + + test('end-to-end: explicit deletion removes everything permanently', async () => { + const sessionId = 'e2e-delete-test'; + + // Create session with chat history + await realSessionManager.createSession(sessionId); + + const messagesKey = `messages:${sessionId}`; + const sessionKey = `session:${sessionId}`; + const mockHistory = [{ role: 'user', content: 'Test message' }]; + await realStorageBackends.database.set(messagesKey, mockHistory); + + // Verify everything exists + expect(await realStorageBackends.database.get(sessionKey)).toBeDefined(); + expect(await realStorageBackends.database.get(messagesKey)).toEqual(mockHistory); + + // Explicitly delete session + await realSessionManager.deleteSession(sessionId); + + // Everything should be gone + expect(realSessionManager['sessions'].has(sessionId)).toBe(false); + expect(await realStorageBackends.database.get(sessionKey)).toBeUndefined(); + + // Note: Chat history is also deleted via session.reset() which calls + // ContextManager's resetConversation() method, but since we're mocking + // ChatSession, we only test session metadata deletion here + }); + + test('end-to-end: multiple sessions can expire and restore independently', async () => { + const sessionIds = ['multi-1', 'multi-2', 'multi-3']; + const histories = sessionIds.map((id, index) => [ + { role: 'user', content: `Hello from session ${index + 1}` }, + ]); + + // Create multiple sessions with different histories + for (let i = 0; i < sessionIds.length; i++) { + await realSessionManager.createSession(sessionIds[i]); + await realStorageBackends.database.set(`messages:${sessionIds[i]}`, histories[i]); + } + + // Mark all as expired and cleanup + await new Promise((resolve) => setTimeout(resolve, 150)); + for (const sessionId of sessionIds) { + const sessionData = await realStorageBackends.database.get(`session:${sessionId}`); + if (sessionData) { + sessionData.lastActivity = Date.now() - 200; + await realStorageBackends.database.set(`session:${sessionId}`, sessionData); + } + } + + await realSessionManager['cleanupExpiredSessions'](); + + // All should be removed from memory + sessionIds.forEach((id) => { + expect(realSessionManager['sessions'].has(id)).toBe(false); + }); + + // Restore sessions one by one and verify independent histories + for (let i = 0; i < sessionIds.length; i++) { + const sessionId = sessionIds[i]!; + const restoredSession = await realSessionManager.getSession(sessionId); + expect(restoredSession).toBeDefined(); + expect(restoredSession!.id).toBe(sessionId); + + const history = await realStorageBackends.database.get(`messages:${sessionId}`); + expect(history).toEqual(histories[i]); + } + + // All should be back in memory + sessionIds.forEach((id) => { + expect(realSessionManager['sessions'].has(id)).toBe(true); + }); + }); + }); +}); diff --git a/dexto/packages/core/src/session/session-manager.ts b/dexto/packages/core/src/session/session-manager.ts new file mode 100644 index 00000000..def804d6 --- /dev/null +++ b/dexto/packages/core/src/session/session-manager.ts @@ -0,0 +1,813 @@ +import { randomUUID } from 'crypto'; +import { ChatSession } from './chat-session.js'; +import { SystemPromptManager } from '../systemPrompt/manager.js'; +import { ToolManager } from '../tools/tool-manager.js'; +import { AgentEventBus } from '../events/index.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import type { AgentStateManager } from '../agent/state-manager.js'; +import type { ValidatedLLMConfig } from '@core/llm/schemas.js'; +import type { StorageManager } from '../storage/index.js'; +import type { PluginManager } from '../plugins/manager.js'; +import { SessionError } from './errors.js'; +import type { TokenUsage } from '../llm/types.js'; + +/** + * Session-level token usage totals (accumulated across all messages). + * All fields required since we track cumulative totals (defaulting to 0). + */ +export type SessionTokenUsage = Required; + +export interface SessionMetadata { + createdAt: number; + lastActivity: number; + messageCount: number; + title?: string; + tokenUsage?: SessionTokenUsage; + estimatedCost?: number; +} + +export interface SessionManagerConfig { + maxSessions?: number; + sessionTTL?: number; +} + +export interface SessionData { + id: string; + userId?: string; + createdAt: number; + lastActivity: number; + messageCount: number; + metadata?: Record; + tokenUsage?: SessionTokenUsage; + estimatedCost?: number; +} + +/** + * Manages multiple chat sessions within a Dexto agent. + * + * The SessionManager is responsible for: + * - Creating and managing multiple isolated chat sessions + * - Enforcing session limits and TTL policies + * - Cleaning up expired sessions + * - Providing session lifecycle management + * - Persisting session data using the simplified storage backends + * + * TODO (Telemetry): Add OpenTelemetry metrics collection later if needed + * - Active session gauges (current count) + * - Session creation/deletion counters + * - Session duration histograms + * - Messages per session histograms + */ +export class SessionManager { + private sessions: Map = new Map(); + private readonly maxSessions: number; + private readonly sessionTTL: number; + private initialized = false; + private cleanupInterval?: NodeJS.Timeout; + private initializationPromise!: Promise; + // Add a Map to track ongoing session creation operations to prevent race conditions + private readonly pendingCreations = new Map>(); + // Per-session mutex for token usage updates to prevent lost updates from concurrent calls + private readonly tokenUsageLocks = new Map>(); + private logger: IDextoLogger; + + constructor( + private services: { + stateManager: AgentStateManager; + systemPromptManager: SystemPromptManager; + toolManager: ToolManager; + agentEventBus: AgentEventBus; + storageManager: StorageManager; + resourceManager: import('../resources/index.js').ResourceManager; + pluginManager: PluginManager; + mcpManager: import('../mcp/manager.js').MCPManager; + }, + config: SessionManagerConfig = {}, + logger: IDextoLogger + ) { + this.maxSessions = config.maxSessions ?? 100; + this.sessionTTL = config.sessionTTL ?? 3600000; // 1 hour + this.logger = logger.createChild(DextoLogComponent.SESSION); + } + + /** + * Initialize the SessionManager with persistent storage. + * This must be called before using any session operations. + */ + public async init(): Promise { + if (this.initialized) { + return; + } + + // Restore any existing sessions from storage + await this.restoreSessionsFromStorage(); + + // Start periodic cleanup to prevent memory leaks from long-lived sessions + // Clean up every 15 minutes or 1/4 of session TTL, whichever is smaller + const cleanupIntervalMs = Math.min(this.sessionTTL / 4, 15 * 60 * 1000); + this.cleanupInterval = setInterval( + () => + this.cleanupExpiredSessions().catch((err) => + this.logger.error(`Periodic session cleanup failed: ${err}`) + ), + cleanupIntervalMs + ); + + this.initialized = true; + this.logger.debug( + `SessionManager initialized with periodic cleanup every ${Math.round(cleanupIntervalMs / 1000 / 60)} minutes` + ); + } + + /** + * Restore sessions from persistent storage on startup. + * This allows sessions to survive application restarts. + */ + private async restoreSessionsFromStorage(): Promise { + try { + // Use the database backend to list sessions with the 'session:' prefix + const sessionKeys = await this.services.storageManager.getDatabase().list('session:'); + this.logger.debug(`Found ${sessionKeys.length} persisted sessions to restore`); + + for (const sessionKey of sessionKeys) { + const sessionId = sessionKey.replace('session:', ''); + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + + if (sessionData) { + // Check if session is still valid (not expired) + const now = Date.now(); + const lastActivity = sessionData.lastActivity; + + if (now - lastActivity <= this.sessionTTL) { + // Session is still valid, but don't create ChatSession until requested + this.logger.debug(`Session ${sessionId} restored from storage`); + } else { + // Session expired, clean it up + await this.services.storageManager.getDatabase().delete(sessionKey); + this.logger.debug(`Expired session ${sessionId} cleaned up during restore`); + } + } + } + } catch (error) { + this.logger.error( + `Failed to restore sessions from storage: ${error instanceof Error ? error.message : String(error)}` + ); + // Continue without restored sessions + } + } + + /** + * Ensures the SessionManager is initialized before operations. + */ + private async ensureInitialized(): Promise { + if (!this.initialized) { + if (!this.initializationPromise) { + this.initializationPromise = this.init(); + } + await this.initializationPromise; + } + } + + /** + * Creates a new chat session or returns an existing one. + * + * @param sessionId Optional session ID. If not provided, a UUID will be generated. + * @returns The created or existing ChatSession + * @throws Error if maximum sessions limit is reached + */ + public async createSession(sessionId?: string): Promise { + await this.ensureInitialized(); + + const id = sessionId ?? randomUUID(); + + // Check if there's already a pending creation for this session ID + if (this.pendingCreations.has(id)) { + return await this.pendingCreations.get(id)!; + } + + // Check if session already exists in memory + if (this.sessions.has(id)) { + await this.updateSessionActivity(id); + return this.sessions.get(id)!; + } + + // Create a promise for the session creation and track it to prevent concurrent operations + const creationPromise = this.createSessionInternal(id); + this.pendingCreations.set(id, creationPromise); + + try { + const session = await creationPromise; + return session; + } finally { + // Always clean up the pending creation tracker + this.pendingCreations.delete(id); + } + } + + /** + * Internal method that handles the actual session creation logic. + * This method implements atomic session creation to prevent race conditions. + */ + private async createSessionInternal(id: string): Promise { + // Clean up expired sessions first + await this.cleanupExpiredSessions(); + + // Check if session exists in storage (could have been created by another process) + const sessionKey = `session:${id}`; + const existingMetadata = await this.services.storageManager + .getDatabase() + .get(sessionKey); + if (existingMetadata) { + // Session exists in storage, restore it + await this.updateSessionActivity(id); + const session = new ChatSession( + { ...this.services, sessionManager: this }, + id, + this.logger + ); + await session.init(); + this.sessions.set(id, session); + this.logger.info(`Restored session from storage: ${id}`); + return session; + } + + // Perform atomic session limit check and creation + // This ensures the limit check and session creation happen as close to atomically as possible + const activeSessionKeys = await this.services.storageManager.getDatabase().list('session:'); + if (activeSessionKeys.length >= this.maxSessions) { + throw SessionError.maxSessionsExceeded(activeSessionKeys.length, this.maxSessions); + } + + // Create new session metadata first to "reserve" the session slot + const sessionData: SessionData = { + id, + createdAt: Date.now(), + lastActivity: Date.now(), + messageCount: 0, + }; + + // Store session metadata in persistent storage immediately to claim the session + try { + await this.services.storageManager.getDatabase().set(sessionKey, sessionData); + } catch (error) { + // If storage fails, another concurrent creation might have succeeded + this.logger.error(`Failed to store session metadata for ${id}:`, { + error: error instanceof Error ? error.message : String(error), + }); + // Re-throw the original error to maintain test compatibility + throw error; + } + + // Now create the actual session object + let session: ChatSession; + try { + session = new ChatSession({ ...this.services, sessionManager: this }, id, this.logger); + await session.init(); + this.sessions.set(id, session); + + // Also store in cache with TTL for faster access + await this.services.storageManager + .getCache() + .set(sessionKey, sessionData, this.sessionTTL / 1000); + + this.logger.info(`Created new session: ${id}`); + return session; + } catch (error) { + // If session creation fails after we've claimed the slot, clean up the metadata + this.logger.error( + `Failed to initialize session ${id}: ${error instanceof Error ? error.message : String(error)}` + ); + await this.services.storageManager.getDatabase().delete(sessionKey); + await this.services.storageManager.getCache().delete(sessionKey); + const reason = error instanceof Error ? error.message : 'unknown error'; + throw SessionError.initializationFailed(id, reason); + } + } + + /** + * Retrieves an existing session by ID. + * + * @param sessionId The session ID to retrieve + * @param restoreFromStorage Whether to restore from storage if not in memory (default: true) + * @returns The ChatSession if found, undefined otherwise + */ + public async getSession( + sessionId: string, + restoreFromStorage: boolean = true + ): Promise { + await this.ensureInitialized(); + + // Check if there's a pending creation for this session ID + if (this.pendingCreations.has(sessionId)) { + return await this.pendingCreations.get(sessionId)!; + } + + // Check memory first + if (this.sessions.has(sessionId)) { + return this.sessions.get(sessionId)!; + } + + // Conditionally check storage if restoreFromStorage is true + if (restoreFromStorage) { + const sessionKey = `session:${sessionId}`; + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + if (sessionData) { + // Restore session to memory + const session = new ChatSession( + { ...this.services, sessionManager: this }, + sessionId, + this.logger + ); + await session.init(); + this.sessions.set(sessionId, session); + return session; + } + } + + return undefined; + } + + /** + * Ends a session by removing it from memory without deleting conversation history. + * Used for cleanup, agent shutdown, and session expiry. + * + * @param sessionId The session ID to end + */ + public async endSession(sessionId: string): Promise { + await this.ensureInitialized(); + + // Remove from memory only - preserve conversation history in storage + const session = this.sessions.get(sessionId); + if (session) { + await session.cleanup(); // Clean up memory resources only + this.sessions.delete(sessionId); + } + + // Remove from cache but preserve database storage + const sessionKey = `session:${sessionId}`; + await this.services.storageManager.getCache().delete(sessionKey); + + this.logger.debug( + `Ended session (removed from memory, chat history preserved): ${sessionId}` + ); + } + + /** + * Deletes a session and its conversation history, removing everything from memory and storage. + * Used for user-initiated permanent deletion. + * + * @param sessionId The session ID to delete + */ + public async deleteSession(sessionId: string): Promise { + await this.ensureInitialized(); + + // Get session (load from storage if not in memory) to clean up memory resources + const session = await this.getSession(sessionId); + if (session) { + await session.cleanup(); // This cleans up memory resources + this.sessions.delete(sessionId); + } + + // Remove session metadata from storage + const sessionKey = `session:${sessionId}`; + await this.services.storageManager.getDatabase().delete(sessionKey); + await this.services.storageManager.getCache().delete(sessionKey); + + const messagesKey = `messages:${sessionId}`; + await this.services.storageManager.getDatabase().delete(messagesKey); + + this.logger.debug(`Deleted session and conversation history: ${sessionId}`); + } + + /** + * Resets the conversation history for a session while keeping the session alive. + * + * @param sessionId The session ID to reset + * @throws Error if session doesn't exist + */ + public async resetSession(sessionId: string): Promise { + await this.ensureInitialized(); + + const session = await this.getSession(sessionId); + if (!session) { + throw SessionError.notFound(sessionId); + } + + await session.reset(); + + // Reset message count in metadata + const sessionKey = `session:${sessionId}`; + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + if (sessionData) { + sessionData.messageCount = 0; + sessionData.lastActivity = Date.now(); + await this.services.storageManager.getDatabase().set(sessionKey, sessionData); + // Update cache as well + await this.services.storageManager + .getCache() + .set(sessionKey, sessionData, this.sessionTTL / 1000); + } + + this.logger.debug(`Reset session conversation: ${sessionId}`); + } + + /** + * Lists all active session IDs. + * + * @returns Array of active session IDs + */ + public async listSessions(): Promise { + await this.ensureInitialized(); + const sessionKeys = await this.services.storageManager.getDatabase().list('session:'); + return sessionKeys.map((key) => key.replace('session:', '')); + } + + /** + * Gets metadata for a specific session. + * + * @param sessionId The session ID + * @returns Session metadata if found, undefined otherwise + */ + public async getSessionMetadata(sessionId: string): Promise { + await this.ensureInitialized(); + const sessionKey = `session:${sessionId}`; + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + if (!sessionData) return undefined; + + return { + createdAt: sessionData.createdAt, + lastActivity: sessionData.lastActivity, + messageCount: sessionData.messageCount, + title: sessionData.metadata?.title, + ...(sessionData.tokenUsage && { tokenUsage: sessionData.tokenUsage }), + ...(sessionData.estimatedCost !== undefined && { + estimatedCost: sessionData.estimatedCost, + }), + }; + } + + /** + * Get the global session manager configuration. + */ + public getConfig(): SessionManagerConfig { + return { + maxSessions: this.maxSessions, + sessionTTL: this.sessionTTL, + }; + } + + /** + * Updates the last activity timestamp for a session. + */ + private async updateSessionActivity(sessionId: string): Promise { + const sessionKey = `session:${sessionId}`; + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + + if (sessionData) { + sessionData.lastActivity = Date.now(); + await this.services.storageManager.getDatabase().set(sessionKey, sessionData); + // Update cache as well + await this.services.storageManager + .getCache() + .set(sessionKey, sessionData, this.sessionTTL / 1000); + } + } + + /** + * Increments the message count for a session. + */ + public async incrementMessageCount(sessionId: string): Promise { + await this.ensureInitialized(); + + const sessionKey = `session:${sessionId}`; + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + + if (sessionData) { + sessionData.messageCount++; + sessionData.lastActivity = Date.now(); + await this.services.storageManager.getDatabase().set(sessionKey, sessionData); + // Update cache as well + await this.services.storageManager + .getCache() + .set(sessionKey, sessionData, this.sessionTTL / 1000); + } + } + + /** + * Accumulates token usage for a session. + * Called after each LLM response to update session-level totals. + * + * Uses per-session locking to prevent lost updates from concurrent calls. + */ + public async accumulateTokenUsage( + sessionId: string, + usage: TokenUsage, + cost?: number + ): Promise { + await this.ensureInitialized(); + + const sessionKey = `session:${sessionId}`; + + // Wait for any in-flight update for this session, then chain ours + const previousLock = this.tokenUsageLocks.get(sessionKey) ?? Promise.resolve(); + + const currentLock = previousLock.then(async () => { + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + + if (!sessionData) return; + + // Initialize if needed + if (!sessionData.tokenUsage) { + sessionData.tokenUsage = { + inputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + totalTokens: 0, + }; + } + + // Accumulate + sessionData.tokenUsage.inputTokens += usage.inputTokens ?? 0; + sessionData.tokenUsage.outputTokens += usage.outputTokens ?? 0; + sessionData.tokenUsage.reasoningTokens += usage.reasoningTokens ?? 0; + sessionData.tokenUsage.cacheReadTokens += usage.cacheReadTokens ?? 0; + sessionData.tokenUsage.cacheWriteTokens += usage.cacheWriteTokens ?? 0; + sessionData.tokenUsage.totalTokens += usage.totalTokens ?? 0; + + // Add cost if provided + if (cost !== undefined) { + sessionData.estimatedCost = (sessionData.estimatedCost ?? 0) + cost; + } + + sessionData.lastActivity = Date.now(); + + // Persist + await this.services.storageManager.getDatabase().set(sessionKey, sessionData); + await this.services.storageManager + .getCache() + .set(sessionKey, sessionData, this.sessionTTL / 1000); + }); + + this.tokenUsageLocks.set(sessionKey, currentLock); + + // Wait for our update to complete, but don't let errors propagate to break the chain + try { + await currentLock; + } finally { + // Clean up lock if this was the last operation + if (this.tokenUsageLocks.get(sessionKey) === currentLock) { + this.tokenUsageLocks.delete(sessionKey); + } + } + } + + /** + * Sets the human-friendly title for a session. + * Title is stored in session metadata and cached with TTL. + */ + public async setSessionTitle( + sessionId: string, + title: string, + opts: { ifUnsetOnly?: boolean } = {} + ): Promise { + await this.ensureInitialized(); + + const sessionKey = `session:${sessionId}`; + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + + if (!sessionData) { + throw SessionError.notFound(sessionId); + } + + const normalized = title.trim().slice(0, 80); + if (opts.ifUnsetOnly && sessionData.metadata?.title) { + return; + } + + sessionData.metadata = sessionData.metadata || {}; + sessionData.metadata.title = normalized; + sessionData.lastActivity = Date.now(); + + await this.services.storageManager.getDatabase().set(sessionKey, sessionData); + await this.services.storageManager + .getCache() + .set(sessionKey, sessionData, this.sessionTTL / 1000); + } + + /** + * Gets the stored title for a session, if any. + */ + public async getSessionTitle(sessionId: string): Promise { + await this.ensureInitialized(); + const sessionKey = `session:${sessionId}`; + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + return sessionData?.metadata?.title; + } + + /** + * Cleans up expired sessions from memory only, preserving chat history in storage. + * This allows inactive sessions to be garbage collected while keeping conversations restorable. + */ + private async cleanupExpiredSessions(): Promise { + const now = Date.now(); + const expiredSessions: string[] = []; + + // Check in-memory sessions for expiry + for (const [sessionId, _session] of this.sessions.entries()) { + const sessionKey = `session:${sessionId}`; + const sessionData = await this.services.storageManager + .getDatabase() + .get(sessionKey); + + if (sessionData && now - sessionData.lastActivity > this.sessionTTL) { + expiredSessions.push(sessionId); + } + } + + // Remove expired sessions from memory only (preserve storage) + for (const sessionId of expiredSessions) { + const session = this.sessions.get(sessionId); + if (session) { + // Only dispose memory resources, don't delete chat history + session.dispose(); + this.sessions.delete(sessionId); + this.logger.debug( + `Removed expired session from memory: ${sessionId} (chat history preserved)` + ); + } + } + + if (expiredSessions.length > 0) { + this.logger.debug( + `Memory cleanup: removed ${expiredSessions.length} inactive sessions, chat history preserved` + ); + } + } + + /** + * Switch LLM for all sessions. + * @param newLLMConfig The new LLM configuration to apply + * @returns Result object with success message and any warnings + */ + public async switchLLMForAllSessions( + newLLMConfig: ValidatedLLMConfig + ): Promise<{ message: string; warnings: string[] }> { + await this.ensureInitialized(); + + const sessionIds = await this.listSessions(); + const failedSessions: string[] = []; + + for (const sId of sessionIds) { + const session = await this.getSession(sId); + if (session) { + try { + // Update state with validated config (validation already done by DextoAgent) + // Using exceptions here for session-specific runtime failures (corruption, disposal, etc.) + // This is different from input validation which uses Result pattern + this.services.stateManager.updateLLM(newLLMConfig, sId); + await session.switchLLM(newLLMConfig); + } catch (error) { + // Session-level failure - continue processing other sessions (isolation) + failedSessions.push(sId); + this.logger.warn( + `Error switching LLM for session ${sId}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + } + + this.services.agentEventBus.emit('llm:switched', { + newConfig: newLLMConfig, + historyRetained: true, + sessionIds: sessionIds.filter((id) => !failedSessions.includes(id)), + }); + + const message = + failedSessions.length > 0 + ? `Successfully switched to ${newLLMConfig.provider}/${newLLMConfig.model} (${failedSessions.length} sessions failed)` + : `Successfully switched to ${newLLMConfig.provider}/${newLLMConfig.model} for all sessions`; + + const warnings = + failedSessions.length > 0 + ? [`Failed to switch LLM for sessions: ${failedSessions.join(', ')}`] + : []; + + return { message, warnings }; + } + + /** + * Switch LLM for a specific session. + * @param newLLMConfig The new LLM configuration to apply + * @param sessionId The session ID to switch LLM for + * @returns Result object with success message and any warnings + */ + public async switchLLMForSpecificSession( + newLLMConfig: ValidatedLLMConfig, + sessionId: string + ): Promise<{ message: string; warnings: string[] }> { + const session = await this.getSession(sessionId); + if (!session) { + throw SessionError.notFound(sessionId); + } + + await session.switchLLM(newLLMConfig); + + this.services.agentEventBus.emit('llm:switched', { + newConfig: newLLMConfig, + historyRetained: true, + sessionIds: [sessionId], + }); + + const message = `Successfully switched to ${newLLMConfig.provider}/${newLLMConfig.model} for session ${sessionId}`; + + return { message, warnings: [] }; + } + + /** + * Get session statistics for monitoring and debugging. + */ + public async getSessionStats(): Promise<{ + totalSessions: number; + inMemorySessions: number; + maxSessions: number; + sessionTTL: number; + }> { + await this.ensureInitialized(); + + const totalSessions = (await this.listSessions()).length; + const inMemorySessions = this.sessions.size; + + return { + totalSessions, + inMemorySessions, + maxSessions: this.maxSessions, + sessionTTL: this.sessionTTL, + }; + } + + /** + * Get the raw session data for a session ID. + * + * @param sessionId The session ID + * @returns Session data if found, undefined otherwise + */ + public async getSessionData(sessionId: string): Promise { + await this.ensureInitialized(); + const sessionKey = `session:${sessionId}`; + return await this.services.storageManager.getDatabase().get(sessionKey); + } + + /** + * Cleanup all sessions and resources. + * This should be called when shutting down the application. + */ + public async cleanup(): Promise { + if (!this.initialized) { + return; + } + + // Stop periodic cleanup + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval); + delete this.cleanupInterval; + this.logger.debug('Periodic session cleanup stopped'); + } + + // End all in-memory sessions (preserve conversation history) + const sessionIds = Array.from(this.sessions.keys()); + for (const sessionId of sessionIds) { + try { + await this.endSession(sessionId); + } catch (error) { + this.logger.error( + `Failed to cleanup session ${sessionId}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + this.sessions.clear(); + this.initialized = false; + this.logger.debug('SessionManager cleanup completed'); + } +} diff --git a/dexto/packages/core/src/session/title-generator.ts b/dexto/packages/core/src/session/title-generator.ts new file mode 100644 index 00000000..de21ca17 --- /dev/null +++ b/dexto/packages/core/src/session/title-generator.ts @@ -0,0 +1,151 @@ +import type { ValidatedLLMConfig } from '@core/llm/schemas.js'; +import type { ToolManager } from '@core/tools/tool-manager.js'; +import type { SystemPromptManager } from '@core/systemPrompt/manager.js'; +import type { ResourceManager } from '@core/resources/index.js'; +import type { IDextoLogger } from '@core/logger/v2/types.js'; +import { createLLMService } from '@core/llm/services/factory.js'; +import { SessionEventBus } from '@core/events/index.js'; +import { MemoryHistoryProvider } from './history/memory.js'; + +export interface GenerateSessionTitleResult { + title?: string; + error?: string; + timedOut?: boolean; +} + +/** + * Generate a concise title for a chat based on the first user message. + * Runs a lightweight, isolated LLM completion that does not touch real history. + */ +export async function generateSessionTitle( + config: ValidatedLLMConfig, + toolManager: ToolManager, + systemPromptManager: SystemPromptManager, + resourceManager: ResourceManager, + userText: string, + logger: IDextoLogger, + opts: { timeoutMs?: number } = {} +): Promise { + const timeoutMs = opts.timeoutMs; + const controller = timeoutMs !== undefined ? new AbortController() : undefined; + let timer: NodeJS.Timeout | undefined; + if (controller && timeoutMs && Number.isFinite(timeoutMs) && timeoutMs > 0) { + timer = setTimeout(() => controller.abort(), timeoutMs); + } + + try { + const history = new MemoryHistoryProvider(logger); + const bus = new SessionEventBus(); + const tempService = createLLMService( + config, + toolManager, + systemPromptManager, + history, + bus, + `titlegen-${Math.random().toString(36).slice(2)}`, + resourceManager, + logger + ); + + const instruction = [ + 'Generate a short conversation title from the following user message.', + 'Rules: 3–8 words; no surrounding punctuation, emojis, or PII; return only the title.', + '', + 'Message:', + sanitizeUserText(userText, 512), + ].join('\n'); + + const streamResult = await tempService.stream( + instruction, + controller ? { signal: controller.signal } : undefined + ); + + const processed = postProcessTitle(streamResult.text); + if (!processed) { + return { error: 'LLM returned empty title' }; + } + return { title: processed }; + } catch (error) { + if (controller?.signal.aborted) { + return { timedOut: true, error: 'Timed out while waiting for LLM response' }; + } + const message = error instanceof Error ? error.message : String(error); + return { error: message }; + } finally { + if (timer) { + clearTimeout(timer); + } + } +} + +/** + * Heuristic fallback when the LLM-based title fails. + */ +export function deriveHeuristicTitle(userText: string): string | undefined { + const sanitized = sanitizeUserText(userText, 120); + if (!sanitized) return undefined; + + const isSlashCommand = sanitized.startsWith('/'); + if (isSlashCommand) { + const [commandTokenRaw, ...rest] = sanitized.split(/\s+/); + if (!commandTokenRaw) { + return undefined; + } + const commandToken = commandTokenRaw.trim(); + const commandName = commandToken.startsWith('/') ? commandToken.slice(1) : commandToken; + if (!commandName) { + return undefined; + } + const command = commandName.replace(/[-_]+/g, ' '); + const commandTitle = toTitleCase(command); + const remainder = rest.join(' ').trim(); + if (remainder) { + return truncateWords(`${commandTitle} — ${remainder}`, 10, 70); + } + return commandTitle || undefined; + } + + const firstLine = sanitized.split(/\r?\n/)[0] ?? sanitized; + const withoutMarkdown = firstLine.replace(/[`*_~>#-]/g, '').trim(); + if (!withoutMarkdown) { + return undefined; + } + return truncateWords(toSentenceCase(withoutMarkdown), 10, 70); +} + +function sanitizeUserText(text: string, maxLen: number): string { + const cleaned = text.replace(/\p{Cc}+/gu, ' ').trim(); + return cleaned.length > maxLen ? cleaned.slice(0, maxLen) : cleaned; +} + +function postProcessTitle(raw: string): string | undefined { + if (!raw) return undefined; + let t = raw.trim(); + t = t.replace(/^["'`\s]+|["'`\s]+$/g, ''); + t = t.replace(/\s+/g, ' ').trim(); + t = t.replace(/[\s\-–—,:;.!?]+$/g, ''); + if (!t) return undefined; + return truncateWords(toSentenceCase(t), 8, 80); +} + +function truncateWords(text: string, maxWords: number, maxChars: number): string { + const words = text.split(' ').filter(Boolean); + let truncated = words.slice(0, maxWords).join(' '); + if (truncated.length > maxChars) { + truncated = truncated.slice(0, maxChars).trimEnd(); + } + return truncated; +} + +function toSentenceCase(text: string): string { + if (!text) return text; + return text.charAt(0).toUpperCase() + text.slice(1); +} + +function toTitleCase(text: string): string { + return text + .split(' ') + .filter(Boolean) + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); +} diff --git a/dexto/packages/core/src/session/types.ts b/dexto/packages/core/src/session/types.ts new file mode 100644 index 00000000..aaf2bb3e --- /dev/null +++ b/dexto/packages/core/src/session/types.ts @@ -0,0 +1,15 @@ +import { ContentPart } from '../context/types.js'; + +export interface QueuedMessage { + id: string; + content: ContentPart[]; + queuedAt: number; + metadata?: Record; +} + +export interface CoalescedMessage { + messages: QueuedMessage[]; + combinedContent: ContentPart[]; + firstQueuedAt: number; + lastQueuedAt: number; +} diff --git a/dexto/packages/core/src/storage/blob/README.md b/dexto/packages/core/src/storage/blob/README.md new file mode 100644 index 00000000..ea855401 --- /dev/null +++ b/dexto/packages/core/src/storage/blob/README.md @@ -0,0 +1,379 @@ +# Blob Storage Provider Pattern + +This module implements a flexible blob storage system using a provider pattern, allowing custom storage backends to be registered at runtime while maintaining type safety and validation. + +## Architecture + +``` +┌─────────────────────────────────────────┐ +│ Core Package │ +│ - BlobStore interface │ +│ - BlobStoreProvider interface │ +│ - Global registry (singleton) │ +│ - Built-in providers (local, memory) │ +│ - createBlobStore(config, logger) │ +└─────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ CLI/Server Layer │ +│ 1. Import core │ +│ 2. Register custom providers │ +│ 3. Load config YAML │ +│ 4. createBlobStore validates & creates │ +└─────────────────────────────────────────┘ +``` + +## Built-in Providers + +### Local Filesystem +```yaml +storage: + blob: + type: local + storePath: /path/to/blobs + maxBlobSize: 52428800 # 50MB + cleanupAfterDays: 30 +``` + +### In-Memory +```yaml +storage: + blob: + type: in-memory + maxBlobSize: 10485760 # 10MB + maxTotalSize: 104857600 # 100MB +``` + +## Creating Custom Providers + +### 1. Define Provider Type and Config + +```typescript +// packages/cli/src/storage/s3-provider.ts +import type { BlobStoreProvider, BlobStore } from '@dexto/core'; +import { z } from 'zod'; + +// Define config interface +interface S3BlobStoreConfig { + type: 's3'; + bucket: string; + region: string; + accessKeyId?: string; + secretAccessKey?: string; +} + +// Create Zod schema +const S3BlobStoreSchema = z.object({ + type: z.literal('s3'), + bucket: z.string(), + region: z.string(), + accessKeyId: z.string().optional(), + secretAccessKey: z.string().optional(), +}).strict(); +``` + +### 2. Implement BlobStore Interface + +```typescript +// packages/cli/src/storage/s3-blob-store.ts +import { S3Client } from '@aws-sdk/client-s3'; +import type { BlobStore, BlobInput, BlobMetadata, BlobReference, BlobData, BlobStats } from '@dexto/core'; + +export class S3BlobStore implements BlobStore { + private client: S3Client; + private config: S3BlobStoreConfig; + + constructor(config: S3BlobStoreConfig, logger: IDextoLogger) { + this.config = config; + this.client = new S3Client({ region: config.region }); + } + + async connect(): Promise { + // Initialize S3 client + } + + async store(input: BlobInput, metadata?: BlobMetadata): Promise { + // Upload to S3 + } + + async retrieve(reference: string, format?: string): Promise { + // Download from S3 + } + + // ... implement other BlobStore methods +} +``` + +### 3. Create Provider Definition + +```typescript +// packages/cli/src/storage/s3-provider.ts (continued) +import { S3BlobStore } from './s3-blob-store.js'; + +export const s3BlobStoreProvider: BlobStoreProvider<'s3', S3BlobStoreConfig> = { + type: 's3', + configSchema: S3BlobStoreSchema, + create: (config, logger) => new S3BlobStore(config, logger), + metadata: { + displayName: 'Amazon S3', + description: 'Store blobs in AWS S3', + requiresNetwork: true, + }, +}; +``` + +### 4. Register Provider at CLI Layer + +```typescript +// packages/cli/src/index.ts +import { blobStoreRegistry } from '@dexto/core'; +import { s3BlobStoreProvider } from './storage/s3-provider.js'; + +// Register BEFORE loading config +blobStoreRegistry.register(s3BlobStoreProvider); + +// Now S3 is available in config +const config = loadAgentConfig(); +const blobStore = createBlobStore(config.storage.blob, logger); +``` + +### 5. Use in Configuration + +```yaml +# agents/my-agent.yml +storage: + blob: + type: s3 # Custom provider now available! + bucket: my-bucket + region: us-east-1 +``` + +## Type Safety + +The provider pattern ensures compile-time and runtime type safety: + +### Compile-Time Safety +```typescript +// ✅ Type-safe: TypeScript enforces 's3' matches config type +const s3Provider: BlobStoreProvider<'s3', S3BlobStoreConfig> = { + type: 's3', // Must be 's3' + configSchema: S3BlobStoreSchema, // Must output S3BlobStoreConfig + create: (config, logger) => { + // config is properly typed as S3BlobStoreConfig + return new S3BlobStore(config, logger); + }, +}; + +// ❌ Compile error: type mismatch +const badProvider: BlobStoreProvider<'s3', S3BlobStoreConfig> = { + type: 'azure', // ERROR: Type '"azure"' is not assignable to type '"s3"' + // ... +}; +``` + +### Runtime Validation +```typescript +// Factory validates config against provider schema +const blobStore = createBlobStore( + { type: 's3', bucket: 'my-bucket', region: 'us-east-1' }, + logger +); + +// If validation fails, Zod throws with detailed error: +// "bucket" is required, "region" must be a string, etc. +``` + +## Benefits + +### ✅ Extensibility +- Add new providers without modifying core +- Register providers at any layer (CLI, server, tests) +- Multiple apps can have different provider sets + +### ✅ Type Safety +- Compile-time verification of provider definitions +- Runtime validation via Zod schemas +- Full autocomplete support + +### ✅ Separation of Concerns +- Core stays lightweight (no cloud SDKs) +- Cloud-specific code in appropriate layers +- Optional dependencies only when needed + +### ✅ Configuration-Driven +- YAML config selects the provider +- No code changes to switch backends +- Environment-specific configurations + +### ✅ Testability +- Register mock providers in tests +- Isolated unit testing per provider +- Registry can be cleared between tests + +## Implementation Details + +### Provider Interface +```typescript +export interface BlobStoreProvider< + TType extends string, + TConfig extends { type: TType } +> { + type: TType; + configSchema: z.ZodType; + create(config: TConfig, logger: IDextoLogger): BlobStore; + metadata?: { + displayName: string; + description: string; + requiresNetwork?: boolean; + }; +} +``` + +### Registry +- Global singleton: `blobStoreRegistry` +- Thread-safe registration +- Validates configs at runtime +- Provides error messages with available types + +### Factory +```typescript +export function createBlobStore( + config: { type: string; [key: string]: any }, + logger: IDextoLogger +): BlobStore { + // 1. Validate config against provider schema + const validatedConfig = blobStoreRegistry.validateConfig(config); + + // 2. Get provider + const provider = blobStoreRegistry.get(validatedConfig.type); + + // 3. Create instance + return provider.create(validatedConfig, logger); +} +``` + +## Migration Guide + +### From Hardcoded Switch Statements + +**Before:** +```typescript +export function createBlobStore(config: BlobStoreConfig, logger: IDextoLogger): BlobStore { + switch (config.type) { + case 'local': + return new LocalBlobStore(config, logger); + case 's3': + return new S3BlobStore(config, logger); + default: + throw new Error(`Unknown type: ${config.type}`); + } +} +``` + +**After:** +```typescript +// Core provides registry +export function createBlobStore(config, logger): BlobStore { + return blobStoreRegistry.validateConfig(config); + return blobStoreRegistry.get(config.type).create(config, logger); +} + +// CLI registers custom providers +blobStoreRegistry.register(s3Provider); +``` + +**Benefits:** +- ✅ No more modifying factory for new providers +- ✅ Providers can live in different packages +- ✅ Type safety maintained +- ✅ Config validation per provider + +## Example: Supabase Provider + +The Supabase provider demonstrates the pattern: + +**Location:** `packages/cli/src/storage/supabase-provider.ts` + +```typescript +export const supabaseBlobStoreProvider: BlobStoreProvider< + 'supabase', + SupabaseBlobStoreConfig +> = { + type: 'supabase', + configSchema: SupabaseBlobStoreSchema, + create: (config, logger) => new SupabaseBlobStore(config, logger), + metadata: { + displayName: 'Supabase Storage', + description: 'Store blobs in Supabase cloud storage', + requiresNetwork: true, + }, +}; +``` + +**Registration:** `packages/cli/src/index.ts` +```typescript +import { blobStoreRegistry } from '@dexto/core'; +import { supabaseBlobStoreProvider } from './storage/supabase-provider.js'; + +blobStoreRegistry.register(supabaseBlobStoreProvider); +``` + +**Usage in config:** +```yaml +storage: + blob: + type: supabase + supabaseUrl: https://xxx.supabase.co + supabaseKey: your-key + bucket: dexto-blobs +``` + +## Testing + +### Register Mock Provider +```typescript +import { blobStoreRegistry, type BlobStoreProvider } from '@dexto/core'; + +const mockProvider: BlobStoreProvider<'mock', MockConfig> = { + type: 'mock', + configSchema: MockConfigSchema, + create: (config, logger) => new MockBlobStore(config, logger), +}; + +beforeEach(() => { + blobStoreRegistry.register(mockProvider); +}); + +afterEach(() => { + blobStoreRegistry.unregister('mock'); +}); +``` + +### Test Provider Registration +```typescript +test('provider registration', () => { + blobStoreRegistry.register(s3Provider); + + expect(blobStoreRegistry.has('s3')).toBe(true); + expect(blobStoreRegistry.getTypes()).toContain('s3'); +}); +``` + +### Test Config Validation +```typescript +test('validates config against provider schema', () => { + blobStoreRegistry.register(s3Provider); + + expect(() => { + blobStoreRegistry.validateConfig({ type: 's3' }); // missing required fields + }).toThrow(); // Zod validation error +}); +``` + +## Future Enhancements + +- [ ] Provider discovery API for CLI help/docs +- [ ] Provider health checks +- [ ] Provider migration utilities +- [ ] Auto-registration via package.json conventions +- [ ] Provider dependency injection for testing diff --git a/dexto/packages/core/src/storage/blob/factory.ts b/dexto/packages/core/src/storage/blob/factory.ts new file mode 100644 index 00000000..bafd766b --- /dev/null +++ b/dexto/packages/core/src/storage/blob/factory.ts @@ -0,0 +1,54 @@ +import type { BlobStore } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { blobStoreRegistry } from './registry.js'; + +/** + * Create a blob store based on configuration using the provider registry. + * + * This factory function: + * 1. Validates the configuration against the registered provider's schema + * 2. Looks up the provider in the registry + * 3. Calls the provider's create method to instantiate the blob store + * + * The configuration type is determined at runtime by the 'type' field, + * which must match a registered provider. Custom providers can be registered + * via blobStoreRegistry.register() before calling this function. + * + * @param config - Blob store configuration with a 'type' discriminator + * @param logger - Logger instance for the blob store + * @returns A BlobStore implementation + * @throws Error if the provider type is not registered or validation fails + * + * @example + * ```typescript + * // Using built-in provider + * const blob = createBlobStore({ type: 'local', storePath: '/tmp/blobs' }, logger); + * + * // Using custom provider (registered beforehand) + * import { blobStoreRegistry } from '@dexto/core'; + * import { s3Provider } from './storage/s3-provider.js'; + * + * blobStoreRegistry.register(s3Provider); + * const blob = createBlobStore({ type: 's3', bucket: 'my-bucket' }, logger); + * ``` + */ +export function createBlobStore( + config: { type: string; [key: string]: any }, + logger: IDextoLogger +): BlobStore { + // Validate config against provider schema and get provider + const validatedConfig = blobStoreRegistry.validateConfig(config); + const provider = blobStoreRegistry.get(validatedConfig.type); + + if (!provider) { + // This should never happen after validateConfig, but handle it defensively + throw new Error(`Provider '${validatedConfig.type}' not found in registry`); + } + + // Log which provider is being used + const providerName = provider.metadata?.displayName || validatedConfig.type; + logger.info(`Using ${providerName} blob store`); + + // Create and return the blob store instance + return provider.create(validatedConfig, logger); +} diff --git a/dexto/packages/core/src/storage/blob/index.ts b/dexto/packages/core/src/storage/blob/index.ts new file mode 100644 index 00000000..982511d8 --- /dev/null +++ b/dexto/packages/core/src/storage/blob/index.ts @@ -0,0 +1,83 @@ +/** + * Blob Storage Module + * + * This module provides a flexible blob storage system with support for + * multiple backends through a provider pattern. + * + * ## Built-in Providers + * - `local`: Store blobs on the local filesystem + * - `in-memory`: Store blobs in RAM (for testing/development) + * + * ## Custom Providers + * Custom providers (e.g., S3, Azure, Supabase) can be registered at the + * CLI/server layer before configuration loading. + * + * ## Usage + * + * ### Using built-in providers + * ```typescript + * import { createBlobStore } from '@dexto/core'; + * + * const blob = createBlobStore({ type: 'local', storePath: '/tmp' }, logger); + * ``` + * + * ### Registering custom providers + * ```typescript + * import { blobStoreRegistry, type BlobStoreProvider } from '@dexto/core'; + * + * const s3Provider: BlobStoreProvider<'s3', S3Config> = { + * type: 's3', + * configSchema: S3ConfigSchema, + * create: (config, logger) => new S3BlobStore(config, logger), + * }; + * + * blobStoreRegistry.register(s3Provider); + * const blob = createBlobStore({ type: 's3', bucket: 'my-bucket' }, logger); + * ``` + */ + +// Import built-in providers +import { blobStoreRegistry } from './registry.js'; +import { localBlobStoreProvider, inMemoryBlobStoreProvider } from './providers/index.js'; + +// Register built-in providers on module load +// This ensures they're available when importing from @dexto/core +// Guard against duplicate registration when module is imported multiple times +if (!blobStoreRegistry.has('local')) { + blobStoreRegistry.register(localBlobStoreProvider); +} +if (!blobStoreRegistry.has('in-memory')) { + blobStoreRegistry.register(inMemoryBlobStoreProvider); +} + +// Export public API +export { createBlobStore } from './factory.js'; +export { blobStoreRegistry, BlobStoreRegistry } from './registry.js'; +export type { BlobStoreProvider } from './provider.js'; + +// Export types and interfaces +export type { + BlobStore, + BlobInput, + BlobMetadata, + BlobReference, + BlobData, + BlobStats, + StoredBlobMetadata, +} from './types.js'; + +// Export schemas and config types +export { + BLOB_STORE_TYPES, + BlobStoreConfigSchema, + InMemoryBlobStoreSchema, + LocalBlobStoreSchema, + type BlobStoreType, + type BlobStoreConfig, + type InMemoryBlobStoreConfig, + type LocalBlobStoreConfig, +} from './schemas.js'; + +// Export concrete implementations (for custom usage and external providers) +export { LocalBlobStore } from './local-blob-store.js'; +export { InMemoryBlobStore } from './memory-blob-store.js'; diff --git a/dexto/packages/core/src/storage/blob/local-blob-store.ts b/dexto/packages/core/src/storage/blob/local-blob-store.ts new file mode 100644 index 00000000..f26205ae --- /dev/null +++ b/dexto/packages/core/src/storage/blob/local-blob-store.ts @@ -0,0 +1,586 @@ +import { promises as fs, createReadStream } from 'fs'; +import path from 'path'; +import { createHash } from 'crypto'; +import { pathToFileURL } from 'url'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { StorageError } from '../errors.js'; +import type { + BlobStore, + BlobInput, + BlobMetadata, + BlobReference, + BlobData, + BlobStats, + StoredBlobMetadata, +} from './types.js'; +import type { LocalBlobStoreConfig } from './schemas.js'; + +/** + * Local filesystem blob store implementation. + * + * Stores blobs on the local filesystem with content-based deduplication + * and metadata tracking. This is the default store for development + * and single-machine deployments. + */ +export class LocalBlobStore implements BlobStore { + private config: LocalBlobStoreConfig; + private storePath: string; + private connected = false; + private statsCache: { count: number; totalSize: number } | null = null; + private statsCachePromise: Promise | null = null; + private lastStatsRefresh: number = 0; + private logger: IDextoLogger; + + private static readonly STATS_REFRESH_INTERVAL_MS = 60000; // 1 minute + + constructor(config: LocalBlobStoreConfig, logger: IDextoLogger) { + this.config = config; + // Store path is provided via CLI enrichment + this.storePath = config.storePath; + this.logger = logger.createChild(DextoLogComponent.STORAGE); + } + + async connect(): Promise { + if (this.connected) return; + + try { + await fs.mkdir(this.storePath, { recursive: true }); + await this.refreshStatsCache(); + this.lastStatsRefresh = Date.now(); + this.connected = true; + this.logger.debug(`LocalBlobStore connected at: ${this.storePath}`); + + // TODO: Implement automatic blob cleanup scheduling + // - Call cleanup() on connect to remove old blobs based on cleanupAfterDays config + // - Start a background interval task (e.g., daily) to periodically run cleanup() + // - Ensure the background task can be cancelled on disconnect() + // - Use the configured cleanupAfterDays value when invoking cleanup() + } catch (error) { + throw StorageError.blobOperationFailed('connect', 'local', error); + } + } + + async disconnect(): Promise { + this.connected = false; + this.logger.debug('LocalBlobStore disconnected'); + } + + /** + * Normalize metadata after JSON parsing to ensure createdAt is a Date object + */ + private normalizeMetadata(parsed: any): StoredBlobMetadata { + return { + ...parsed, + createdAt: + parsed.createdAt instanceof Date ? parsed.createdAt : new Date(parsed.createdAt), + }; + } + + isConnected(): boolean { + return this.connected; + } + + getStoreType(): string { + return 'local'; + } + + async store(input: BlobInput, metadata: BlobMetadata = {}): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('local'); + } + + // Convert input to buffer for processing + const buffer = await this.inputToBuffer(input); + + // Check size limits + if (buffer.length > this.config.maxBlobSize) { + throw StorageError.blobSizeExceeded(buffer.length, this.config.maxBlobSize); + } + + // Generate content-based hash for deduplication + const hash = createHash('sha256').update(buffer).digest('hex').substring(0, 16); + const id = hash; + + const blobPath = path.join(this.storePath, `${id}.dat`); + const metaPath = path.join(this.storePath, `${id}.meta.json`); + + // Check if blob already exists (deduplication) + try { + const existingMeta = await fs.readFile(metaPath, 'utf-8'); + const parsed = JSON.parse(existingMeta) as any; + const existingMetadata: StoredBlobMetadata = this.normalizeMetadata(parsed); + this.logger.debug( + `Blob ${id} already exists, returning existing reference (deduplication)` + ); + + return { + id, + uri: `blob:${id}`, + metadata: existingMetadata, + }; + } catch { + // Blob doesn't exist, continue with storage + } + + // Check total storage size if configured (only for new blobs) + const maxTotalSize = this.config.maxTotalSize; + if (maxTotalSize) { + const stats = await this.ensureStatsCache(); + if (stats.totalSize + buffer.length > maxTotalSize) { + throw StorageError.blobTotalSizeExceeded( + stats.totalSize + buffer.length, + maxTotalSize + ); + } + } + + // Create complete metadata + const storedMetadata: StoredBlobMetadata = { + id, + mimeType: metadata.mimeType || this.detectMimeType(buffer, metadata.originalName), + originalName: metadata.originalName, + createdAt: metadata.createdAt || new Date(), + source: metadata.source || 'system', + size: buffer.length, + hash, + }; + + try { + // Store blob data and metadata atomically + await Promise.all([ + fs.writeFile(blobPath, buffer), + fs.writeFile(metaPath, JSON.stringify(storedMetadata, null, 2)), + ]); + + this.logger.debug( + `Stored blob ${id} (${buffer.length} bytes, ${storedMetadata.mimeType})` + ); + + this.updateStatsCacheAfterStore(buffer.length); + + return { + id, + uri: `blob:${id}`, + metadata: storedMetadata, + }; + } catch (error) { + // Cleanup on failure + await Promise.allSettled([ + fs.unlink(blobPath).catch(() => {}), + fs.unlink(metaPath).catch(() => {}), + ]); + throw StorageError.blobOperationFailed('store', 'local', error); + } + } + + async retrieve( + reference: string, + format: 'base64' | 'buffer' | 'path' | 'stream' | 'url' = 'buffer' + ): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('local'); + } + + const id = this.parseReference(reference); + const blobPath = path.join(this.storePath, `${id}.dat`); + const metaPath = path.join(this.storePath, `${id}.meta.json`); + + try { + // Load metadata + const metaContent = await fs.readFile(metaPath, 'utf-8'); + const parsed = JSON.parse(metaContent) as any; + const metadata: StoredBlobMetadata = this.normalizeMetadata(parsed); + + // Return data in requested format + switch (format) { + case 'base64': { + const buffer = await fs.readFile(blobPath); + return { format: 'base64', data: buffer.toString('base64'), metadata }; + } + case 'buffer': { + const buffer = await fs.readFile(blobPath); + return { format: 'buffer', data: buffer, metadata }; + } + case 'path': { + // Verify file exists + await fs.access(blobPath); + return { format: 'path', data: blobPath, metadata }; + } + case 'stream': { + const stream = createReadStream(blobPath); + return { format: 'stream', data: stream, metadata }; + } + case 'url': { + // For local store, return file:// URL (cross-platform) + const absolutePath = path.resolve(blobPath); + return { + format: 'url', + data: pathToFileURL(absolutePath).toString(), + metadata, + }; + } + default: + throw StorageError.blobInvalidInput(format, `Unsupported format: ${format}`); + } + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { + throw StorageError.blobNotFound(reference); + } + throw StorageError.blobOperationFailed('retrieve', 'local', error); + } + } + + async exists(reference: string): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('local'); + } + + const id = this.parseReference(reference); + const blobPath = path.join(this.storePath, `${id}.dat`); + const metaPath = path.join(this.storePath, `${id}.meta.json`); + + try { + await Promise.all([fs.access(blobPath), fs.access(metaPath)]); + return true; + } catch { + return false; + } + } + + async delete(reference: string): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('local'); + } + + const id = this.parseReference(reference); + const blobPath = path.join(this.storePath, `${id}.dat`); + const metaPath = path.join(this.storePath, `${id}.meta.json`); + + try { + const metaContent = await fs.readFile(metaPath, 'utf-8'); + const parsed = JSON.parse(metaContent) as any; + const metadata: StoredBlobMetadata = this.normalizeMetadata(parsed); + await Promise.all([fs.unlink(blobPath), fs.unlink(metaPath)]); + this.logger.debug(`Deleted blob: ${id}`); + this.updateStatsCacheAfterDelete(metadata.size); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { + throw StorageError.blobNotFound(reference); + } + throw StorageError.blobOperationFailed('delete', 'local', error); + } + } + + async cleanup(olderThan?: Date): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('local'); + } + + const cleanupDays = this.config.cleanupAfterDays; + const cutoffDate = olderThan || new Date(Date.now() - cleanupDays * 24 * 60 * 60 * 1000); + let deletedCount = 0; + + try { + const files = await fs.readdir(this.storePath); + const metaFiles = files.filter((f) => f.endsWith('.meta.json')); + + for (const metaFile of metaFiles) { + const metaPath = path.join(this.storePath, metaFile); + const id = metaFile.replace('.meta.json', ''); + const blobPath = path.join(this.storePath, `${id}.dat`); + + try { + const metaContent = await fs.readFile(metaPath, 'utf-8'); + const parsed = JSON.parse(metaContent) as any; + const metadata: StoredBlobMetadata = this.normalizeMetadata(parsed); + + if (metadata.createdAt < cutoffDate) { + await Promise.all([ + fs.unlink(blobPath).catch(() => {}), + fs.unlink(metaPath).catch(() => {}), + ]); + deletedCount++; + this.updateStatsCacheAfterDelete(metadata.size); + this.logger.debug(`Cleaned up old blob: ${id}`); + } + } catch (error) { + this.logger.warn( + `Failed to process blob metadata ${metaFile}: ${String(error)}` + ); + } + } + + if (deletedCount > 0) { + this.logger.info(`Blob cleanup: removed ${deletedCount} old blobs`); + } + + return deletedCount; + } catch (error) { + throw StorageError.blobCleanupFailed('local', error); + } + } + + async getStats(): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('local'); + } + + const stats = await this.ensureStatsCache(); + + return { + count: stats.count, + totalSize: stats.totalSize, + backendType: 'local', + storePath: this.storePath, + }; + } + + getStoragePath(): string | undefined { + return this.storePath; + } + + async listBlobs(): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('local'); + } + + try { + const files = await fs.readdir(this.storePath); + const metaFiles = files.filter((f) => f.endsWith('.meta.json')); + const blobs: BlobReference[] = []; + + for (const metaFile of metaFiles) { + try { + const metaPath = path.join(this.storePath, metaFile); + const metaContent = await fs.readFile(metaPath, 'utf-8'); + const parsed = JSON.parse(metaContent) as any; + const metadata: StoredBlobMetadata = this.normalizeMetadata(parsed); + const blobId = metaFile.replace('.meta.json', ''); + + blobs.push({ + id: blobId, + uri: `blob:${blobId}`, + metadata, + }); + } catch (error) { + this.logger.warn( + `Failed to process blob metadata ${metaFile}: ${String(error)}` + ); + } + } + + return blobs; + } catch (error) { + this.logger.warn(`Failed to list blobs: ${String(error)}`); + return []; + } + } + + private async ensureStatsCache(): Promise<{ count: number; totalSize: number }> { + const now = Date.now(); + const cacheAge = now - this.lastStatsRefresh; + const isStale = cacheAge > LocalBlobStore.STATS_REFRESH_INTERVAL_MS; + + // Refresh if cache is missing OR stale + if (!this.statsCache || isStale) { + if (!this.statsCachePromise) { + this.statsCachePromise = this.refreshStatsCache(); + } + + try { + await this.statsCachePromise; + this.lastStatsRefresh = now; + } finally { + this.statsCachePromise = null; + } + } + + return this.statsCache ?? { count: 0, totalSize: 0 }; + } + + private async refreshStatsCache(): Promise { + const stats = { count: 0, totalSize: 0 }; + try { + const files = await fs.readdir(this.storePath); + const datFiles = files.filter((f) => f.endsWith('.dat')); + stats.count = datFiles.length; + + for (const datFile of datFiles) { + try { + const stat = await fs.stat(path.join(this.storePath, datFile)); + stats.totalSize += stat.size; + } catch (error) { + this.logger.debug( + `Skipping size calculation for ${datFile}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + } catch (error) { + this.logger.warn(`Failed to refresh blob stats cache: ${String(error)}`); + } + + this.statsCache = stats; + } + + private updateStatsCacheAfterStore(size: number): void { + if (!this.statsCache) { + this.statsCache = { count: 1, totalSize: size }; + return; + } + this.statsCache.count += 1; + this.statsCache.totalSize += size; + } + + private updateStatsCacheAfterDelete(size: number): void { + if (!this.statsCache) { + this.statsCache = { count: 0, totalSize: 0 }; + return; + } + this.statsCache.count = Math.max(0, this.statsCache.count - 1); + this.statsCache.totalSize = Math.max(0, this.statsCache.totalSize - size); + } + + /** + * Convert various input types to Buffer + */ + private async inputToBuffer(input: BlobInput): Promise { + if (Buffer.isBuffer(input)) { + return input; + } + + if (input instanceof Uint8Array) { + return Buffer.from(input); + } + + if (input instanceof ArrayBuffer) { + return Buffer.from(new Uint8Array(input)); + } + + if (typeof input === 'string') { + // Handle data URI + if (input.startsWith('data:')) { + const commaIndex = input.indexOf(','); + if (commaIndex !== -1 && input.includes(';base64,')) { + const base64Data = input.substring(commaIndex + 1); + return Buffer.from(base64Data, 'base64'); + } + throw StorageError.blobEncodingError( + 'inputToBuffer', + 'Unsupported data URI format' + ); + } + + // Handle file path - only if it looks like an actual file path + if ((input.includes('/') || input.includes('\\')) && input.length > 1) { + try { + await fs.access(input); + return await fs.readFile(input); + } catch { + // If file doesn't exist, treat as base64 + } + } + + // Assume base64 string + try { + return Buffer.from(input, 'base64'); + } catch { + throw StorageError.blobEncodingError('inputToBuffer', 'Invalid base64 string'); + } + } + + throw StorageError.blobInvalidInput(input, `Unsupported input type: ${typeof input}`); + } + + /** + * Parse blob reference to extract ID + */ + private parseReference(reference: string): string { + if (!reference) { + throw StorageError.blobInvalidReference(reference, 'Empty reference'); + } + + if (reference.startsWith('blob:')) { + const id = reference.substring(5); + if (!id) { + throw StorageError.blobInvalidReference(reference, 'Empty blob ID after prefix'); + } + return id; + } + + return reference; + } + + /** + * Detect MIME type from buffer content and/or filename + */ + private detectMimeType(buffer: Buffer, filename?: string): string { + // Basic magic number detection + const header = buffer.subarray(0, 16); + + // Check common file signatures + if (header.length >= 3) { + const jpegSignature = header.subarray(0, 3); + if (jpegSignature.equals(Buffer.from([0xff, 0xd8, 0xff]))) { + return 'image/jpeg'; + } + } + + if (header.length >= 4) { + const signature = header.subarray(0, 4); + if (signature.equals(Buffer.from([0x89, 0x50, 0x4e, 0x47]))) return 'image/png'; + if (signature.equals(Buffer.from([0x47, 0x49, 0x46, 0x38]))) return 'image/gif'; + if (signature.equals(Buffer.from([0x25, 0x50, 0x44, 0x46]))) return 'application/pdf'; + } + + // Try filename extension + if (filename) { + const ext = path.extname(filename).toLowerCase(); + const mimeTypes: Record = { + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.gif': 'image/gif', + '.pdf': 'application/pdf', + '.txt': 'text/plain', + '.json': 'application/json', + '.xml': 'text/xml', + '.html': 'text/html', + '.css': 'text/css', + '.js': 'text/javascript', + '.mp3': 'audio/mpeg', + '.mp4': 'video/mp4', + '.wav': 'audio/wav', + }; + if (mimeTypes[ext]) return mimeTypes[ext]; + } + + // Check if content looks like text + if (this.isTextBuffer(buffer)) { + return 'text/plain'; + } + + return 'application/octet-stream'; + } + + /** + * Check if buffer contains text content + */ + private isTextBuffer(buffer: Buffer): boolean { + // Simple heuristic: check if most bytes are printable ASCII or common UTF-8 + let printableCount = 0; + const sampleSize = Math.min(512, buffer.length); + + for (let i = 0; i < sampleSize; i++) { + const byte = buffer[i]; + if ( + byte !== undefined && + ((byte >= 32 && byte <= 126) || byte === 9 || byte === 10 || byte === 13) + ) { + printableCount++; + } + } + + return printableCount / sampleSize > 0.7; + } +} diff --git a/dexto/packages/core/src/storage/blob/memory-blob-store.ts b/dexto/packages/core/src/storage/blob/memory-blob-store.ts new file mode 100644 index 00000000..0cb17acc --- /dev/null +++ b/dexto/packages/core/src/storage/blob/memory-blob-store.ts @@ -0,0 +1,418 @@ +import { createHash } from 'crypto'; +import { Readable } from 'stream'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { StorageError } from '../errors.js'; +import type { + BlobStore, + BlobInput, + BlobMetadata, + BlobReference, + BlobData, + BlobStats, + StoredBlobMetadata, +} from './types.js'; +import type { InMemoryBlobStoreConfig } from './schemas.js'; + +/** + * In-memory blob store implementation. + * + * Stores blobs in memory with content-based deduplication and size limits. + * Suitable for development, testing, and scenarios where blob persistence + * across restarts is not required. + * + * Features: + * - Content-based deduplication (same as LocalBlobStore) + * - Configurable size limits (per-blob and total) + * - Automatic cleanup of old blobs + * - MIME type detection + * - Multi-format retrieval (base64, buffer, stream, data URI) + * + * Limitations: + * - Data lost on restart (no persistence) + * - Path format not supported (no filesystem) + * - Memory usage proportional to blob size + */ +export class InMemoryBlobStore implements BlobStore { + private config: InMemoryBlobStoreConfig; + private blobs: Map = new Map(); + private connected = false; + private logger: IDextoLogger; + + constructor(config: InMemoryBlobStoreConfig, logger: IDextoLogger) { + this.config = config; + this.logger = logger.createChild(DextoLogComponent.STORAGE); + } + + async connect(): Promise { + if (this.connected) return; + this.connected = true; + this.logger.debug('InMemoryBlobStore connected'); + } + + async disconnect(): Promise { + // Clear all blobs on disconnect + this.blobs.clear(); + this.connected = false; + this.logger.debug('InMemoryBlobStore disconnected'); + } + + isConnected(): boolean { + return this.connected; + } + + getStoreType(): string { + return 'in-memory'; + } + + async store(input: BlobInput, metadata: BlobMetadata = {}): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('in-memory'); + } + + // Convert input to buffer for processing + const buffer = await this.inputToBuffer(input); + + // Check per-blob size limit + if (buffer.length > this.config.maxBlobSize) { + throw StorageError.blobSizeExceeded(buffer.length, this.config.maxBlobSize); + } + + // Generate content-based hash for deduplication + const hash = createHash('sha256').update(buffer).digest('hex').substring(0, 16); + const id = hash; + + // Check if blob already exists (deduplication) + const existing = this.blobs.get(id); + if (existing) { + this.logger.debug( + `Blob ${id} already exists, returning existing reference (deduplication)` + ); + return { + id, + uri: `blob:${id}`, + metadata: existing.metadata, + }; + } + + // Check total storage size (only for new blobs) + const currentSize = this.getTotalSize(); + if (currentSize + buffer.length > this.config.maxTotalSize) { + throw StorageError.blobTotalSizeExceeded( + currentSize + buffer.length, + this.config.maxTotalSize + ); + } + + // Create complete metadata + const storedMetadata: StoredBlobMetadata = { + id, + mimeType: metadata.mimeType || this.detectMimeType(buffer, metadata.originalName), + originalName: metadata.originalName, + createdAt: metadata.createdAt || new Date(), + source: metadata.source || 'system', + size: buffer.length, + hash, + }; + + // Store blob in memory + this.blobs.set(id, { data: buffer, metadata: storedMetadata }); + + this.logger.debug(`Stored blob ${id} (${buffer.length} bytes, ${storedMetadata.mimeType})`); + + return { + id, + uri: `blob:${id}`, + metadata: storedMetadata, + }; + } + + async retrieve( + reference: string, + format: 'base64' | 'buffer' | 'path' | 'stream' | 'url' = 'base64' + ): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('in-memory'); + } + + const id = this.parseReference(reference); + const blob = this.blobs.get(id); + + if (!blob) { + throw StorageError.blobNotFound(reference); + } + + // Return data in requested format + switch (format) { + case 'base64': + return { + format: 'base64', + data: blob.data.toString('base64'), + metadata: blob.metadata, + }; + + case 'buffer': + return { + format: 'buffer', + data: Buffer.from(blob.data), + metadata: { ...blob.metadata }, + }; + + case 'path': + // Path format not supported for in-memory blobs + throw new Error( + 'Path format not supported for in-memory blobs. Use local blob storage for filesystem paths.' + ); + + case 'stream': { + // Create readable stream from buffer + const stream = Readable.from(blob.data); + return { + format: 'stream', + data: stream as NodeJS.ReadableStream, + metadata: blob.metadata, + }; + } + + case 'url': { + // Return data URI for in-memory blobs + const base64 = blob.data.toString('base64'); + const mimeType = blob.metadata.mimeType || 'application/octet-stream'; + const dataUrl = `data:${mimeType};base64,${base64}`; + return { + format: 'url', + data: dataUrl, + metadata: blob.metadata, + }; + } + + default: + throw StorageError.blobInvalidInput(format, `Unsupported format: ${format}`); + } + } + + async exists(reference: string): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('in-memory'); + } + + const id = this.parseReference(reference); + return this.blobs.has(id); + } + + async delete(reference: string): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('in-memory'); + } + + const id = this.parseReference(reference); + + if (!this.blobs.has(id)) { + throw StorageError.blobNotFound(reference); + } + + this.blobs.delete(id); + this.logger.debug(`Deleted blob: ${id}`); + } + + async cleanup(olderThan?: Date): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('in-memory'); + } + + // Default cutoff: clean up blobs older than 30 days + const cutoffDate = olderThan || new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); + let deletedCount = 0; + + for (const [id, { metadata }] of this.blobs.entries()) { + if (metadata.createdAt < cutoffDate) { + this.blobs.delete(id); + deletedCount++; + this.logger.debug(`Cleaned up old blob: ${id}`); + } + } + + if (deletedCount > 0) { + this.logger.info(`Blob cleanup: removed ${deletedCount} old blobs from memory`); + } + + return deletedCount; + } + + async getStats(): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('in-memory'); + } + + // Stats are computed instantly from Map (no caching needed) + return { + count: this.blobs.size, + totalSize: this.getTotalSize(), + backendType: 'in-memory', + storePath: 'memory://', + }; + } + + async listBlobs(): Promise { + if (!this.connected) { + throw StorageError.blobBackendNotConnected('in-memory'); + } + + return Array.from(this.blobs.entries()).map(([id, { metadata }]) => ({ + id, + uri: `blob:${id}`, + metadata, + })); + } + + getStoragePath(): string | undefined { + // In-memory store has no filesystem path + return undefined; + } + + /** + * Calculate total size of all blobs in memory + */ + private getTotalSize(): number { + return Array.from(this.blobs.values()).reduce((sum, { data }) => sum + data.length, 0); + } + + /** + * Convert various input types to Buffer. + * Copied from LocalBlobStore with minor adaptations. + */ + private async inputToBuffer(input: BlobInput): Promise { + if (Buffer.isBuffer(input)) { + return input; + } + + if (input instanceof Uint8Array) { + return Buffer.from(input); + } + + if (input instanceof ArrayBuffer) { + return Buffer.from(new Uint8Array(input)); + } + + if (typeof input === 'string') { + // Handle data URI + if (input.startsWith('data:')) { + const commaIndex = input.indexOf(','); + if (commaIndex !== -1 && input.includes(';base64,')) { + const base64Data = input.substring(commaIndex + 1); + return Buffer.from(base64Data, 'base64'); + } + throw StorageError.blobEncodingError( + 'inputToBuffer', + 'Unsupported data URI format' + ); + } + + // For in-memory store, we don't read from filesystem paths + // Assume base64 string + try { + return Buffer.from(input, 'base64'); + } catch { + throw StorageError.blobEncodingError('inputToBuffer', 'Invalid base64 string'); + } + } + + throw StorageError.blobInvalidInput(input, `Unsupported input type: ${typeof input}`); + } + + /** + * Parse blob reference to extract ID. + * Copied from LocalBlobStore. + */ + private parseReference(reference: string): string { + if (!reference) { + throw StorageError.blobInvalidReference(reference, 'Empty reference'); + } + + if (reference.startsWith('blob:')) { + const id = reference.substring(5); + if (!id) { + throw StorageError.blobInvalidReference(reference, 'Empty blob ID after prefix'); + } + return id; + } + + return reference; + } + + /** + * Detect MIME type from buffer content and/or filename. + * Copied from LocalBlobStore. + */ + private detectMimeType(buffer: Buffer, filename?: string): string { + // Basic magic number detection + const header = buffer.subarray(0, 16); + + // Check common file signatures + if (header.length >= 3) { + const jpegSignature = header.subarray(0, 3); + if (jpegSignature.equals(Buffer.from([0xff, 0xd8, 0xff]))) { + return 'image/jpeg'; + } + } + + if (header.length >= 4) { + const signature = header.subarray(0, 4); + if (signature.equals(Buffer.from([0x89, 0x50, 0x4e, 0x47]))) return 'image/png'; + if (signature.equals(Buffer.from([0x47, 0x49, 0x46, 0x38]))) return 'image/gif'; + if (signature.equals(Buffer.from([0x25, 0x50, 0x44, 0x46]))) return 'application/pdf'; + } + + // Try filename extension + if (filename) { + const ext = filename.split('.').pop()?.toLowerCase(); + const mimeTypes: Record = { + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + png: 'image/png', + gif: 'image/gif', + pdf: 'application/pdf', + txt: 'text/plain', + json: 'application/json', + xml: 'text/xml', + html: 'text/html', + css: 'text/css', + js: 'text/javascript', + md: 'text/markdown', + mp3: 'audio/mpeg', + mp4: 'video/mp4', + wav: 'audio/wav', + }; + if (ext && mimeTypes[ext]) return mimeTypes[ext]; + } + + // Check if content looks like text + if (this.isTextBuffer(buffer)) { + return 'text/plain'; + } + + return 'application/octet-stream'; + } + + /** + * Check if buffer contains text content. + * Copied from LocalBlobStore. + */ + private isTextBuffer(buffer: Buffer): boolean { + // Simple heuristic: check if most bytes are printable ASCII or common UTF-8 + let printableCount = 0; + const sampleSize = Math.min(512, buffer.length); + + for (let i = 0; i < sampleSize; i++) { + const byte = buffer[i]; + if ( + byte !== undefined && + ((byte >= 32 && byte <= 126) || byte === 9 || byte === 10 || byte === 13) + ) { + printableCount++; + } + } + + return printableCount / sampleSize > 0.7; + } +} diff --git a/dexto/packages/core/src/storage/blob/provider.ts b/dexto/packages/core/src/storage/blob/provider.ts new file mode 100644 index 00000000..2894a38b --- /dev/null +++ b/dexto/packages/core/src/storage/blob/provider.ts @@ -0,0 +1,54 @@ +import type { z } from 'zod'; +import type { BlobStore } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; + +/** + * Provider interface for creating blob store instances. + * + * This interface uses TypeScript generics to enforce type safety: + * - TType: The literal type string (e.g., 'local', 's3') + * - TConfig: The configuration type with discriminator { type: TType } + * + * This ensures that the provider type matches the config type discriminator, + * providing compile-time safety for provider implementations. + */ +export interface BlobStoreProvider< + TType extends string = string, + TConfig extends { type: TType } = any, +> { + /** + * Unique identifier for this provider (e.g., 'local', 'supabase', 's3'). + * Must match the 'type' field in the configuration. + */ + type: TType; + + /** + * Zod schema for validating provider-specific configuration. + * The schema must output TConfig type. + * + * Note: Uses z.ZodType with relaxed generics to allow input/output type variance. + * This is necessary because Zod schemas with `.optional().default()` have + * input types that include undefined, but output types that don't. + */ + configSchema: z.ZodType; + + /** + * Factory function to create a BlobStore instance. + * @param config - Validated configuration specific to this provider + * @param logger - Logger instance for the blob store + * @returns A BlobStore implementation + */ + create(config: TConfig, logger: IDextoLogger): BlobStore; + + /** + * Optional metadata for documentation, UIs, and discovery. + */ + metadata?: { + /** Human-readable name (e.g., "Local Filesystem", "Amazon S3") */ + displayName: string; + /** Brief description of this storage backend */ + description: string; + /** Whether this provider requires network connectivity */ + requiresNetwork?: boolean; + }; +} diff --git a/dexto/packages/core/src/storage/blob/providers/index.ts b/dexto/packages/core/src/storage/blob/providers/index.ts new file mode 100644 index 00000000..8fc77749 --- /dev/null +++ b/dexto/packages/core/src/storage/blob/providers/index.ts @@ -0,0 +1,8 @@ +/** + * Built-in blob store providers. + * + * These providers are automatically registered when importing from @dexto/core. + */ + +export { localBlobStoreProvider } from './local.js'; +export { inMemoryBlobStoreProvider } from './memory.js'; diff --git a/dexto/packages/core/src/storage/blob/providers/local.ts b/dexto/packages/core/src/storage/blob/providers/local.ts new file mode 100644 index 00000000..d4f281c4 --- /dev/null +++ b/dexto/packages/core/src/storage/blob/providers/local.ts @@ -0,0 +1,28 @@ +import type { BlobStoreProvider } from '../provider.js'; +import type { LocalBlobStoreConfig } from '../schemas.js'; +import { LocalBlobStoreSchema } from '../schemas.js'; +import { LocalBlobStore } from '../local-blob-store.js'; + +/** + * Provider for local filesystem blob storage. + * + * This provider stores blobs on the local filesystem with content-based + * deduplication and metadata tracking. It's ideal for development and + * single-machine deployments. + * + * Features: + * - Zero external dependencies (uses Node.js fs module) + * - Content-based deduplication (same hash = same blob) + * - Automatic cleanup of old blobs + * - No network required + */ +export const localBlobStoreProvider: BlobStoreProvider<'local', LocalBlobStoreConfig> = { + type: 'local', + configSchema: LocalBlobStoreSchema, + create: (config, logger) => new LocalBlobStore(config, logger), + metadata: { + displayName: 'Local Filesystem', + description: 'Store blobs on the local filesystem with automatic deduplication', + requiresNetwork: false, + }, +}; diff --git a/dexto/packages/core/src/storage/blob/providers/memory.ts b/dexto/packages/core/src/storage/blob/providers/memory.ts new file mode 100644 index 00000000..2fcbc90c --- /dev/null +++ b/dexto/packages/core/src/storage/blob/providers/memory.ts @@ -0,0 +1,28 @@ +import type { BlobStoreProvider } from '../provider.js'; +import type { InMemoryBlobStoreConfig } from '../schemas.js'; +import { InMemoryBlobStoreSchema } from '../schemas.js'; +import { InMemoryBlobStore } from '../memory-blob-store.js'; + +/** + * Provider for in-memory blob storage. + * + * This provider stores blobs in RAM, making it ideal for testing and + * development. All data is lost when the process exits. + * + * Features: + * - Zero dependencies + * - Extremely fast (no I/O) + * - Configurable size limits + * - No network required + * - Perfect for unit tests + */ +export const inMemoryBlobStoreProvider: BlobStoreProvider<'in-memory', InMemoryBlobStoreConfig> = { + type: 'in-memory', + configSchema: InMemoryBlobStoreSchema, + create: (config, logger) => new InMemoryBlobStore(config, logger), + metadata: { + displayName: 'In-Memory', + description: 'Store blobs in RAM (ephemeral, for testing and development)', + requiresNetwork: false, + }, +}; diff --git a/dexto/packages/core/src/storage/blob/registry.test.ts b/dexto/packages/core/src/storage/blob/registry.test.ts new file mode 100644 index 00000000..966f942d --- /dev/null +++ b/dexto/packages/core/src/storage/blob/registry.test.ts @@ -0,0 +1,548 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { z } from 'zod'; +import { BlobStoreRegistry } from './registry.js'; +import type { BlobStoreProvider } from './provider.js'; +import type { BlobStore } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { StorageErrorCode } from '../error-codes.js'; +import { ErrorScope, ErrorType } from '../../errors/types.js'; + +// Mock logger for testing +const mockLogger: IDextoLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), +} as any; + +// Mock BlobStore implementation +class MockBlobStore implements BlobStore { + constructor( + private config: any, + private logger: IDextoLogger + ) {} + + async store(): Promise { + throw new Error('Not implemented'); + } + async retrieve(): Promise { + throw new Error('Not implemented'); + } + async exists() { + return false; + } + async delete() {} + async cleanup() { + return 0; + } + async getStats() { + return { count: 0, totalSize: 0, backendType: 'mock', storePath: '/mock' }; + } + async listBlobs() { + return []; + } + getStoragePath() { + return undefined; + } + async connect() {} + async disconnect() {} + isConnected() { + return true; + } + getStoreType() { + return this.config.type; + } +} + +// Mock provider configurations and schemas +const MockProviderASchema = z + .object({ + type: z.literal('mock-a'), + endpoint: z.string(), + timeout: z.number().optional().default(5000), + }) + .strict(); + +type MockProviderAConfig = z.output; + +const mockProviderA: BlobStoreProvider<'mock-a', MockProviderAConfig> = { + type: 'mock-a', + configSchema: MockProviderASchema, + create: (config, logger) => new MockBlobStore(config, logger), + metadata: { + displayName: 'Mock Provider A', + description: 'A mock provider for testing', + requiresNetwork: true, + }, +}; + +const MockProviderBSchema = z + .object({ + type: z.literal('mock-b'), + storePath: z.string(), + maxSize: z.number().int().positive().optional().default(1024), + }) + .strict(); + +type MockProviderBConfig = z.output; + +const mockProviderB: BlobStoreProvider<'mock-b', MockProviderBConfig> = { + type: 'mock-b', + configSchema: MockProviderBSchema, + create: (config, logger) => new MockBlobStore(config, logger), + metadata: { + displayName: 'Mock Provider B', + description: 'Another mock provider for testing', + requiresNetwork: false, + }, +}; + +describe('BlobStoreRegistry', () => { + let registry: BlobStoreRegistry; + + beforeEach(() => { + registry = new BlobStoreRegistry(); + }); + + describe('register()', () => { + it('successfully registers a provider', () => { + expect(() => registry.register(mockProviderA)).not.toThrow(); + expect(registry.has('mock-a')).toBe(true); + }); + + it('registers multiple providers with different types', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + expect(registry.has('mock-a')).toBe(true); + expect(registry.has('mock-b')).toBe(true); + expect(registry.getTypes()).toEqual(['mock-a', 'mock-b']); + }); + + it('throws error when registering duplicate provider type', () => { + registry.register(mockProviderA); + + expect(() => registry.register(mockProviderA)).toThrow( + expect.objectContaining({ + code: StorageErrorCode.BLOB_PROVIDER_ALREADY_REGISTERED, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + }) + ); + }); + + it('error message includes provider type for duplicate registration', () => { + registry.register(mockProviderA); + + expect(() => registry.register(mockProviderA)).toThrow(/mock-a/); + }); + }); + + describe('unregister()', () => { + it('successfully unregisters an existing provider', () => { + registry.register(mockProviderA); + expect(registry.has('mock-a')).toBe(true); + + const result = registry.unregister('mock-a'); + expect(result).toBe(true); + expect(registry.has('mock-a')).toBe(false); + }); + + it('returns false when unregistering non-existent provider', () => { + const result = registry.unregister('non-existent'); + expect(result).toBe(false); + }); + + it('returns false when unregistering already unregistered provider', () => { + registry.register(mockProviderA); + registry.unregister('mock-a'); + + const result = registry.unregister('mock-a'); + expect(result).toBe(false); + }); + + it('does not affect other registered providers', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + registry.unregister('mock-a'); + + expect(registry.has('mock-a')).toBe(false); + expect(registry.has('mock-b')).toBe(true); + }); + }); + + describe('get()', () => { + it('returns registered provider by type', () => { + registry.register(mockProviderA); + + const provider = registry.get('mock-a'); + expect(provider).toBe(mockProviderA); + }); + + it('returns undefined for non-existent provider', () => { + const provider = registry.get('non-existent'); + expect(provider).toBeUndefined(); + }); + + it('returns correct provider when multiple are registered', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + const providerA = registry.get('mock-a'); + const providerB = registry.get('mock-b'); + + expect(providerA).toBe(mockProviderA); + expect(providerB).toBe(mockProviderB); + }); + + it('returns undefined after provider is unregistered', () => { + registry.register(mockProviderA); + registry.unregister('mock-a'); + + const provider = registry.get('mock-a'); + expect(provider).toBeUndefined(); + }); + }); + + describe('has()', () => { + it('returns true for registered provider', () => { + registry.register(mockProviderA); + expect(registry.has('mock-a')).toBe(true); + }); + + it('returns false for non-existent provider', () => { + expect(registry.has('non-existent')).toBe(false); + }); + + it('returns false after provider is unregistered', () => { + registry.register(mockProviderA); + registry.unregister('mock-a'); + expect(registry.has('mock-a')).toBe(false); + }); + + it('works correctly with multiple providers', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + expect(registry.has('mock-a')).toBe(true); + expect(registry.has('mock-b')).toBe(true); + expect(registry.has('mock-c')).toBe(false); + }); + }); + + describe('getTypes()', () => { + it('returns empty array when no providers are registered', () => { + expect(registry.getTypes()).toEqual([]); + }); + + it('returns array with single provider type', () => { + registry.register(mockProviderA); + expect(registry.getTypes()).toEqual(['mock-a']); + }); + + it('returns array with all registered provider types', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + const types = registry.getTypes(); + expect(types).toHaveLength(2); + expect(types).toContain('mock-a'); + expect(types).toContain('mock-b'); + }); + + it('updates after unregistering a provider', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + registry.unregister('mock-a'); + + expect(registry.getTypes()).toEqual(['mock-b']); + }); + }); + + describe('getProviders()', () => { + it('returns empty array when no providers are registered', () => { + expect(registry.getProviders()).toEqual([]); + }); + + it('returns array with single provider', () => { + registry.register(mockProviderA); + expect(registry.getProviders()).toEqual([mockProviderA]); + }); + + it('returns array with all registered providers', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + const providers = registry.getProviders(); + expect(providers).toHaveLength(2); + expect(providers).toContain(mockProviderA); + expect(providers).toContain(mockProviderB); + }); + + it('updates after unregistering a provider', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + registry.unregister('mock-a'); + + expect(registry.getProviders()).toEqual([mockProviderB]); + }); + }); + + describe('validateConfig()', () => { + beforeEach(() => { + registry.register(mockProviderA); + registry.register(mockProviderB); + }); + + it('validates config with correct type and structure', () => { + const config = { + type: 'mock-a', + endpoint: 'https://example.com', + }; + + const validated = registry.validateConfig(config); + expect(validated).toEqual({ + type: 'mock-a', + endpoint: 'https://example.com', + timeout: 5000, // default value applied + }); + }); + + it('applies default values from provider schema', () => { + const config = { + type: 'mock-b', + storePath: '/tmp/blobs', + }; + + const validated = registry.validateConfig(config); + expect(validated).toEqual({ + type: 'mock-b', + storePath: '/tmp/blobs', + maxSize: 1024, // default value applied + }); + }); + + it('throws error for missing type field', () => { + const config = { + endpoint: 'https://example.com', + }; + + expect(() => registry.validateConfig(config)).toThrow(); + }); + + it('throws error for unknown provider type', () => { + const config = { + type: 'unknown-provider', + someField: 'value', + }; + + expect(() => registry.validateConfig(config)).toThrow( + expect.objectContaining({ + code: StorageErrorCode.BLOB_PROVIDER_UNKNOWN, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + }) + ); + }); + + it('error message includes unknown type and available types', () => { + const config = { + type: 'unknown-provider', + }; + + try { + registry.validateConfig(config); + expect.fail('Should have thrown error'); + } catch (error: any) { + expect(error.message).toContain('unknown-provider'); + expect(error.context?.availableTypes).toEqual(['mock-a', 'mock-b']); + } + }); + + it('throws validation error for invalid config structure', () => { + const config = { + type: 'mock-a', + endpoint: 123, // should be string + }; + + expect(() => registry.validateConfig(config)).toThrow(); + }); + + it('throws validation error for missing required fields', () => { + const config = { + type: 'mock-a', + // missing required 'endpoint' field + }; + + expect(() => registry.validateConfig(config)).toThrow(); + }); + + it('rejects extra fields in strict schema', () => { + const config = { + type: 'mock-a', + endpoint: 'https://example.com', + extraField: 'not-allowed', + }; + + expect(() => registry.validateConfig(config)).toThrow(); + }); + + it('validates multiple different configs successfully', () => { + const configA = { + type: 'mock-a', + endpoint: 'https://api.example.com', + timeout: 10000, + }; + + const configB = { + type: 'mock-b', + storePath: '/var/blobs', + maxSize: 2048, + }; + + const validatedA = registry.validateConfig(configA); + const validatedB = registry.validateConfig(configB); + + expect(validatedA.type).toBe('mock-a'); + expect(validatedB.type).toBe('mock-b'); + }); + }); + + describe('clear()', () => { + it('removes all registered providers', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + expect(registry.getTypes()).toHaveLength(2); + + registry.clear(); + + expect(registry.getTypes()).toEqual([]); + expect(registry.getProviders()).toEqual([]); + }); + + it('works when no providers are registered', () => { + expect(() => registry.clear()).not.toThrow(); + expect(registry.getTypes()).toEqual([]); + }); + + it('allows registering new providers after clear', () => { + registry.register(mockProviderA); + registry.clear(); + registry.register(mockProviderB); + + expect(registry.has('mock-a')).toBe(false); + expect(registry.has('mock-b')).toBe(true); + }); + + it('allows re-registering same provider after clear', () => { + registry.register(mockProviderA); + registry.clear(); + + expect(() => registry.register(mockProviderA)).not.toThrow(); + expect(registry.has('mock-a')).toBe(true); + }); + }); + + describe('Provider Creation', () => { + it('can create blob store instances using validated config', () => { + registry.register(mockProviderA); + + const config = { + type: 'mock-a', + endpoint: 'https://example.com', + }; + + const validated = registry.validateConfig(config); + const provider = registry.get('mock-a'); + + expect(provider).toBeDefined(); + const store = provider!.create(validated, mockLogger); + expect(store).toBeInstanceOf(MockBlobStore); + expect(store.getStoreType()).toBe('mock-a'); + }); + + it('passes validated config with defaults to create method', () => { + registry.register(mockProviderA); + + const config = { + type: 'mock-a', + endpoint: 'https://example.com', + }; + + const validated = registry.validateConfig(config); + expect(validated.timeout).toBe(5000); // default applied + + const provider = registry.get('mock-a'); + const store = provider!.create(validated, mockLogger); + + expect(store).toBeDefined(); + }); + }); + + describe('Edge Cases', () => { + it('handles empty string as provider type', () => { + const config = { + type: '', + }; + + expect(() => registry.validateConfig(config)).toThrow( + expect.objectContaining({ + code: StorageErrorCode.BLOB_PROVIDER_UNKNOWN, + }) + ); + }); + + it('handles null config gracefully', () => { + expect(() => registry.validateConfig(null)).toThrow(); + }); + + it('handles undefined config gracefully', () => { + expect(() => registry.validateConfig(undefined)).toThrow(); + }); + + it('handles array as config gracefully', () => { + expect(() => registry.validateConfig([])).toThrow(); + }); + + it('handles string as config gracefully', () => { + expect(() => registry.validateConfig('not-an-object')).toThrow(); + }); + + it('handles number as config gracefully', () => { + expect(() => registry.validateConfig(42)).toThrow(); + }); + }); + + describe('Provider Metadata', () => { + it('preserves provider metadata after registration', () => { + registry.register(mockProviderA); + + const provider = registry.get('mock-a'); + expect(provider?.metadata).toEqual({ + displayName: 'Mock Provider A', + description: 'A mock provider for testing', + requiresNetwork: true, + }); + }); + + it('handles providers without metadata', () => { + const providerWithoutMetadata: BlobStoreProvider<'no-meta', { type: 'no-meta' }> = { + type: 'no-meta', + configSchema: z.object({ type: z.literal('no-meta') }).strict(), + create: (config, logger) => new MockBlobStore(config, logger), + }; + + registry.register(providerWithoutMetadata); + const provider = registry.get('no-meta'); + + expect(provider?.metadata).toBeUndefined(); + }); + }); +}); diff --git a/dexto/packages/core/src/storage/blob/registry.ts b/dexto/packages/core/src/storage/blob/registry.ts new file mode 100644 index 00000000..76495c66 --- /dev/null +++ b/dexto/packages/core/src/storage/blob/registry.ts @@ -0,0 +1,59 @@ +import type { BlobStoreProvider } from './provider.js'; +import { StorageError } from '../errors.js'; +import { BaseRegistry, type RegistryErrorFactory } from '../../providers/base-registry.js'; + +/** + * Error factory for blob store registry errors. + * Uses StorageError for consistent error handling. + */ +const blobStoreErrorFactory: RegistryErrorFactory = { + alreadyRegistered: (type: string) => StorageError.blobProviderAlreadyRegistered(type), + notFound: (type: string, availableTypes: string[]) => + StorageError.unknownBlobProvider(type, availableTypes), +}; + +/** + * Registry for blob store providers. + * + * This registry manages available blob store implementations and provides + * runtime validation for configurations. Providers can be registered from + * both core (built-in) and application layers (custom). + * + * The registry follows a global singleton pattern to allow registration + * before configuration loading, while maintaining type safety through + * provider interfaces. + * + * Extends BaseRegistry for common registry functionality. + */ +export class BlobStoreRegistry extends BaseRegistry> { + constructor() { + super(blobStoreErrorFactory); + } + + /** + * Get all registered providers. + * Alias for getAll() for backward compatibility. + * + * @returns Array of providers + */ + getProviders(): BlobStoreProvider[] { + return this.getAll(); + } +} + +/** + * Global singleton registry for blob store providers. + * + * This registry is used by the createBlobStore factory and can be extended + * with custom providers before configuration loading. + * + * Example usage in CLI/server layer: + * ```typescript + * import { blobStoreRegistry } from '@dexto/core'; + * import { s3Provider } from './storage/s3-provider.js'; + * + * // Register custom provider before loading config + * blobStoreRegistry.register(s3Provider); + * ``` + */ +export const blobStoreRegistry = new BlobStoreRegistry(); diff --git a/dexto/packages/core/src/storage/blob/schemas.ts b/dexto/packages/core/src/storage/blob/schemas.ts new file mode 100644 index 00000000..e1242168 --- /dev/null +++ b/dexto/packages/core/src/storage/blob/schemas.ts @@ -0,0 +1,110 @@ +import { z } from 'zod'; + +/** + * Built-in blob store types (core providers only). + * Custom providers registered at runtime are not included in this list. + */ +export const BLOB_STORE_TYPES = ['in-memory', 'local'] as const; +export type BlobStoreType = (typeof BLOB_STORE_TYPES)[number]; + +/** + * In-memory blob store configuration + */ +const InMemoryBlobStoreSchema = z + .object({ + type: z.literal('in-memory').describe('Blob store type identifier'), + maxBlobSize: z + .number() + .int() + .positive() + .optional() + .default(10 * 1024 * 1024) // 10MB + .describe('Maximum size per blob in bytes'), + maxTotalSize: z + .number() + .int() + .positive() + .optional() + .default(100 * 1024 * 1024) // 100MB + .describe('Maximum total storage size in bytes'), + }) + .strict(); + +export type InMemoryBlobStoreConfig = z.output; + +/** + * Local filesystem blob store configuration + */ +const LocalBlobStoreSchema = z + .object({ + type: z.literal('local').describe('Blob store type identifier'), + storePath: z + .string() + .describe( + 'Blob storage directory path (required for local storage - CLI enrichment provides per-agent default)' + ), + maxBlobSize: z + .number() + .int() + .positive() + .optional() + .default(50 * 1024 * 1024) // 50MB + .describe('Maximum size per blob in bytes'), + maxTotalSize: z + .number() + .int() + .positive() + .optional() + .default(1024 * 1024 * 1024) // 1GB + .describe('Maximum total storage size in bytes'), + cleanupAfterDays: z + .number() + .int() + .positive() + .optional() + .default(30) + .describe('Auto-cleanup blobs older than N days'), + }) + .strict(); + +export type LocalBlobStoreConfig = z.output; + +/** + * Blob store configuration schema. + * + * This schema uses `.passthrough()` to accept any provider-specific configuration. + * It only validates that a `type` field exists as a string. + * + * Detailed validation happens at runtime via blobStoreRegistry.validateConfig(), + * which looks up the registered provider and validates against its specific schema. + * + * This approach allows: + * - Custom providers to be registered at the CLI/server layer + * - Each provider to define its own configuration structure + * - Type safety through the provider registry pattern + * + * Example flow: + * 1. Config passes this schema (basic structure check) + * 2. blobStoreRegistry.validateConfig(config) validates against provider schema + * 3. Provider's create() method receives validated, typed config + */ +export const BlobStoreConfigSchema = z + .object({ + type: z.string().describe('Blob store provider type'), + }) + .passthrough() + .describe('Blob store configuration (validated at runtime by provider registry)'); + +/** + * Blob store configuration type. + * + * Union type including built-in providers (local, in-memory) and a catch-all + * for custom providers registered at runtime. + */ +export type BlobStoreConfig = + | InMemoryBlobStoreConfig + | LocalBlobStoreConfig + | { type: string; [key: string]: any }; // Custom provider configs + +// Export individual schemas for use in providers +export { InMemoryBlobStoreSchema, LocalBlobStoreSchema }; diff --git a/dexto/packages/core/src/storage/blob/types.ts b/dexto/packages/core/src/storage/blob/types.ts new file mode 100644 index 00000000..8086afeb --- /dev/null +++ b/dexto/packages/core/src/storage/blob/types.ts @@ -0,0 +1,163 @@ +/** + * Core types for Blob Storage + * + * Blob storage handles large, unstructured data using various backends. + * This module is part of the storage system. + */ + +/** + * Input data for blob storage - supports various formats + */ +export type BlobInput = + | string // base64, data URI, or file path + | Uint8Array + | Buffer + | ArrayBuffer; + +/** + * Metadata associated with a stored blob + */ +export interface BlobMetadata { + mimeType?: string | undefined; + originalName?: string | undefined; + createdAt?: Date | undefined; + source?: 'tool' | 'user' | 'system' | undefined; + size?: number | undefined; +} + +/** + * Complete blob information including stored metadata + */ +export interface StoredBlobMetadata { + id: string; + mimeType: string; + originalName?: string | undefined; + createdAt: Date; + size: number; + hash: string; + source?: 'tool' | 'user' | 'system' | undefined; +} + +/** + * Reference to a stored blob + */ +export interface BlobReference { + id: string; + uri: string; // blob:id format for compatibility + metadata: StoredBlobMetadata; +} + +/** + * Retrieved blob data in requested format + */ +export type BlobData = + | { format: 'base64'; data: string; metadata: StoredBlobMetadata } + | { format: 'buffer'; data: Buffer; metadata: StoredBlobMetadata } + | { format: 'path'; data: string; metadata: StoredBlobMetadata } + | { format: 'stream'; data: NodeJS.ReadableStream; metadata: StoredBlobMetadata } + | { format: 'url'; data: string; metadata: StoredBlobMetadata }; + +/** + * Storage statistics for monitoring and management + */ +export interface BlobStats { + count: number; + totalSize: number; + backendType: string; + storePath: string; +} + +/** + * BlobStore interface for storing and retrieving large, unstructured data. + * All implementations must provide these methods for blob operations. + * + * This interface follows the storage module conventions where: + * - Interface name: BlobStore + * - Implementation names: MemoryBlobStore, LocalBlobStore, etc. + * - Lifecycle methods: connect(), disconnect(), isConnected() + * - Type identifier: getStoreType() + */ +export interface BlobStore { + /** + * Store blob data and return a reference. + * @param input - The blob data to store (string, Uint8Array, Buffer, ArrayBuffer) + * @param metadata - Optional metadata to associate with the blob + * @returns Promise resolving to a BlobReference with id, uri, and metadata + */ + store(input: BlobInput, metadata?: BlobMetadata): Promise; + + /** + * Retrieve blob data in the specified format. + * @param reference - The blob reference (id or uri) + * @param format - The desired output format (base64, buffer, path, stream, url) + * @returns Promise resolving to BlobData with the requested format + */ + retrieve( + reference: string, + format?: 'base64' | 'buffer' | 'path' | 'stream' | 'url' + ): Promise; + + /** + * Check if a blob exists. + * @param reference - The blob reference (id or uri) + * @returns Promise resolving to true if the blob exists, false otherwise + */ + exists(reference: string): Promise; + + /** + * Delete a blob. + * @param reference - The blob reference (id or uri) + * @returns Promise resolving when the blob is deleted + */ + delete(reference: string): Promise; + + /** + * Cleanup old blobs based on age. + * @param olderThan - Optional date to delete blobs created before + * @returns Promise resolving to the number of blobs deleted + */ + cleanup(olderThan?: Date): Promise; + + /** + * Get storage statistics. + * @returns Promise resolving to BlobStats with count, size, type, and path info + */ + getStats(): Promise; + + /** + * List all blob references (for resource enumeration). + * @returns Promise resolving to an array of BlobReferences + */ + listBlobs(): Promise; + + /** + * Get the local filesystem storage path for this store, if applicable. + * Used to prevent conflicts with filesystem resource scanning. + * @returns The storage path string, or undefined for remote stores (S3, Azure, etc.) + */ + getStoragePath(): string | undefined; + + /** + * Connect to the blob store and initialize resources. + * @returns Promise resolving when the store is ready for operations + */ + connect(): Promise; + + /** + * Disconnect from the blob store and cleanup resources. + * @returns Promise resolving when the store is fully disconnected + */ + disconnect(): Promise; + + /** + * Check if the blob store is currently connected. + * @returns true if connected, false otherwise + */ + isConnected(): boolean; + + /** + * Get the type identifier for this blob store implementation. + * @returns A string identifying the store type (e.g., 'memory', 'local', 's3') + */ + getStoreType(): string; +} diff --git a/dexto/packages/core/src/storage/cache/factory.ts b/dexto/packages/core/src/storage/cache/factory.ts new file mode 100644 index 00000000..89425621 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/factory.ts @@ -0,0 +1,54 @@ +import type { Cache } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { cacheRegistry } from './registry.js'; + +/** + * Create a cache based on configuration using the provider registry. + * + * This factory function: + * 1. Validates the configuration against the registered provider's schema + * 2. Looks up the provider in the registry + * 3. Calls the provider's create method to instantiate the cache + * + * The configuration type is determined at runtime by the 'type' field, + * which must match a registered provider. Custom providers can be registered + * via cacheRegistry.register() before calling this function. + * + * @param config - Cache configuration with a 'type' discriminator + * @param logger - Logger instance for the cache + * @returns A Cache implementation + * @throws Error if the provider type is not registered or validation fails + * + * @example + * ```typescript + * // Using built-in provider + * const cache = await createCache({ type: 'redis', host: 'localhost' }, logger); + * + * // Using custom provider (registered beforehand) + * import { cacheRegistry } from '@dexto/core'; + * import { memcachedProvider } from './storage/memcached-provider.js'; + * + * cacheRegistry.register(memcachedProvider); + * const cache = await createCache({ type: 'memcached', servers: ['...'] }, logger); + * ``` + */ +export async function createCache( + config: { type: string; [key: string]: any }, + logger: IDextoLogger +): Promise { + // Validate config against provider schema and get provider + const validatedConfig = cacheRegistry.validateConfig(config); + const provider = cacheRegistry.get(validatedConfig.type); + + if (!provider) { + // This should never happen after validateConfig, but handle it defensively + throw new Error(`Provider '${validatedConfig.type}' not found in registry`); + } + + // Log which provider is being used + const providerName = provider.metadata?.displayName || validatedConfig.type; + logger.info(`Using ${providerName} cache`); + + // Create and return the cache instance (may be async for lazy-loaded dependencies) + return provider.create(validatedConfig, logger); +} diff --git a/dexto/packages/core/src/storage/cache/index.ts b/dexto/packages/core/src/storage/cache/index.ts new file mode 100644 index 00000000..7ec3db39 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/index.ts @@ -0,0 +1,74 @@ +/** + * Cache Module + * + * This module provides a flexible caching system with support for + * multiple backends through a provider pattern. + * + * ## Built-in Providers + * - `in-memory`: Store data in RAM (for testing/development) + * - `redis`: Store data in Redis server + * + * ## Custom Providers + * Custom providers (e.g., Memcached, Hazelcast) can be registered at the + * CLI/server layer before configuration loading. + * + * ## Usage + * + * ### Using built-in providers + * ```typescript + * import { createCache } from '@dexto/core'; + * + * const cache = await createCache({ type: 'redis', host: 'localhost' }, logger); + * ``` + * + * ### Registering custom providers + * ```typescript + * import { cacheRegistry, type CacheProvider } from '@dexto/core'; + * + * const memcachedProvider: CacheProvider<'memcached', MemcachedConfig> = { + * type: 'memcached', + * configSchema: MemcachedConfigSchema, + * create: (config, logger) => new MemcachedCache(config, logger), + * }; + * + * cacheRegistry.register(memcachedProvider); + * const cache = await createCache({ type: 'memcached', servers: ['...'] }, logger); + * ``` + */ + +// Import built-in providers +import { cacheRegistry } from './registry.js'; +import { inMemoryCacheProvider, redisCacheProvider } from './providers/index.js'; + +// Register built-in providers on module load +// This ensures they're available when importing from @dexto/core +// Guard against duplicate registration when module is imported multiple times +if (!cacheRegistry.has('in-memory')) { + cacheRegistry.register(inMemoryCacheProvider); +} +if (!cacheRegistry.has('redis')) { + cacheRegistry.register(redisCacheProvider); +} + +// Export public API +export { createCache } from './factory.js'; +export { cacheRegistry, CacheRegistry } from './registry.js'; +export type { CacheProvider } from './provider.js'; + +// Export types and interfaces +export type { Cache } from './types.js'; + +// Export schemas and config types +export { + CACHE_TYPES, + CacheConfigSchema, + InMemoryCacheSchema, + RedisCacheSchema, + type CacheType, + type CacheConfig, + type InMemoryCacheConfig, + type RedisCacheConfig, +} from './schemas.js'; + +// Export concrete implementations (for custom usage and external providers) +export { MemoryCacheStore } from './memory-cache-store.js'; diff --git a/dexto/packages/core/src/storage/cache/memory-cache-store.ts b/dexto/packages/core/src/storage/cache/memory-cache-store.ts new file mode 100644 index 00000000..6c87a25f --- /dev/null +++ b/dexto/packages/core/src/storage/cache/memory-cache-store.ts @@ -0,0 +1,99 @@ +import type { Cache } from './types.js'; +import { StorageError } from '../errors.js'; + +/** + * In-memory cache store for development and testing. + * Supports TTL for automatic cleanup of temporary data. + * Data is lost when the process restarts. + */ +export class MemoryCacheStore implements Cache { + private data = new Map(); + private ttls = new Map(); + private connected = false; + + constructor() {} + + async connect(): Promise { + this.connected = true; + } + + async disconnect(): Promise { + this.connected = false; + this.data.clear(); + this.ttls.clear(); + } + + isConnected(): boolean { + return this.connected; + } + + getStoreType(): string { + return 'memory'; + } + + async get(key: string): Promise { + this.checkConnection(); + try { + this.checkTTL(key); + return this.data.get(key); + } catch (error) { + throw StorageError.readFailed( + 'get', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async set(key: string, value: T, ttlSeconds?: number): Promise { + this.checkConnection(); + try { + this.data.set(key, value); + + if (ttlSeconds) { + this.ttls.set(key, Date.now() + ttlSeconds * 1000); + } else { + this.ttls.delete(key); + } + } catch (error) { + throw StorageError.writeFailed( + 'set', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async delete(key: string): Promise { + this.checkConnection(); + this.data.delete(key); + this.ttls.delete(key); + } + + // Helper methods + private checkConnection(): void { + if (!this.connected) { + throw StorageError.notConnected('MemoryCacheStore'); + } + } + + private checkTTL(key: string): void { + const expiry = this.ttls.get(key); + if (expiry && Date.now() > expiry) { + this.data.delete(key); + this.ttls.delete(key); + } + } + + // Development helpers + async clear(): Promise { + this.data.clear(); + this.ttls.clear(); + } + + async dump(): Promise<{ data: Record }> { + return { + data: Object.fromEntries(this.data.entries()), + }; + } +} diff --git a/dexto/packages/core/src/storage/cache/provider.ts b/dexto/packages/core/src/storage/cache/provider.ts new file mode 100644 index 00000000..87336724 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/provider.ts @@ -0,0 +1,60 @@ +import type { z } from 'zod'; +import type { Cache } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; + +/** + * Provider interface for creating cache instances. + * + * This interface uses TypeScript generics to enforce type safety: + * - TType: The literal type string (e.g., 'redis', 'in-memory') + * - TConfig: The configuration type with discriminator { type: TType } + * + * This ensures that the provider type matches the config type discriminator, + * providing compile-time safety for provider implementations. + */ +export interface CacheProvider< + TType extends string = string, + TConfig extends { type: TType } = any, +> { + /** + * Unique identifier for this provider (e.g., 'redis', 'in-memory', 'memcached'). + * Must match the 'type' field in the configuration. + */ + type: TType; + + /** + * Zod schema for validating provider-specific configuration. + * The schema must output TConfig type. + * + * Note: Uses z.ZodType with relaxed generics to allow input/output type variance. + * This is necessary because Zod schemas with `.optional().default()` have + * input types that include undefined, but output types that don't. + */ + configSchema: z.ZodType; + + /** + * Factory function to create a Cache instance. + * + * Unlike blob store providers (which are sync), cache providers may return + * a Promise to support lazy loading of optional dependencies (e.g., ioredis). + * + * @param config - Validated configuration specific to this provider + * @param logger - Logger instance for the cache + * @returns A Cache implementation (or Promise for async providers) + */ + create(config: TConfig, logger: IDextoLogger): Cache | Promise; + + /** + * Optional metadata for documentation, UIs, and discovery. + */ + metadata?: { + /** Human-readable name (e.g., "Redis", "Memcached") */ + displayName: string; + /** Brief description of this cache backend */ + description: string; + /** Whether this provider requires network connectivity */ + requiresNetwork?: boolean; + /** Whether this provider supports TTL (time-to-live) */ + supportsTTL?: boolean; + }; +} diff --git a/dexto/packages/core/src/storage/cache/providers/index.ts b/dexto/packages/core/src/storage/cache/providers/index.ts new file mode 100644 index 00000000..a138ba43 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/providers/index.ts @@ -0,0 +1,8 @@ +/** + * Built-in cache providers. + * + * These providers are automatically registered when importing from @dexto/core. + */ + +export { inMemoryCacheProvider } from './memory.js'; +export { redisCacheProvider } from './redis.js'; diff --git a/dexto/packages/core/src/storage/cache/providers/memory.ts b/dexto/packages/core/src/storage/cache/providers/memory.ts new file mode 100644 index 00000000..550f67d4 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/providers/memory.ts @@ -0,0 +1,29 @@ +import type { CacheProvider } from '../provider.js'; +import type { InMemoryCacheConfig } from '../schemas.js'; +import { InMemoryCacheSchema } from '../schemas.js'; +import { MemoryCacheStore } from '../memory-cache-store.js'; + +/** + * Provider for in-memory cache storage. + * + * This provider stores data in RAM and is ideal for development, + * testing, and ephemeral use cases where persistence is not required. + * + * Features: + * - Zero external dependencies + * - Fast in-memory operations + * - TTL support for automatic expiration + * - No network required + * - Data is lost on restart + */ +export const inMemoryCacheProvider: CacheProvider<'in-memory', InMemoryCacheConfig> = { + type: 'in-memory', + configSchema: InMemoryCacheSchema, + create: (_config, _logger) => new MemoryCacheStore(), + metadata: { + displayName: 'In-Memory', + description: 'Store cache data in RAM (ephemeral, for testing and development)', + requiresNetwork: false, + supportsTTL: true, + }, +}; diff --git a/dexto/packages/core/src/storage/cache/providers/redis.ts b/dexto/packages/core/src/storage/cache/providers/redis.ts new file mode 100644 index 00000000..d9bf1896 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/providers/redis.ts @@ -0,0 +1,48 @@ +import type { CacheProvider } from '../provider.js'; +import type { RedisCacheConfig } from '../schemas.js'; +import { RedisCacheSchema } from '../schemas.js'; +import { StorageError } from '../../errors.js'; + +/** + * Provider for Redis cache storage. + * + * This provider stores data in a Redis server using the ioredis package. + * It's ideal for production deployments requiring scalability, persistence, + * and multi-machine access. + * + * Features: + * - Native TTL support + * - Connection pooling + * - Pub/sub capabilities (via additional methods) + * - Suitable for distributed deployments + * + * Note: ioredis is an optional dependency. Install it with: + * npm install ioredis + */ +export const redisCacheProvider: CacheProvider<'redis', RedisCacheConfig> = { + type: 'redis', + configSchema: RedisCacheSchema, + create: async (config, logger) => { + try { + const module = await import('../redis-store.js'); + logger.info(`Connecting to Redis at ${config.host || config.url}`); + return new module.RedisStore(config, logger); + } catch (error: unknown) { + const err = error as NodeJS.ErrnoException; + if (err.code === 'ERR_MODULE_NOT_FOUND') { + throw StorageError.dependencyNotInstalled( + 'Redis', + 'ioredis', + 'npm install ioredis' + ); + } + throw error; + } + }, + metadata: { + displayName: 'Redis', + description: 'Production Redis cache with TTL and pub/sub support', + requiresNetwork: true, + supportsTTL: true, + }, +}; diff --git a/dexto/packages/core/src/storage/cache/redis-store.ts b/dexto/packages/core/src/storage/cache/redis-store.ts new file mode 100644 index 00000000..b9e74d14 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/redis-store.ts @@ -0,0 +1,182 @@ +import { Redis } from 'ioredis'; +import type { Cache } from './types.js'; +import type { RedisCacheConfig } from './schemas.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { StorageError } from '../errors.js'; + +/** + * Redis cache store for production cache operations. + * Implements the Cache interface with connection pooling and optimizations. + * EXPERIMENTAL - NOT FULLY TESTED YET + */ +export class RedisStore implements Cache { + private redis: Redis | null = null; + private connected = false; + private logger: IDextoLogger; + + constructor( + private config: RedisCacheConfig, + logger: IDextoLogger + ) { + this.logger = logger.createChild(DextoLogComponent.STORAGE); + } + + async connect(): Promise { + if (this.connected) return; + + this.redis = new Redis({ + ...(this.config.host && { host: this.config.host }), + ...(this.config.port && { port: this.config.port }), + ...(this.config.password && { password: this.config.password }), + db: this.config.database || 0, + family: 4, // IPv4 by default + ...(this.config.connectionTimeoutMillis && { + connectTimeout: this.config.connectionTimeoutMillis, + }), + ...(this.config.connectionTimeoutMillis && { + commandTimeout: this.config.connectionTimeoutMillis, + }), + maxRetriesPerRequest: 3, + lazyConnect: true, + ...this.config.options, + }); + + // Set up error handling + this.redis.on('error', (error) => { + console.error('Redis connection error:', error); + }); + + this.redis.on('connect', () => { + this.connected = true; + }); + + this.redis.on('close', () => { + this.connected = false; + }); + + await this.redis.connect(); + } + + async disconnect(): Promise { + if (this.redis) { + await this.redis.quit(); + this.redis = null; + } + this.connected = false; + } + + isConnected(): boolean { + return this.connected && this.redis?.status === 'ready'; + } + + getStoreType(): string { + return 'redis'; + } + + // Core operations + async get(key: string): Promise { + this.checkConnection(); + try { + const value = await this.redis!.get(key); + return value ? JSON.parse(value) : undefined; + } catch (error) { + throw StorageError.readFailed( + 'get', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async set(key: string, value: T, ttlSeconds?: number): Promise { + this.checkConnection(); + try { + const serialized = JSON.stringify(value); + + if (ttlSeconds) { + await this.redis!.setex(key, ttlSeconds, serialized); + } else { + await this.redis!.set(key, serialized); + } + } catch (error) { + throw StorageError.writeFailed( + 'set', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async delete(key: string): Promise { + this.checkConnection(); + try { + await this.redis!.del(key); + } catch (error) { + throw StorageError.deleteFailed( + 'delete', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + // Redis-specific optimizations + async mget(keys: string[]): Promise<(T | undefined)[]> { + this.checkConnection(); + if (keys.length === 0) return []; + + const values = await this.redis!.mget(...keys); + return values.map((value) => (value ? JSON.parse(value) : undefined)); + } + + async mset(entries: [string, T][]): Promise { + this.checkConnection(); + if (entries.length === 0) return; + + const pipeline = this.redis!.pipeline(); + for (const [key, value] of entries) { + pipeline.set(key, JSON.stringify(value)); + } + await pipeline.exec(); + } + + async exists(key: string): Promise { + this.checkConnection(); + const result = await this.redis!.exists(key); + return result === 1; + } + + async expire(key: string, ttlSeconds: number): Promise { + this.checkConnection(); + await this.redis!.expire(key, ttlSeconds); + } + + // Cache-specific operations + async increment(key: string, by: number = 1): Promise { + this.checkConnection(); + return await this.redis!.incrby(key, by); + } + + async decrement(key: string, by: number = 1): Promise { + this.checkConnection(); + return await this.redis!.decrby(key, by); + } + + private checkConnection(): void { + if (!this.connected || !this.redis || this.redis.status !== 'ready') { + throw StorageError.notConnected('RedisStore'); + } + } + + // Maintenance operations + async flushdb(): Promise { + this.checkConnection(); + await this.redis!.flushdb(); + } + + async info(): Promise { + this.checkConnection(); + return await this.redis!.info(); + } +} diff --git a/dexto/packages/core/src/storage/cache/registry.test.ts b/dexto/packages/core/src/storage/cache/registry.test.ts new file mode 100644 index 00000000..9bfd8707 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/registry.test.ts @@ -0,0 +1,215 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { z } from 'zod'; +import { CacheRegistry } from './registry.js'; +import type { CacheProvider } from './provider.js'; +import type { Cache } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { StorageErrorCode } from '../error-codes.js'; +import { ErrorScope, ErrorType } from '../../errors/types.js'; + +// Mock logger for testing +const mockLogger: IDextoLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), +} as any; + +// Mock Cache implementation +class MockCache implements Cache { + constructor(private config: any) {} + + async get(): Promise { + return undefined; + } + async set(): Promise {} + async delete(): Promise {} + async connect(): Promise {} + async disconnect(): Promise {} + isConnected(): boolean { + return true; + } + getStoreType(): string { + return this.config.type; + } +} + +// Mock provider configurations and schemas +const MockProviderASchema = z + .object({ + type: z.literal('mock-cache-a'), + maxSize: z.number().optional().default(100), + }) + .strict(); + +type MockProviderAConfig = z.output; + +const mockProviderA: CacheProvider<'mock-cache-a', MockProviderAConfig> = { + type: 'mock-cache-a', + configSchema: MockProviderASchema, + create: (config, _logger) => new MockCache(config), + metadata: { + displayName: 'Mock Cache A', + description: 'A mock cache provider for testing', + requiresNetwork: false, + supportsTTL: true, + }, +}; + +const MockProviderBSchema = z + .object({ + type: z.literal('mock-cache-b'), + host: z.string(), + port: z.number().optional().default(6379), + }) + .strict(); + +type MockProviderBConfig = z.output; + +const mockProviderB: CacheProvider<'mock-cache-b', MockProviderBConfig> = { + type: 'mock-cache-b', + configSchema: MockProviderBSchema, + create: (config, _logger) => new MockCache(config), + metadata: { + displayName: 'Mock Cache B', + description: 'Another mock cache provider', + requiresNetwork: true, + supportsTTL: true, + }, +}; + +describe('CacheRegistry', () => { + let registry: CacheRegistry; + + beforeEach(() => { + registry = new CacheRegistry(); + }); + + describe('register()', () => { + it('successfully registers a provider', () => { + expect(() => registry.register(mockProviderA)).not.toThrow(); + expect(registry.has('mock-cache-a')).toBe(true); + }); + + it('registers multiple providers with different types', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + expect(registry.has('mock-cache-a')).toBe(true); + expect(registry.has('mock-cache-b')).toBe(true); + }); + + it('throws error when registering duplicate provider type', () => { + registry.register(mockProviderA); + + expect(() => registry.register(mockProviderA)).toThrow( + expect.objectContaining({ + code: StorageErrorCode.CACHE_PROVIDER_ALREADY_REGISTERED, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + }) + ); + }); + }); + + describe('validateConfig()', () => { + beforeEach(() => { + registry.register(mockProviderA); + registry.register(mockProviderB); + }); + + it('validates config with correct type and structure', () => { + const config = { + type: 'mock-cache-a', + }; + + const validated = registry.validateConfig(config); + expect(validated).toEqual({ + type: 'mock-cache-a', + maxSize: 100, // default value applied + }); + }); + + it('throws error for unknown provider type', () => { + const config = { + type: 'unknown-provider', + someField: 'value', + }; + + expect(() => registry.validateConfig(config)).toThrow( + expect.objectContaining({ + code: StorageErrorCode.CACHE_PROVIDER_UNKNOWN, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + }) + ); + }); + + it('error message includes available types', () => { + const config = { + type: 'unknown-provider', + }; + + try { + registry.validateConfig(config); + expect.fail('Should have thrown error'); + } catch (error: any) { + expect(error.message).toContain('unknown-provider'); + expect(error.context?.availableTypes).toContain('mock-cache-a'); + expect(error.context?.availableTypes).toContain('mock-cache-b'); + } + }); + }); + + describe('getProviders()', () => { + it('returns all registered providers', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + const providers = registry.getProviders(); + expect(providers).toHaveLength(2); + expect(providers).toContain(mockProviderA); + expect(providers).toContain(mockProviderB); + }); + }); + + describe('Provider Creation', () => { + it('can create cache instances using validated config', async () => { + registry.register(mockProviderA); + + const config = { + type: 'mock-cache-a', + }; + + const validated = registry.validateConfig(config); + const provider = registry.get('mock-cache-a'); + + expect(provider).toBeDefined(); + const store = await provider!.create(validated, mockLogger); + expect(store).toBeInstanceOf(MockCache); + if (!(store instanceof MockCache)) { + throw new Error('Expected MockCache instance'); + } + expect(store.getStoreType()).toBe('mock-cache-a'); + }); + }); + + describe('Provider Metadata', () => { + it('preserves provider metadata after registration', () => { + registry.register(mockProviderA); + + const provider = registry.get('mock-cache-a'); + expect(provider?.metadata).toEqual({ + displayName: 'Mock Cache A', + description: 'A mock cache provider for testing', + requiresNetwork: false, + supportsTTL: true, + }); + }); + }); +}); diff --git a/dexto/packages/core/src/storage/cache/registry.ts b/dexto/packages/core/src/storage/cache/registry.ts new file mode 100644 index 00000000..dda114d7 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/registry.ts @@ -0,0 +1,59 @@ +import type { CacheProvider } from './provider.js'; +import { StorageError } from '../errors.js'; +import { BaseRegistry, type RegistryErrorFactory } from '../../providers/base-registry.js'; + +/** + * Error factory for cache registry errors. + * Uses StorageError for consistent error handling. + */ +const cacheErrorFactory: RegistryErrorFactory = { + alreadyRegistered: (type: string) => StorageError.cacheProviderAlreadyRegistered(type), + notFound: (type: string, availableTypes: string[]) => + StorageError.unknownCacheProvider(type, availableTypes), +}; + +/** + * Registry for cache providers. + * + * This registry manages available cache implementations and provides + * runtime validation for configurations. Providers can be registered from + * both core (built-in) and application layers (custom). + * + * The registry follows a global singleton pattern to allow registration + * before configuration loading, while maintaining type safety through + * provider interfaces. + * + * Extends BaseRegistry for common registry functionality. + */ +export class CacheRegistry extends BaseRegistry> { + constructor() { + super(cacheErrorFactory); + } + + /** + * Get all registered providers. + * Alias for getAll() for backward compatibility. + * + * @returns Array of providers + */ + getProviders(): CacheProvider[] { + return this.getAll(); + } +} + +/** + * Global singleton registry for cache providers. + * + * This registry is used by the createCache factory and can be extended + * with custom providers before configuration loading. + * + * Example usage in CLI/server layer: + * ```typescript + * import { cacheRegistry } from '@dexto/core'; + * import { memcachedProvider } from './storage/memcached-provider.js'; + * + * // Register custom provider before loading config + * cacheRegistry.register(memcachedProvider); + * ``` + */ +export const cacheRegistry = new CacheRegistry(); diff --git a/dexto/packages/core/src/storage/cache/schemas.ts b/dexto/packages/core/src/storage/cache/schemas.ts new file mode 100644 index 00000000..8d351759 --- /dev/null +++ b/dexto/packages/core/src/storage/cache/schemas.ts @@ -0,0 +1,77 @@ +import { z } from 'zod'; +import { EnvExpandedString } from '@core/utils/result.js'; +import { StorageErrorCode } from '../error-codes.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; + +export const CACHE_TYPES = ['in-memory', 'redis'] as const; +export type CacheType = (typeof CACHE_TYPES)[number]; + +const BaseCacheSchema = z.object({ + maxConnections: z.number().int().positive().optional().describe('Maximum connections'), + idleTimeoutMillis: z + .number() + .int() + .positive() + .optional() + .describe('Idle timeout in milliseconds'), + connectionTimeoutMillis: z + .number() + .int() + .positive() + .optional() + .describe('Connection timeout in milliseconds'), + options: z.record(z.any()).optional().describe('Backend-specific options'), +}); + +// Memory cache - minimal configuration +export const InMemoryCacheSchema = BaseCacheSchema.extend({ + type: z.literal('in-memory'), + // In-memory cache doesn't need connection options, but inherits pool options for consistency +}).strict(); + +export type InMemoryCacheConfig = z.output; + +// Redis cache configuration +export const RedisCacheSchema = BaseCacheSchema.extend({ + type: z.literal('redis'), + url: EnvExpandedString().optional().describe('Redis connection URL (redis://...)'), + host: z.string().optional().describe('Redis host'), + port: z.number().int().positive().optional().describe('Redis port'), + password: z.string().optional().describe('Redis password'), + database: z.number().int().nonnegative().optional().describe('Redis database number'), +}).strict(); + +export type RedisCacheConfig = z.output; + +// Cache configuration using discriminated union +export const CacheConfigSchema = z + .discriminatedUnion('type', [InMemoryCacheSchema, RedisCacheSchema], { + errorMap: (issue, ctx) => { + if (issue.code === z.ZodIssueCode.invalid_union_discriminator) { + return { + message: `Invalid cache type. Expected 'in-memory' or 'redis'.`, + }; + } + return { message: ctx.defaultError }; + }, + }) + .describe('Cache configuration') + .superRefine((data, ctx) => { + // Validate Redis requirements + if (data.type === 'redis') { + if (!data.url && !data.host) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Redis cache requires either 'url' or 'host' to be specified", + path: ['url'], + params: { + code: StorageErrorCode.CONNECTION_CONFIG_MISSING, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + }, + }); + } + } + }); + +export type CacheConfig = z.output; diff --git a/dexto/packages/core/src/storage/cache/types.ts b/dexto/packages/core/src/storage/cache/types.ts new file mode 100644 index 00000000..27dd536b --- /dev/null +++ b/dexto/packages/core/src/storage/cache/types.ts @@ -0,0 +1,16 @@ +/** + * Fast, ephemeral storage for temporary data and performance optimization. + * Supports TTL for automatic cleanup of temporary data. + */ +export interface Cache { + // Basic operations + get(key: string): Promise; + set(key: string, value: T, ttlSeconds?: number): Promise; + delete(key: string): Promise; + + // Connection management + connect(): Promise; + disconnect(): Promise; + isConnected(): boolean; + getStoreType(): string; +} diff --git a/dexto/packages/core/src/storage/database/factory.ts b/dexto/packages/core/src/storage/database/factory.ts new file mode 100644 index 00000000..7e3cd917 --- /dev/null +++ b/dexto/packages/core/src/storage/database/factory.ts @@ -0,0 +1,56 @@ +import type { Database } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { databaseRegistry } from './registry.js'; + +/** + * Create a database based on configuration using the provider registry. + * + * This factory function: + * 1. Validates the configuration against the registered provider's schema + * 2. Looks up the provider in the registry + * 3. Calls the provider's create method to instantiate the database + * + * The configuration type is determined at runtime by the 'type' field, + * which must match a registered provider. Custom providers can be registered + * via databaseRegistry.register() before calling this function. + * + * Database paths are provided via CLI enrichment layer (for sqlite). + * + * @param config - Database configuration with a 'type' discriminator + * @param logger - Logger instance for the database + * @returns A Database implementation + * @throws Error if the provider type is not registered or validation fails + * + * @example + * ```typescript + * // Using built-in provider + * const db = await createDatabase({ type: 'sqlite', path: '/tmp/data.db' }, logger); + * + * // Using custom provider (registered beforehand) + * import { databaseRegistry } from '@dexto/core'; + * import { mongoProvider } from './storage/mongo-provider.js'; + * + * databaseRegistry.register(mongoProvider); + * const db = await createDatabase({ type: 'mongodb', uri: '...' }, logger); + * ``` + */ +export async function createDatabase( + config: { type: string; [key: string]: any }, + logger: IDextoLogger +): Promise { + // Validate config against provider schema and get provider + const validatedConfig = databaseRegistry.validateConfig(config); + const provider = databaseRegistry.get(validatedConfig.type); + + if (!provider) { + // This should never happen after validateConfig, but handle it defensively + throw new Error(`Provider '${validatedConfig.type}' not found in registry`); + } + + // Log which provider is being used + const providerName = provider.metadata?.displayName || validatedConfig.type; + logger.info(`Using ${providerName} database`); + + // Create and return the database instance (may be async for lazy-loaded dependencies) + return provider.create(validatedConfig, logger); +} diff --git a/dexto/packages/core/src/storage/database/index.ts b/dexto/packages/core/src/storage/database/index.ts new file mode 100644 index 00000000..85060851 --- /dev/null +++ b/dexto/packages/core/src/storage/database/index.ts @@ -0,0 +1,84 @@ +/** + * Database Module + * + * This module provides a flexible database system with support for + * multiple backends through a provider pattern. + * + * ## Built-in Providers + * - `in-memory`: Store data in RAM (for testing/development) + * - `sqlite`: Store data in a local SQLite file + * - `postgres`: Store data in PostgreSQL server + * + * ## Custom Providers + * Custom providers (e.g., MongoDB, DynamoDB) can be registered at the + * CLI/server layer before configuration loading. + * + * ## Usage + * + * ### Using built-in providers + * ```typescript + * import { createDatabase } from '@dexto/core'; + * + * const db = await createDatabase({ type: 'sqlite', path: '/tmp/data.db' }, logger); + * ``` + * + * ### Registering custom providers + * ```typescript + * import { databaseRegistry, type DatabaseProvider } from '@dexto/core'; + * + * const mongoProvider: DatabaseProvider<'mongodb', MongoConfig> = { + * type: 'mongodb', + * configSchema: MongoConfigSchema, + * create: (config, logger) => new MongoDatabase(config, logger), + * }; + * + * databaseRegistry.register(mongoProvider); + * const db = await createDatabase({ type: 'mongodb', uri: '...' }, logger); + * ``` + */ + +// Import built-in providers +import { databaseRegistry } from './registry.js'; +import { + inMemoryDatabaseProvider, + sqliteDatabaseProvider, + postgresDatabaseProvider, +} from './providers/index.js'; + +// Register built-in providers on module load +// This ensures they're available when importing from @dexto/core +// Guard against duplicate registration when module is imported multiple times +if (!databaseRegistry.has('in-memory')) { + databaseRegistry.register(inMemoryDatabaseProvider); +} +if (!databaseRegistry.has('sqlite')) { + databaseRegistry.register(sqliteDatabaseProvider); +} +if (!databaseRegistry.has('postgres')) { + databaseRegistry.register(postgresDatabaseProvider); +} + +// Export public API +export { createDatabase } from './factory.js'; +export { databaseRegistry, DatabaseRegistry } from './registry.js'; +export type { DatabaseProvider } from './provider.js'; + +// Export types and interfaces +export type { Database } from './types.js'; + +// Export schemas and config types +export { + DATABASE_TYPES, + DatabaseConfigSchema, + InMemoryDatabaseSchema, + SqliteDatabaseSchema, + PostgresDatabaseSchema, + type DatabaseType, + type DatabaseConfig, + type InMemoryDatabaseConfig, + type SqliteDatabaseConfig, + type PostgresDatabaseConfig, +} from './schemas.js'; + +// Export concrete implementations (for custom usage and external providers) +export { MemoryDatabaseStore } from './memory-database-store.js'; diff --git a/dexto/packages/core/src/storage/database/memory-database-store.ts b/dexto/packages/core/src/storage/database/memory-database-store.ts new file mode 100644 index 00000000..faa93ba8 --- /dev/null +++ b/dexto/packages/core/src/storage/database/memory-database-store.ts @@ -0,0 +1,121 @@ +import type { Database } from './types.js'; +import { StorageError } from '../errors.js'; + +/** + * In-memory database store for development and testing. + * Supports list operations for message history and enumeration for settings. + * Data is lost when the process restarts. + */ +export class MemoryDatabaseStore implements Database { + private data = new Map(); + private lists = new Map(); + private connected = false; + + constructor() {} + + async connect(): Promise { + this.connected = true; + } + + async disconnect(): Promise { + this.connected = false; + this.data.clear(); + this.lists.clear(); + } + + isConnected(): boolean { + return this.connected; + } + + getStoreType(): string { + return 'memory'; + } + + async get(key: string): Promise { + this.checkConnection(); + try { + return this.data.get(key); + } catch (error) { + throw StorageError.readFailed( + 'get', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async set(key: string, value: T): Promise { + this.checkConnection(); + try { + this.data.set(key, value); + } catch (error) { + throw StorageError.writeFailed( + 'set', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async delete(key: string): Promise { + this.checkConnection(); + this.data.delete(key); + this.lists.delete(key); + } + + async list(prefix: string): Promise { + this.checkConnection(); + const keys: string[] = []; + + // Search in regular data + for (const key of Array.from(this.data.keys())) { + if (key.startsWith(prefix)) { + keys.push(key); + } + } + + // Search in list data + for (const key of Array.from(this.lists.keys())) { + if (key.startsWith(prefix)) { + keys.push(key); + } + } + + // Return unique sorted keys + return Array.from(new Set(keys)).sort(); + } + + async append(key: string, item: T): Promise { + this.checkConnection(); + if (!this.lists.has(key)) { + this.lists.set(key, []); + } + this.lists.get(key)!.push(item); + } + + async getRange(key: string, start: number, count: number): Promise { + this.checkConnection(); + const list = this.lists.get(key) || []; + return list.slice(start, start + count); + } + + // Helper methods + private checkConnection(): void { + if (!this.connected) { + throw StorageError.notConnected('MemoryDatabaseStore'); + } + } + + // Development helpers + async clear(): Promise { + this.data.clear(); + this.lists.clear(); + } + + async dump(): Promise<{ data: Record; lists: Record }> { + return { + data: Object.fromEntries(this.data.entries()), + lists: Object.fromEntries(this.lists.entries()), + }; + } +} diff --git a/dexto/packages/core/src/storage/database/postgres-store.ts b/dexto/packages/core/src/storage/database/postgres-store.ts new file mode 100644 index 00000000..1e92924e --- /dev/null +++ b/dexto/packages/core/src/storage/database/postgres-store.ts @@ -0,0 +1,407 @@ +import { Pool, PoolClient } from 'pg'; +import type { Database } from './types.js'; +import type { PostgresDatabaseConfig } from './schemas.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import { StorageError } from '../errors.js'; + +/** + * PostgreSQL database store for production database operations. + * Implements the Database interface with connection pooling and JSONB support. + * EXPERIMENTAL - NOT FULLY TESTED YET + */ +export class PostgresStore implements Database { + private pool: Pool | null = null; + private connected = false; + private logger: IDextoLogger; + + constructor( + private config: PostgresDatabaseConfig, + logger: IDextoLogger + ) { + this.logger = logger.createChild(DextoLogComponent.STORAGE); + } + + async connect(): Promise { + if (this.connected) return; + + const connectionString = this.config.connectionString || this.config.url; + + // Validate connection string is not an unexpanded env variable + if (connectionString?.startsWith('$')) { + throw StorageError.connectionFailed( + `PostgreSQL: Connection string contains unexpanded environment variable: ${connectionString}. ` + + `Ensure the environment variable is set in your .env file.` + ); + } + + if (!connectionString) { + throw StorageError.connectionFailed( + 'PostgreSQL: No connection string provided. Set url or connectionString in database config.' + ); + } + + // Extract schema from options (custom option for schema-based isolation) + const { schema, ...pgOptions } = (this.config.options || {}) as { + schema?: string; + [key: string]: unknown; + }; + + this.logger.info('Connecting to PostgreSQL database...'); + + this.pool = new Pool({ + connectionString, + max: this.config.maxConnections || 20, + // Shorter idle timeout for serverless DBs (Neon) - connections go stale quickly + idleTimeoutMillis: this.config.idleTimeoutMillis || 10000, + connectionTimeoutMillis: this.config.connectionTimeoutMillis || 10000, + // Enable TCP keepalive to detect dead connections + keepAlive: true, + keepAliveInitialDelayMillis: 10000, + ...pgOptions, + }); + + // Handle pool-level errors gracefully to prevent process crash + this.pool.on('error', (err) => { + this.logger.warn(`PostgreSQL pool error (will retry on next query): ${err.message}`); + // Don't throw - the pool will create a new connection on next acquire + }); + + // Set search_path for every connection from the pool when using custom schema + if (schema) { + // Validate schema name to prevent SQL injection (alphanumeric and underscores only) + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(schema)) { + throw StorageError.connectionFailed( + `PostgreSQL: Invalid schema name "${schema}". Schema names must start with a letter or underscore and contain only alphanumeric characters and underscores.` + ); + } + this.pool.on('connect', async (client) => { + try { + await client.query(`SET search_path TO "${schema}", public`); + } catch (err) { + this.logger.error(`Failed to set search_path to "${schema}": ${err}`); + } + }); + this.logger.info(`Using custom schema: "${schema}"`); + } + + // Test connection with better error handling + let client; + try { + client = await this.pool.connect(); + await client.query('SELECT NOW()'); + + // Create schema if using custom schema + if (schema) { + await this.createSchema(client, schema); + } + + await this.createTables(client); + this.connected = true; + this.logger.info('PostgreSQL database connected successfully'); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`PostgreSQL connection failed: ${errorMessage}`); + + // Clean up pool on failure + if (this.pool) { + await this.pool.end().catch(() => {}); + this.pool = null; + } + + throw StorageError.connectionFailed(`PostgreSQL: ${errorMessage}`); + } finally { + if (client) { + client.release(); + } + } + } + + /** + * Creates a PostgreSQL schema if it doesn't exist. + */ + private async createSchema(client: PoolClient, schemaName: string): Promise { + // Validate schema name to prevent SQL injection (alphanumeric and underscores only) + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(schemaName)) { + throw StorageError.connectionFailed( + `PostgreSQL: Invalid schema name "${schemaName}". Schema names must start with a letter or underscore and contain only alphanumeric characters and underscores.` + ); + } + + try { + await client.query(`CREATE SCHEMA IF NOT EXISTS "${schemaName}"`); + this.logger.debug(`Schema "${schemaName}" ready`); + } catch (error) { + // Schema creation might fail due to permissions - that's OK if schema already exists + this.logger.warn( + `Could not create schema "${schemaName}": ${error}. Assuming it exists.` + ); + } + } + + async disconnect(): Promise { + if (this.pool) { + await this.pool.end(); + this.pool = null; + } + this.connected = false; + } + + isConnected(): boolean { + return this.connected && this.pool !== null; + } + + getStoreType(): string { + return 'postgres'; + } + + // Core operations - all use withRetry for serverless DB resilience + async get(key: string): Promise { + try { + return await this.withRetry('get', async (client) => { + const result = await client.query('SELECT value FROM kv WHERE key = $1', [key]); + return result.rows[0] ? result.rows[0].value : undefined; + }); + } catch (error) { + throw StorageError.readFailed( + 'get', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async set(key: string, value: T): Promise { + try { + await this.withRetry('set', async (client) => { + // Explicitly stringify for JSONB - the pg driver doesn't always handle this correctly + const jsonValue = JSON.stringify(value); + await client.query( + 'INSERT INTO kv (key, value, updated_at) VALUES ($1, $2::jsonb, $3) ON CONFLICT (key) DO UPDATE SET value = $2::jsonb, updated_at = $3', + [key, jsonValue, new Date()] + ); + }); + } catch (error) { + throw StorageError.writeFailed( + 'set', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async delete(key: string): Promise { + await this.withRetry('delete', async (client) => { + await client.query('BEGIN'); + try { + await client.query('DELETE FROM kv WHERE key = $1', [key]); + await client.query('DELETE FROM lists WHERE key = $1', [key]); + await client.query('COMMIT'); + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } + }); + } + + // List operations + async list(prefix: string): Promise { + return await this.withRetry('list', async (client) => { + const kvResult = await client.query('SELECT key FROM kv WHERE key LIKE $1', [ + `${prefix}%`, + ]); + const listResult = await client.query( + 'SELECT DISTINCT key FROM lists WHERE key LIKE $1', + [`${prefix}%`] + ); + + const allKeys = new Set([ + ...kvResult.rows.map((row) => row.key), + ...listResult.rows.map((row) => row.key), + ]); + + return Array.from(allKeys).sort(); + }); + } + + async append(key: string, item: T): Promise { + try { + await this.withRetry('append', async (client) => { + // Explicitly stringify for JSONB - the pg driver doesn't always handle this correctly + const jsonItem = JSON.stringify(item); + await client.query( + 'INSERT INTO lists (key, item, created_at) VALUES ($1, $2::jsonb, $3)', + [key, jsonItem, new Date()] + ); + }); + } catch (error) { + throw StorageError.writeFailed( + 'append', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async getRange(key: string, start: number, count: number): Promise { + return await this.withRetry('getRange', async (client) => { + const result = await client.query( + 'SELECT item FROM lists WHERE key = $1 ORDER BY created_at ASC LIMIT $2 OFFSET $3', + [key, count, start] + ); + return result.rows.map((row) => row.item); + }); + } + + // Schema management + private async createTables(client: PoolClient): Promise { + // Key-value table with JSONB support + await client.query(` + CREATE TABLE IF NOT EXISTS kv ( + key VARCHAR(255) PRIMARY KEY, + value JSONB NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() + ) + `); + + // Lists table with JSONB support + await client.query(` + CREATE TABLE IF NOT EXISTS lists ( + id BIGSERIAL PRIMARY KEY, + key VARCHAR(255) NOT NULL, + item JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() + ) + `); + + // Indexes for performance + await client.query('CREATE INDEX IF NOT EXISTS idx_kv_key ON kv(key)'); + await client.query('CREATE INDEX IF NOT EXISTS idx_lists_key ON lists(key)'); + await client.query( + 'CREATE INDEX IF NOT EXISTS idx_lists_created_at ON lists(key, created_at DESC)' + ); + } + + private checkConnection(): void { + if (!this.connected || !this.pool) { + throw StorageError.notConnected('PostgresStore'); + } + } + + /** + * Check if an error is a connection error that should trigger a retry. + * Common with serverless databases (Neon) where connections go stale. + */ + private isConnectionError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + const code = (error as any).code; + return ( + code === 'ETIMEDOUT' || + code === 'ECONNRESET' || + code === 'ECONNREFUSED' || + code === 'EPIPE' || + code === '57P01' || // admin_shutdown + code === '57P02' || // crash_shutdown + code === '57P03' || // cannot_connect_now + error.message.includes('Connection terminated') || + error.message.includes('connection lost') + ); + } + + /** + * Execute a database operation with automatic retry on connection errors. + * Handles serverless DB connection issues (Neon cold starts, stale connections). + */ + private async withRetry( + operation: string, + fn: (client: PoolClient) => Promise, + maxRetries = 2 + ): Promise { + this.checkConnection(); + + let lastError: Error | undefined; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + let client: PoolClient | undefined; + try { + client = await this.pool!.connect(); + const result = await fn(client); + return result; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + + if (client) { + // Release with error to remove potentially bad connection from pool + client.release(true); + client = undefined; + } + + if (this.isConnectionError(error) && attempt < maxRetries) { + this.logger.warn( + `PostgreSQL ${operation} failed with connection error (attempt ${attempt + 1}/${maxRetries + 1}): ${lastError.message}. Retrying...` + ); + // Brief delay before retry + await new Promise((resolve) => setTimeout(resolve, 100 * (attempt + 1))); + continue; + } + + throw error; + } finally { + if (client) { + client.release(); + } + } + } + + throw lastError; + } + + // Advanced operations + + /** + * Execute a callback within a database transaction. + * Note: On connection failure, the entire callback will be retried on a new connection. + * Ensure callback operations are idempotent or use this only for read operations. + */ + async transaction(callback: (client: PoolClient) => Promise): Promise { + return await this.withRetry('transaction', async (client) => { + await client.query('BEGIN'); + try { + const result = await callback(client); + await client.query('COMMIT'); + return result; + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } + }); + } + + async getStats(): Promise<{ + kvCount: number; + listCount: number; + totalSize: string; + }> { + return await this.withRetry('getStats', async (client) => { + const kvResult = await client.query('SELECT COUNT(*) as count FROM kv'); + const listResult = await client.query('SELECT COUNT(*) as count FROM lists'); + const sizeResult = await client.query( + 'SELECT pg_size_pretty(pg_total_relation_size($1)) as size', + ['kv'] + ); + + return { + kvCount: parseInt(kvResult.rows[0].count), + listCount: parseInt(listResult.rows[0].count), + totalSize: sizeResult.rows[0].size, + }; + }); + } + + // Maintenance operations + async vacuum(): Promise { + await this.withRetry('vacuum', async (client) => { + await client.query('VACUUM ANALYZE kv, lists'); + }); + } +} diff --git a/dexto/packages/core/src/storage/database/provider.ts b/dexto/packages/core/src/storage/database/provider.ts new file mode 100644 index 00000000..dc40188b --- /dev/null +++ b/dexto/packages/core/src/storage/database/provider.ts @@ -0,0 +1,60 @@ +import type { z } from 'zod'; +import type { Database } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; + +/** + * Provider interface for creating database instances. + * + * This interface uses TypeScript generics to enforce type safety: + * - TType: The literal type string (e.g., 'sqlite', 'postgres') + * - TConfig: The configuration type with discriminator { type: TType } + * + * This ensures that the provider type matches the config type discriminator, + * providing compile-time safety for provider implementations. + */ +export interface DatabaseProvider< + TType extends string = string, + TConfig extends { type: TType } = any, +> { + /** + * Unique identifier for this provider (e.g., 'sqlite', 'postgres', 'in-memory'). + * Must match the 'type' field in the configuration. + */ + type: TType; + + /** + * Zod schema for validating provider-specific configuration. + * The schema must output TConfig type. + * + * Note: Uses z.ZodType with relaxed generics to allow input/output type variance. + * This is necessary because Zod schemas with `.optional().default()` have + * input types that include undefined, but output types that don't. + */ + configSchema: z.ZodType; + + /** + * Factory function to create a Database instance. + * + * Unlike blob store providers (which are sync), database providers may return + * a Promise to support lazy loading of optional dependencies (e.g., better-sqlite3, pg). + * + * @param config - Validated configuration specific to this provider + * @param logger - Logger instance for the database + * @returns A Database implementation (or Promise for async providers) + */ + create(config: TConfig, logger: IDextoLogger): Database | Promise; + + /** + * Optional metadata for documentation, UIs, and discovery. + */ + metadata?: { + /** Human-readable name (e.g., "SQLite", "PostgreSQL") */ + displayName: string; + /** Brief description of this storage backend */ + description: string; + /** Whether this provider requires network connectivity */ + requiresNetwork?: boolean; + /** Whether this provider supports list operations (append/getRange) */ + supportsListOperations?: boolean; + }; +} diff --git a/dexto/packages/core/src/storage/database/providers/index.ts b/dexto/packages/core/src/storage/database/providers/index.ts new file mode 100644 index 00000000..8bc0a251 --- /dev/null +++ b/dexto/packages/core/src/storage/database/providers/index.ts @@ -0,0 +1,9 @@ +/** + * Built-in database providers. + * + * These providers are automatically registered when importing from @dexto/core. + */ + +export { inMemoryDatabaseProvider } from './memory.js'; +export { sqliteDatabaseProvider } from './sqlite.js'; +export { postgresDatabaseProvider } from './postgres.js'; diff --git a/dexto/packages/core/src/storage/database/providers/memory.ts b/dexto/packages/core/src/storage/database/providers/memory.ts new file mode 100644 index 00000000..4203f761 --- /dev/null +++ b/dexto/packages/core/src/storage/database/providers/memory.ts @@ -0,0 +1,28 @@ +import type { DatabaseProvider } from '../provider.js'; +import type { InMemoryDatabaseConfig } from '../schemas.js'; +import { InMemoryDatabaseSchema } from '../schemas.js'; +import { MemoryDatabaseStore } from '../memory-database-store.js'; + +/** + * Provider for in-memory database storage. + * + * This provider stores data in RAM and is ideal for development, + * testing, and ephemeral use cases where persistence is not required. + * + * Features: + * - Zero external dependencies + * - Fast in-memory operations + * - No network required + * - Data is lost on restart + */ +export const inMemoryDatabaseProvider: DatabaseProvider<'in-memory', InMemoryDatabaseConfig> = { + type: 'in-memory', + configSchema: InMemoryDatabaseSchema, + create: (_config, _logger) => new MemoryDatabaseStore(), + metadata: { + displayName: 'In-Memory', + description: 'Store data in RAM (ephemeral, for testing and development)', + requiresNetwork: false, + supportsListOperations: true, + }, +}; diff --git a/dexto/packages/core/src/storage/database/providers/postgres.ts b/dexto/packages/core/src/storage/database/providers/postgres.ts new file mode 100644 index 00000000..680b1ab4 --- /dev/null +++ b/dexto/packages/core/src/storage/database/providers/postgres.ts @@ -0,0 +1,43 @@ +import type { DatabaseProvider } from '../provider.js'; +import type { PostgresDatabaseConfig } from '../schemas.js'; +import { PostgresDatabaseSchema } from '../schemas.js'; +import { StorageError } from '../../errors.js'; + +/** + * Provider for PostgreSQL database storage. + * + * This provider stores data in a PostgreSQL database server using the pg package. + * It's ideal for production deployments requiring scalability and multi-machine access. + * + * Features: + * - Connection pooling for efficient resource usage + * - JSONB storage for flexible data types + * - Transaction support + * - Suitable for distributed deployments + * + * Note: pg is an optional dependency. Install it with: + * npm install pg + */ +export const postgresDatabaseProvider: DatabaseProvider<'postgres', PostgresDatabaseConfig> = { + type: 'postgres', + configSchema: PostgresDatabaseSchema, + create: async (config, logger) => { + try { + const module = await import('../postgres-store.js'); + logger.info('Connecting to PostgreSQL database'); + return new module.PostgresStore(config, logger); + } catch (error: unknown) { + const err = error as NodeJS.ErrnoException; + if (err.code === 'ERR_MODULE_NOT_FOUND') { + throw StorageError.dependencyNotInstalled('PostgreSQL', 'pg', 'npm install pg'); + } + throw error; + } + }, + metadata: { + displayName: 'PostgreSQL', + description: 'Production PostgreSQL database with connection pooling', + requiresNetwork: true, + supportsListOperations: true, + }, +}; diff --git a/dexto/packages/core/src/storage/database/providers/sqlite.ts b/dexto/packages/core/src/storage/database/providers/sqlite.ts new file mode 100644 index 00000000..22874cb1 --- /dev/null +++ b/dexto/packages/core/src/storage/database/providers/sqlite.ts @@ -0,0 +1,52 @@ +import type { DatabaseProvider } from '../provider.js'; +import type { SqliteDatabaseConfig } from '../schemas.js'; +import { SqliteDatabaseSchema } from '../schemas.js'; +import { StorageError } from '../../errors.js'; + +/** + * Provider for SQLite database storage. + * + * This provider stores data in a local SQLite database file using better-sqlite3. + * It's ideal for single-machine deployments and development scenarios where + * persistence is required without the overhead of a database server. + * + * Features: + * - Uses better-sqlite3 for synchronous, fast operations + * - WAL mode enabled for better concurrency + * - No external server required + * - Persistent storage survives restarts + * + * Note: better-sqlite3 is an optional dependency. Install it with: + * npm install better-sqlite3 + */ +export const sqliteDatabaseProvider: DatabaseProvider<'sqlite', SqliteDatabaseConfig> = { + type: 'sqlite', + configSchema: SqliteDatabaseSchema, + create: async (config, logger) => { + try { + const module = await import('../sqlite-store.js'); + logger.info(`Creating SQLite database store: ${config.path}`); + return new module.SQLiteStore(config, logger); + } catch (error: unknown) { + const err = error as NodeJS.ErrnoException; + if ( + err.code === 'ERR_MODULE_NOT_FOUND' && + typeof err.message === 'string' && + err.message.includes('better-sqlite3') + ) { + throw StorageError.dependencyNotInstalled( + 'SQLite', + 'better-sqlite3', + 'npm install better-sqlite3' + ); + } + throw error; + } + }, + metadata: { + displayName: 'SQLite', + description: 'Local SQLite database for persistent storage', + requiresNetwork: false, + supportsListOperations: true, + }, +}; diff --git a/dexto/packages/core/src/storage/database/registry.test.ts b/dexto/packages/core/src/storage/database/registry.test.ts new file mode 100644 index 00000000..094d9bc6 --- /dev/null +++ b/dexto/packages/core/src/storage/database/registry.test.ts @@ -0,0 +1,224 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { z } from 'zod'; +import { DatabaseRegistry } from './registry.js'; +import type { DatabaseProvider } from './provider.js'; +import type { Database } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { StorageErrorCode } from '../error-codes.js'; +import { ErrorScope, ErrorType } from '../../errors/types.js'; + +// Mock logger for testing +const mockLogger: IDextoLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), +} as any; + +// Mock Database implementation +class MockDatabase implements Database { + constructor(private config: any) {} + + async get(): Promise { + return undefined; + } + async set(): Promise {} + async delete(): Promise {} + async list(): Promise { + return []; + } + async append(): Promise {} + async getRange(): Promise { + return []; + } + async connect(): Promise {} + async disconnect(): Promise {} + isConnected(): boolean { + return true; + } + getStoreType(): string { + return this.config.type; + } +} + +// Mock provider configurations and schemas +const MockProviderASchema = z + .object({ + type: z.literal('mock-db-a'), + path: z.string(), + maxSize: z.number().optional().default(1000), + }) + .strict(); + +type MockProviderAConfig = z.output; + +const mockProviderA: DatabaseProvider<'mock-db-a', MockProviderAConfig> = { + type: 'mock-db-a', + configSchema: MockProviderASchema, + create: (config, _logger) => new MockDatabase(config), + metadata: { + displayName: 'Mock Database A', + description: 'A mock database provider for testing', + requiresNetwork: false, + supportsListOperations: true, + }, +}; + +const MockProviderBSchema = z + .object({ + type: z.literal('mock-db-b'), + connectionString: z.string(), + }) + .strict(); + +type MockProviderBConfig = z.output; + +const mockProviderB: DatabaseProvider<'mock-db-b', MockProviderBConfig> = { + type: 'mock-db-b', + configSchema: MockProviderBSchema, + create: (config, _logger) => new MockDatabase(config), + metadata: { + displayName: 'Mock Database B', + description: 'Another mock database provider', + requiresNetwork: true, + }, +}; + +describe('DatabaseRegistry', () => { + let registry: DatabaseRegistry; + + beforeEach(() => { + registry = new DatabaseRegistry(); + }); + + describe('register()', () => { + it('successfully registers a provider', () => { + expect(() => registry.register(mockProviderA)).not.toThrow(); + expect(registry.has('mock-db-a')).toBe(true); + }); + + it('registers multiple providers with different types', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + expect(registry.has('mock-db-a')).toBe(true); + expect(registry.has('mock-db-b')).toBe(true); + }); + + it('throws error when registering duplicate provider type', () => { + registry.register(mockProviderA); + + expect(() => registry.register(mockProviderA)).toThrow( + expect.objectContaining({ + code: StorageErrorCode.DATABASE_PROVIDER_ALREADY_REGISTERED, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + }) + ); + }); + }); + + describe('validateConfig()', () => { + beforeEach(() => { + registry.register(mockProviderA); + registry.register(mockProviderB); + }); + + it('validates config with correct type and structure', () => { + const config = { + type: 'mock-db-a', + path: '/tmp/data.db', + }; + + const validated = registry.validateConfig(config); + expect(validated).toEqual({ + type: 'mock-db-a', + path: '/tmp/data.db', + maxSize: 1000, // default value applied + }); + }); + + it('throws error for unknown provider type', () => { + const config = { + type: 'unknown-provider', + someField: 'value', + }; + + expect(() => registry.validateConfig(config)).toThrow( + expect.objectContaining({ + code: StorageErrorCode.DATABASE_PROVIDER_UNKNOWN, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + }) + ); + }); + + it('error message includes available types', () => { + const config = { + type: 'unknown-provider', + }; + + try { + registry.validateConfig(config); + expect.fail('Should have thrown error'); + } catch (error: any) { + expect(error.message).toContain('unknown-provider'); + expect(error.context?.availableTypes).toContain('mock-db-a'); + expect(error.context?.availableTypes).toContain('mock-db-b'); + } + }); + }); + + describe('getProviders()', () => { + it('returns all registered providers', () => { + registry.register(mockProviderA); + registry.register(mockProviderB); + + const providers = registry.getProviders(); + expect(providers).toHaveLength(2); + expect(providers).toContain(mockProviderA); + expect(providers).toContain(mockProviderB); + }); + }); + + describe('Provider Creation', () => { + it('can create database instances using validated config', async () => { + registry.register(mockProviderA); + + const config = { + type: 'mock-db-a', + path: '/tmp/test.db', + }; + + const validated = registry.validateConfig(config); + const provider = registry.get('mock-db-a'); + + expect(provider).toBeDefined(); + const store = await provider!.create(validated, mockLogger); + expect(store).toBeInstanceOf(MockDatabase); + if (!(store instanceof MockDatabase)) { + throw new Error('Expected MockDatabase instance'); + } + expect(store.getStoreType()).toBe('mock-db-a'); + }); + }); + + describe('Provider Metadata', () => { + it('preserves provider metadata after registration', () => { + registry.register(mockProviderA); + + const provider = registry.get('mock-db-a'); + expect(provider?.metadata).toEqual({ + displayName: 'Mock Database A', + description: 'A mock database provider for testing', + requiresNetwork: false, + supportsListOperations: true, + }); + }); + }); +}); diff --git a/dexto/packages/core/src/storage/database/registry.ts b/dexto/packages/core/src/storage/database/registry.ts new file mode 100644 index 00000000..0d25f999 --- /dev/null +++ b/dexto/packages/core/src/storage/database/registry.ts @@ -0,0 +1,59 @@ +import type { DatabaseProvider } from './provider.js'; +import { StorageError } from '../errors.js'; +import { BaseRegistry, type RegistryErrorFactory } from '../../providers/base-registry.js'; + +/** + * Error factory for database registry errors. + * Uses StorageError for consistent error handling. + */ +const databaseErrorFactory: RegistryErrorFactory = { + alreadyRegistered: (type: string) => StorageError.databaseProviderAlreadyRegistered(type), + notFound: (type: string, availableTypes: string[]) => + StorageError.unknownDatabaseProvider(type, availableTypes), +}; + +/** + * Registry for database providers. + * + * This registry manages available database implementations and provides + * runtime validation for configurations. Providers can be registered from + * both core (built-in) and application layers (custom). + * + * The registry follows a global singleton pattern to allow registration + * before configuration loading, while maintaining type safety through + * provider interfaces. + * + * Extends BaseRegistry for common registry functionality. + */ +export class DatabaseRegistry extends BaseRegistry> { + constructor() { + super(databaseErrorFactory); + } + + /** + * Get all registered providers. + * Alias for getAll() for backward compatibility. + * + * @returns Array of providers + */ + getProviders(): DatabaseProvider[] { + return this.getAll(); + } +} + +/** + * Global singleton registry for database providers. + * + * This registry is used by the createDatabase factory and can be extended + * with custom providers before configuration loading. + * + * Example usage in CLI/server layer: + * ```typescript + * import { databaseRegistry } from '@dexto/core'; + * import { mongoProvider } from './storage/mongo-provider.js'; + * + * // Register custom provider before loading config + * databaseRegistry.register(mongoProvider); + * ``` + */ +export const databaseRegistry = new DatabaseRegistry(); diff --git a/dexto/packages/core/src/storage/database/schemas.ts b/dexto/packages/core/src/storage/database/schemas.ts new file mode 100644 index 00000000..83af28b5 --- /dev/null +++ b/dexto/packages/core/src/storage/database/schemas.ts @@ -0,0 +1,101 @@ +import { z } from 'zod'; +import { EnvExpandedString } from '@core/utils/result.js'; +import { StorageErrorCode } from '../error-codes.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; + +export const DATABASE_TYPES = ['in-memory', 'sqlite', 'postgres'] as const; +export type DatabaseType = (typeof DATABASE_TYPES)[number]; + +const BaseDatabaseSchema = z.object({ + maxConnections: z.number().int().positive().optional().describe('Maximum connections'), + idleTimeoutMillis: z + .number() + .int() + .positive() + .optional() + .describe('Idle timeout in milliseconds'), + connectionTimeoutMillis: z + .number() + .int() + .positive() + .optional() + .describe('Connection timeout in milliseconds'), + options: z.record(z.any()).optional().describe('Backend-specific options'), +}); + +// Memory database - minimal configuration +export const InMemoryDatabaseSchema = BaseDatabaseSchema.extend({ + type: z.literal('in-memory'), + // In-memory database doesn't need connection options, but inherits pool options for consistency +}).strict(); + +export type InMemoryDatabaseConfig = z.output; + +// SQLite database configuration +export const SqliteDatabaseSchema = BaseDatabaseSchema.extend({ + type: z.literal('sqlite'), + path: z + .string() + .describe( + 'SQLite database file path (required for SQLite - CLI enrichment provides per-agent path)' + ), +}).strict(); + +export type SqliteDatabaseConfig = z.output; + +// PostgreSQL database configuration +export const PostgresDatabaseSchema = BaseDatabaseSchema.extend({ + type: z.literal('postgres'), + url: EnvExpandedString().optional().describe('PostgreSQL connection URL (postgresql://...)'), + connectionString: EnvExpandedString().optional().describe('PostgreSQL connection string'), + host: z.string().optional().describe('PostgreSQL host'), + port: z.number().int().positive().optional().describe('PostgreSQL port'), + database: z.string().optional().describe('PostgreSQL database name'), + password: z.string().optional().describe('PostgreSQL password'), + // TODO: keyPrefix is reserved for future use - allows namespacing keys when multiple + // agents or environments share the same database (e.g., "dev:agent1:" vs "prod:agent2:") + keyPrefix: z + .string() + .optional() + .describe('Optional key prefix for namespacing (e.g., "dev:myagent:")'), +}).strict(); + +export type PostgresDatabaseConfig = z.output; + +// Database configuration using discriminated union +export const DatabaseConfigSchema = z + .discriminatedUnion( + 'type', + [InMemoryDatabaseSchema, SqliteDatabaseSchema, PostgresDatabaseSchema], + { + errorMap: (issue, ctx) => { + if (issue.code === z.ZodIssueCode.invalid_union_discriminator) { + return { + message: `Invalid database type. Expected 'in-memory', 'sqlite', or 'postgres'.`, + }; + } + return { message: ctx.defaultError }; + }, + } + ) + .describe('Database configuration') + .superRefine((data, ctx) => { + // Validate PostgreSQL requirements + if (data.type === 'postgres') { + if (!data.url && !data.connectionString && !data.host) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "PostgreSQL database requires one of 'url', 'connectionString', or 'host' to be specified", + path: ['url'], + params: { + code: StorageErrorCode.CONNECTION_CONFIG_MISSING, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + }, + }); + } + } + }); + +export type DatabaseConfig = z.output; diff --git a/dexto/packages/core/src/storage/database/sqlite-store.ts b/dexto/packages/core/src/storage/database/sqlite-store.ts new file mode 100644 index 00000000..765c1b78 --- /dev/null +++ b/dexto/packages/core/src/storage/database/sqlite-store.ts @@ -0,0 +1,319 @@ +import { dirname } from 'path'; +import { mkdirSync } from 'fs'; +import type { Database } from './types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import { DextoLogComponent } from '../../logger/v2/types.js'; +import type { SqliteDatabaseConfig } from './schemas.js'; +import { StorageError } from '../errors.js'; + +// Dynamic import for better-sqlite3 +let BetterSqlite3Database: any; + +/** + * SQLite database store for local development and production. + * Implements the Database interface with proper schema and connection handling. + */ +export class SQLiteStore implements Database { + private db: any | null = null; // Database.Database + private dbPath: string; + private config: SqliteDatabaseConfig; + private logger: IDextoLogger; + + constructor(config: SqliteDatabaseConfig, logger: IDextoLogger) { + this.config = config; + // Path is provided via CLI enrichment + this.dbPath = ''; + this.logger = logger.createChild(DextoLogComponent.STORAGE); + } + + private initializeTables(): void { + this.logger.debug('SQLite initializing database schema...'); + + try { + // Create key-value table + this.db.exec(` + CREATE TABLE IF NOT EXISTS kv_store ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + created_at INTEGER DEFAULT (strftime('%s', 'now')), + updated_at INTEGER DEFAULT (strftime('%s', 'now')) + ) + `); + + // Create list table for append operations + this.db.exec(` + CREATE TABLE IF NOT EXISTS list_store ( + key TEXT NOT NULL, + value TEXT NOT NULL, + sequence INTEGER, + created_at INTEGER DEFAULT (strftime('%s', 'now')), + PRIMARY KEY (key, sequence) + ) + `); + + // Create indexes for better performance + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_kv_store_key ON kv_store(key); + CREATE INDEX IF NOT EXISTS idx_list_store_key ON list_store(key); + CREATE INDEX IF NOT EXISTS idx_list_store_sequence ON list_store(key, sequence); + `); + + this.logger.debug( + 'SQLite database schema initialized: kv_store, list_store tables with indexes' + ); + } catch (error) { + throw StorageError.migrationFailed( + error instanceof Error ? error.message : String(error), + { + operation: 'table_initialization', + backend: 'sqlite', + } + ); + } + } + + async connect(): Promise { + // Dynamic import of better-sqlite3 + if (!BetterSqlite3Database) { + try { + const module = await import('better-sqlite3'); + BetterSqlite3Database = (module as any).default || module; + } catch (error: unknown) { + const err = error as NodeJS.ErrnoException; + if (err.code === 'ERR_MODULE_NOT_FOUND') { + throw StorageError.dependencyNotInstalled( + 'SQLite', + 'better-sqlite3', + 'npm install better-sqlite3' + ); + } + throw StorageError.connectionFailed( + `Failed to import better-sqlite3: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + // Initialize database path from config (full path is provided via enrichment) + this.dbPath = this.config.path; + + this.logger.info(`SQLite using database file: ${this.dbPath}`); + + // Ensure directory exists + const dir = dirname(this.dbPath); + this.logger.debug(`SQLite ensuring directory exists: ${dir}`); + try { + mkdirSync(dir, { recursive: true }); + } catch (error) { + // Directory might already exist, that's fine + this.logger.debug(`Directory creation result: ${error ? 'exists' : 'created'}`); + } + + // Initialize SQLite database + const sqliteOptions = this.config.options || {}; + this.logger.debug(`SQLite initializing database with config:`, { + readonly: sqliteOptions.readonly || false, + fileMustExist: sqliteOptions.fileMustExist || false, + timeout: sqliteOptions.timeout || 5000, + }); + + this.db = new BetterSqlite3Database(this.dbPath, { + readonly: sqliteOptions.readonly || false, + fileMustExist: sqliteOptions.fileMustExist || false, + timeout: sqliteOptions.timeout || 5000, + verbose: sqliteOptions.verbose + ? (message?: unknown, ...additionalArgs: unknown[]) => { + const messageStr = + typeof message === 'string' + ? message + : typeof message === 'object' && message !== null + ? JSON.stringify(message) + : String(message); + this.logger.debug( + messageStr, + additionalArgs.length > 0 ? { args: additionalArgs } : undefined + ); + } + : undefined, + }); + + // Enable WAL mode for better concurrency + this.db.pragma('journal_mode = WAL'); + this.logger.debug('SQLite enabled WAL mode for better concurrency'); + + // Create tables if they don't exist + this.initializeTables(); + + this.logger.info(`✅ SQLite store successfully connected to: ${this.dbPath}`); + } + + async disconnect(): Promise { + if (this.db) { + this.db.close(); + this.db = null; + } + } + + isConnected(): boolean { + return this.db !== null; + } + + getStoreType(): string { + return 'sqlite'; + } + + // Core operations + async get(key: string): Promise { + this.checkConnection(); + try { + const row = this.db.prepare('SELECT value FROM kv_store WHERE key = ?').get(key) as + | { value: string } + | undefined; + return row ? JSON.parse(row.value) : undefined; + } catch (error) { + throw StorageError.readFailed( + 'get', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async set(key: string, value: T): Promise { + this.checkConnection(); + try { + const serialized = JSON.stringify(value); + this.db + .prepare( + 'INSERT OR REPLACE INTO kv_store (key, value, updated_at) VALUES (?, ?, ?)' + ) + .run(key, serialized, Date.now()); + } catch (error) { + throw StorageError.writeFailed( + 'set', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async delete(key: string): Promise { + this.checkConnection(); + try { + this.db.prepare('DELETE FROM kv_store WHERE key = ?').run(key); + this.db.prepare('DELETE FROM list_store WHERE key = ?').run(key); + } catch (error) { + throw StorageError.deleteFailed( + 'delete', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + // List operations + async list(prefix: string): Promise { + this.checkConnection(); + try { + // Get keys from both tables + const kvKeys = this.db + .prepare('SELECT key FROM kv_store WHERE key LIKE ?') + .all(`${prefix}%`) as { key: string }[]; + const listKeys = this.db + .prepare('SELECT DISTINCT key FROM list_store WHERE key LIKE ?') + .all(`${prefix}%`) as { key: string }[]; + + const allKeys = new Set([ + ...kvKeys.map((row) => row.key), + ...listKeys.map((row) => row.key), + ]); + + return Array.from(allKeys).sort(); + } catch (error) { + throw StorageError.readFailed( + 'list', + error instanceof Error ? error.message : String(error), + { prefix } + ); + } + } + + async append(key: string, item: T): Promise { + this.checkConnection(); + try { + const serialized = JSON.stringify(item); + + // Use atomic subquery to calculate next sequence and insert in single statement + // This eliminates race conditions under WAL mode + this.db + .prepare( + 'INSERT INTO list_store (key, value, sequence) VALUES (?, ?, (SELECT COALESCE(MAX(sequence), 0) + 1 FROM list_store WHERE key = ?))' + ) + .run(key, serialized, key); + } catch (error) { + throw StorageError.writeFailed( + 'append', + error instanceof Error ? error.message : String(error), + { key } + ); + } + } + + async getRange(key: string, start: number, count: number): Promise { + this.checkConnection(); + try { + const rows = this.db + .prepare( + 'SELECT value FROM list_store WHERE key = ? ORDER BY sequence ASC LIMIT ? OFFSET ?' + ) + .all(key, count, start) as { value: string }[]; + + return rows.map((row) => JSON.parse(row.value)); + } catch (error) { + throw StorageError.readFailed( + 'getRange', + error instanceof Error ? error.message : String(error), + { key, start, count } + ); + } + } + + // Schema management + + private checkConnection(): void { + if (!this.db) { + throw StorageError.notConnected('SQLiteStore'); + } + } + + // Maintenance operations + async vacuum(): Promise { + this.checkConnection(); + this.db.exec('VACUUM'); + } + + async getStats(): Promise<{ + kvCount: number; + listCount: number; + dbSize: number; + }> { + this.checkConnection(); + + const kvCount = this.db.prepare('SELECT COUNT(*) as count FROM kv_store').get() as { + count: number; + }; + const listCount = this.db.prepare('SELECT COUNT(*) as count FROM list_store').get() as { + count: number; + }; + const dbSize = this.db + .prepare( + 'SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()' + ) + .get() as { size: number }; + + return { + kvCount: kvCount.count, + listCount: listCount.count, + dbSize: dbSize.size, + }; + } +} diff --git a/dexto/packages/core/src/storage/database/types.ts b/dexto/packages/core/src/storage/database/types.ts new file mode 100644 index 00000000..9853493b --- /dev/null +++ b/dexto/packages/core/src/storage/database/types.ts @@ -0,0 +1,24 @@ +/** + * Persistent, reliable storage for important data with list operations for message history. + * Data survives restarts and supports enumeration for settings management. + */ +export interface Database { + // Basic operations + get(key: string): Promise; + set(key: string, value: T): Promise; + delete(key: string): Promise; + + // Enumeration for settings/user data + list(prefix: string): Promise; + + // List operations for message history + append(key: string, item: T): Promise; + /** Get a range of items in chronological order (oldest first, matching insertion order) */ + getRange(key: string, start: number, count: number): Promise; + + // Connection management + connect(): Promise; + disconnect(): Promise; + isConnected(): boolean; + getStoreType(): string; +} diff --git a/dexto/packages/core/src/storage/error-codes.ts b/dexto/packages/core/src/storage/error-codes.ts new file mode 100644 index 00000000..f26f46dc --- /dev/null +++ b/dexto/packages/core/src/storage/error-codes.ts @@ -0,0 +1,60 @@ +/** + * Storage-specific error codes + * Includes cache, database, and blob storage errors + */ +export enum StorageErrorCode { + // Manager lifecycle + MANAGER_NOT_INITIALIZED = 'storage_manager_not_initialized', + MANAGER_NOT_CONNECTED = 'storage_manager_not_connected', + + // Dependencies + DEPENDENCY_NOT_INSTALLED = 'storage_dependency_not_installed', + + // Connection + CONNECTION_FAILED = 'storage_connection_failed', + CONNECTION_CONFIG_MISSING = 'storage_connection_config_missing', + + // Operations + READ_FAILED = 'storage_read_failed', + WRITE_FAILED = 'storage_write_failed', + DELETE_FAILED = 'storage_delete_failed', + + // Database specific + MIGRATION_FAILED = 'storage_migration_failed', + DATABASE_INVALID_CONFIG = 'storage_database_invalid_config', + + // Blob storage - Configuration errors + BLOB_INVALID_CONFIG = 'BLOB_INVALID_CONFIG', + + // Blob storage - Storage errors + BLOB_SIZE_EXCEEDED = 'BLOB_SIZE_EXCEEDED', + BLOB_TOTAL_SIZE_EXCEEDED = 'BLOB_TOTAL_SIZE_EXCEEDED', + BLOB_INVALID_INPUT = 'BLOB_INVALID_INPUT', + BLOB_ENCODING_ERROR = 'BLOB_ENCODING_ERROR', + + // Blob storage - Retrieval errors + BLOB_NOT_FOUND = 'BLOB_NOT_FOUND', + BLOB_INVALID_REFERENCE = 'BLOB_INVALID_REFERENCE', + BLOB_ACCESS_DENIED = 'BLOB_ACCESS_DENIED', + BLOB_CORRUPTED = 'BLOB_CORRUPTED', + + // Blob storage - Backend errors + BLOB_BACKEND_NOT_CONNECTED = 'BLOB_BACKEND_NOT_CONNECTED', + BLOB_BACKEND_UNAVAILABLE = 'BLOB_BACKEND_UNAVAILABLE', + + // Blob storage - Operation errors + BLOB_CLEANUP_FAILED = 'BLOB_CLEANUP_FAILED', + BLOB_OPERATION_FAILED = 'BLOB_OPERATION_FAILED', + + // Blob storage - Provider registry errors + BLOB_PROVIDER_UNKNOWN = 'BLOB_PROVIDER_UNKNOWN', + BLOB_PROVIDER_ALREADY_REGISTERED = 'BLOB_PROVIDER_ALREADY_REGISTERED', + + // Database - Provider registry errors + DATABASE_PROVIDER_UNKNOWN = 'DATABASE_PROVIDER_UNKNOWN', + DATABASE_PROVIDER_ALREADY_REGISTERED = 'DATABASE_PROVIDER_ALREADY_REGISTERED', + + // Cache - Provider registry errors + CACHE_PROVIDER_UNKNOWN = 'CACHE_PROVIDER_UNKNOWN', + CACHE_PROVIDER_ALREADY_REGISTERED = 'CACHE_PROVIDER_ALREADY_REGISTERED', +} diff --git a/dexto/packages/core/src/storage/errors.ts b/dexto/packages/core/src/storage/errors.ts new file mode 100644 index 00000000..19174032 --- /dev/null +++ b/dexto/packages/core/src/storage/errors.ts @@ -0,0 +1,428 @@ +import { DextoRuntimeError, DextoValidationError } from '@core/errors/index.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { StorageErrorCode } from './error-codes.js'; + +/** + * Storage error factory with typed methods for creating storage-specific errors + * Includes cache, database, and blob storage errors + * Each method creates a properly typed error with STORAGE scope + */ +export class StorageError { + /** + * Connection failed error + */ + static connectionFailed(reason: string, config?: Record) { + return new DextoRuntimeError( + StorageErrorCode.CONNECTION_FAILED, + ErrorScope.STORAGE, + ErrorType.THIRD_PARTY, + `Storage connection failed: ${reason}`, + { reason, config } + ); + } + + /** + * Backend not connected error + */ + static notConnected(backendType: string) { + return new DextoRuntimeError( + StorageErrorCode.CONNECTION_FAILED, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `${backendType} not connected`, + { backendType } + ); + } + + /** + * Storage manager not initialized + */ + static managerNotInitialized(method: string) { + return new DextoRuntimeError( + StorageErrorCode.MANAGER_NOT_INITIALIZED, + ErrorScope.STORAGE, + ErrorType.USER, + `StorageManager is not initialized. Call initialize() before ${method}()`, + { method, hint: 'Call await manager.initialize() first' } + ); + } + + /** + * Storage manager not connected + */ + static managerNotConnected(method: string) { + return new DextoRuntimeError( + StorageErrorCode.MANAGER_NOT_CONNECTED, + ErrorScope.STORAGE, + ErrorType.USER, + `StorageManager is not connected. Call connect() before ${method}()`, + { method, hint: 'Call await manager.connect() after initialize()' } + ); + } + + /** + * Required storage dependency not installed + */ + static dependencyNotInstalled( + backendType: string, + packageName: string, + installCommand: string + ) { + return new DextoRuntimeError( + StorageErrorCode.DEPENDENCY_NOT_INSTALLED, + ErrorScope.STORAGE, + ErrorType.USER, + `${backendType} storage configured but '${packageName}' package is not installed`, + { + backendType, + packageName, + hint: `Install with: ${installCommand}`, + recovery: `Either install the package or change storage type to 'in-memory'`, + } + ); + } + + /** + * Read operation failed + */ + static readFailed(operation: string, reason: string, details?: Record) { + return new DextoRuntimeError( + StorageErrorCode.READ_FAILED, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `Storage read failed for ${operation}: ${reason}`, + { operation, reason, ...details } + ); + } + + /** + * Write operation failed + */ + static writeFailed(operation: string, reason: string, details?: Record) { + return new DextoRuntimeError( + StorageErrorCode.WRITE_FAILED, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `Storage write failed for ${operation}: ${reason}`, + { operation, reason, ...details } + ); + } + + /** + * Delete operation failed + */ + static deleteFailed(operation: string, reason: string, details?: Record) { + return new DextoRuntimeError( + StorageErrorCode.DELETE_FAILED, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `Storage delete failed for ${operation}: ${reason}`, + { operation, reason, ...details } + ); + } + + /** + * Migration failed error + */ + static migrationFailed(reason: string, details?: Record) { + return new DextoRuntimeError( + StorageErrorCode.MIGRATION_FAILED, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `Database migration failed: ${reason}`, + { reason, ...details } + ); + } + + /** + * Invalid database configuration + */ + static databaseInvalidConfig( + message: string, + context?: Record + ): DextoValidationError { + return new DextoValidationError([ + { + code: StorageErrorCode.DATABASE_INVALID_CONFIG, + message, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + severity: 'error' as const, + context: context || {}, + }, + ]); + } + + // ==================== Blob Storage Errors ==================== + + /** + * Invalid blob configuration + */ + static blobInvalidConfig( + message: string, + context?: Record + ): DextoValidationError { + return new DextoValidationError([ + { + code: StorageErrorCode.BLOB_INVALID_CONFIG, + message, + scope: ErrorScope.STORAGE, + type: ErrorType.USER, + severity: 'error' as const, + context: context || {}, + }, + ]); + } + + /** + * Blob size exceeded maximum + */ + static blobSizeExceeded(size: number, maxSize: number): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_SIZE_EXCEEDED, + ErrorScope.STORAGE, + ErrorType.USER, + `Blob size ${size} bytes exceeds maximum ${maxSize} bytes`, + { size, maxSize } + ); + } + + /** + * Total blob storage size exceeded + */ + static blobTotalSizeExceeded(totalSize: number, maxTotalSize: number): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_TOTAL_SIZE_EXCEEDED, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `Total storage size ${totalSize} bytes exceeds maximum ${maxTotalSize} bytes`, + { totalSize, maxTotalSize } + ); + } + + /** + * Invalid blob input + */ + static blobInvalidInput(input: unknown, reason: string): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_INVALID_INPUT, + ErrorScope.STORAGE, + ErrorType.USER, + `Invalid blob input: ${reason}`, + { inputType: typeof input, reason } + ); + } + + /** + * Blob encoding error + */ + static blobEncodingError(operation: string, error: unknown): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_ENCODING_ERROR, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `Blob ${operation} failed: encoding error`, + { operation, originalError: error instanceof Error ? error.message : String(error) } + ); + } + + /** + * Blob not found + */ + static blobNotFound(reference: string): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_NOT_FOUND, + ErrorScope.STORAGE, + ErrorType.NOT_FOUND, + `Blob not found: ${reference}`, + { reference } + ); + } + + /** + * Invalid blob reference + */ + static blobInvalidReference(reference: string, reason: string): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_INVALID_REFERENCE, + ErrorScope.STORAGE, + ErrorType.USER, + `Invalid blob reference '${reference}': ${reason}`, + { reference, reason } + ); + } + + /** + * Blob access denied + */ + static blobAccessDenied(reference: string, operation: string): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_ACCESS_DENIED, + ErrorScope.STORAGE, + ErrorType.FORBIDDEN, + `Access denied for blob ${operation}: ${reference}`, + { reference, operation } + ); + } + + /** + * Blob data corrupted + */ + static blobCorrupted(reference: string, reason: string): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_CORRUPTED, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `Blob data corrupted: ${reference} (${reason})`, + { reference, reason } + ); + } + + /** + * Blob backend not connected + */ + static blobBackendNotConnected(backendType: string): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_BACKEND_NOT_CONNECTED, + ErrorScope.STORAGE, + ErrorType.THIRD_PARTY, + `Blob backend ${backendType} is not connected`, + { backendType } + ); + } + + /** + * Blob backend unavailable + */ + static blobBackendUnavailable(backendType: string, error: unknown): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_BACKEND_UNAVAILABLE, + ErrorScope.STORAGE, + ErrorType.THIRD_PARTY, + `Blob backend ${backendType} is unavailable`, + { backendType, originalError: error instanceof Error ? error.message : String(error) } + ); + } + + /** + * Blob cleanup failed + */ + static blobCleanupFailed(backendType: string, error: unknown): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_CLEANUP_FAILED, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `Blob cleanup failed for backend ${backendType}`, + { backendType, originalError: error instanceof Error ? error.message : String(error) } + ); + } + + /** + * Blob operation failed + */ + static blobOperationFailed( + operation: string, + backendType: string, + error: unknown + ): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_OPERATION_FAILED, + ErrorScope.STORAGE, + ErrorType.SYSTEM, + `Blob ${operation} failed for backend ${backendType}`, + { + operation, + backendType, + originalError: error instanceof Error ? error.message : String(error), + } + ); + } + + /** + * Unknown blob provider type + */ + static unknownBlobProvider(type: string, availableTypes: string[]): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_PROVIDER_UNKNOWN, + ErrorScope.STORAGE, + ErrorType.USER, + `Unknown blob store type: '${type}'`, + { type, availableTypes }, + `Available types: ${availableTypes.length > 0 ? availableTypes.join(', ') : 'none'}` + ); + } + + /** + * Blob provider already registered + */ + static blobProviderAlreadyRegistered(type: string): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.BLOB_PROVIDER_ALREADY_REGISTERED, + ErrorScope.STORAGE, + ErrorType.USER, + `Blob store provider '${type}' is already registered`, + { type }, + `Use unregister() first if you need to replace it` + ); + } + + // ==================== Database Provider Registry Errors ==================== + + /** + * Unknown database provider type + */ + static unknownDatabaseProvider(type: string, availableTypes: string[]): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.DATABASE_PROVIDER_UNKNOWN, + ErrorScope.STORAGE, + ErrorType.USER, + `Unknown database type: '${type}'`, + { type, availableTypes }, + `Available types: ${availableTypes.length > 0 ? availableTypes.join(', ') : 'none'}` + ); + } + + /** + * Database provider already registered + */ + static databaseProviderAlreadyRegistered(type: string): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.DATABASE_PROVIDER_ALREADY_REGISTERED, + ErrorScope.STORAGE, + ErrorType.USER, + `Database provider '${type}' is already registered`, + { type }, + `Use unregister() first if you need to replace it` + ); + } + + // ==================== Cache Provider Registry Errors ==================== + + /** + * Unknown cache provider type + */ + static unknownCacheProvider(type: string, availableTypes: string[]): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.CACHE_PROVIDER_UNKNOWN, + ErrorScope.STORAGE, + ErrorType.USER, + `Unknown cache type: '${type}'`, + { type, availableTypes }, + `Available types: ${availableTypes.length > 0 ? availableTypes.join(', ') : 'none'}` + ); + } + + /** + * Cache provider already registered + */ + static cacheProviderAlreadyRegistered(type: string): DextoRuntimeError { + return new DextoRuntimeError( + StorageErrorCode.CACHE_PROVIDER_ALREADY_REGISTERED, + ErrorScope.STORAGE, + ErrorType.USER, + `Cache provider '${type}' is already registered`, + { type }, + `Use unregister() first if you need to replace it` + ); + } +} diff --git a/dexto/packages/core/src/storage/index.ts b/dexto/packages/core/src/storage/index.ts new file mode 100644 index 00000000..a4e647da --- /dev/null +++ b/dexto/packages/core/src/storage/index.ts @@ -0,0 +1,113 @@ +/** + * Dexto Storage Layer + * + * A storage system with three storage types: + * - Cache: Fast, ephemeral storage (Redis, Memory) with TTL support + * - Database: Persistent, reliable storage (PostgreSQL, SQLite, Memory) with list operations + * - Blob: Large object storage (Local, Memory) for files and binary data + * + * All storage types use a provider pattern for extensibility. + * + * Usage: + * + * ```typescript + * // Create and initialize a storage manager + * const manager = await createStorageManager({ + * cache: { type: 'in-memory' }, + * database: { type: 'in-memory' }, + * blob: { type: 'local', storePath: '/tmp/blobs' } + * }); + * + * // Access cache and database via getters + * const cache = manager.getCache(); + * const database = manager.getDatabase(); + * const blobStore = manager.getBlobStore(); + * + * // Use cache for temporary data + * await cache.set('session:123', sessionData, 3600); // 1 hour TTL + * const sessionData = await cache.get('session:123'); + * + * // Use database for persistent data + * await database.set('user:456', userData); + * await database.append('messages:789', message); + * const messages = await database.getRange('messages:789', 0, 50); + * + * // Cleanup when done + * await manager.disconnect(); + * ``` + * + * ## Registering Custom Providers + * + * ```typescript + * import { databaseRegistry, cacheRegistry, blobStoreRegistry } from '@dexto/core'; + * + * // Register before loading config + * databaseRegistry.register(mongoProvider); + * cacheRegistry.register(memcachedProvider); + * blobStoreRegistry.register(s3Provider); + * ``` + */ + +// Main storage manager and utilities +export { StorageManager, createStorageManager } from './storage-manager.js'; + +export type { StorageConfig, ValidatedStorageConfig } from './schemas.js'; + +export { CACHE_TYPES, DATABASE_TYPES, BLOB_STORE_TYPES } from './schemas.js'; +export type { CacheType, DatabaseType, BlobStoreType } from './schemas.js'; +export { + CacheConfigSchema, + DatabaseConfigSchema, + BlobStoreConfigSchema, + StorageSchema, +} from './schemas.js'; + +export { createDatabase, databaseRegistry, DatabaseRegistry } from './database/index.js'; +export type { DatabaseProvider } from './database/index.js'; + +export type { Database } from './database/types.js'; + +export { + inMemoryDatabaseProvider, + sqliteDatabaseProvider, + postgresDatabaseProvider, +} from './database/providers/index.js'; + +export type { + DatabaseConfig, + InMemoryDatabaseConfig, + SqliteDatabaseConfig, + PostgresDatabaseConfig, +} from './schemas.js'; + +export { MemoryDatabaseStore } from './database/memory-database-store.js'; + +export { createCache, cacheRegistry, CacheRegistry } from './cache/index.js'; +export type { CacheProvider } from './cache/index.js'; + +export type { Cache } from './cache/types.js'; + +export { inMemoryCacheProvider, redisCacheProvider } from './cache/providers/index.js'; + +export type { CacheConfig, InMemoryCacheConfig, RedisCacheConfig } from './schemas.js'; + +export { MemoryCacheStore } from './cache/memory-cache-store.js'; + +export { createBlobStore, blobStoreRegistry, BlobStoreRegistry } from './blob/index.js'; +export type { BlobStoreProvider } from './blob/index.js'; + +export type { BlobStore } from './blob/types.js'; +export type { + BlobInput, + BlobMetadata, + StoredBlobMetadata, + BlobReference, + BlobData, + BlobStats, +} from './blob/types.js'; + +export { localBlobStoreProvider, inMemoryBlobStoreProvider } from './blob/providers/index.js'; + +export type { BlobStoreConfig, InMemoryBlobStoreConfig, LocalBlobStoreConfig } from './schemas.js'; + +export { LocalBlobStore, InMemoryBlobStore } from './blob/index.js'; diff --git a/dexto/packages/core/src/storage/schemas.test.ts b/dexto/packages/core/src/storage/schemas.test.ts new file mode 100644 index 00000000..8b5fa441 --- /dev/null +++ b/dexto/packages/core/src/storage/schemas.test.ts @@ -0,0 +1,436 @@ +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { + StorageSchema, + type StorageConfig, + type InMemoryCacheConfig, + type RedisCacheConfig, + type InMemoryDatabaseConfig, + type SqliteDatabaseConfig, + type PostgresDatabaseConfig, + type CacheConfig, + type DatabaseConfig, +} from './schemas.js'; + +// Test helper: default blob config for tests +const testBlobConfig = { type: 'local' as const, storePath: '/tmp/test-blobs' }; + +describe('StorageSchema', () => { + describe('Backend Configuration - In-Memory', () => { + it('should accept minimal in-memory backend config', () => { + const config = { type: 'in-memory' as const }; + const result = StorageSchema.safeParse({ + cache: config, + database: config, + blob: testBlobConfig, + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.cache.type).toBe('in-memory'); + expect(result.data.database.type).toBe('in-memory'); + } + }); + + it('should accept in-memory backend with optional connection options', () => { + const cacheConfig: InMemoryCacheConfig = { + type: 'in-memory', + maxConnections: 10, + idleTimeoutMillis: 5000, + connectionTimeoutMillis: 3000, + }; + + const dbConfig: InMemoryDatabaseConfig = { + type: 'in-memory', + maxConnections: 10, + idleTimeoutMillis: 5000, + connectionTimeoutMillis: 3000, + }; + + const result = StorageSchema.safeParse({ + cache: cacheConfig, + database: dbConfig, + blob: testBlobConfig, + }); + expect(result.success).toBe(true); + }); + }); + + describe('Backend Configuration - Redis', () => { + it('should accept Redis backend with URL', () => { + const config: RedisCacheConfig = { + type: 'redis', + url: 'redis://localhost:6379', + }; + + const result = StorageSchema.parse({ + cache: config, + database: { type: 'in-memory' }, + blob: testBlobConfig, + }); + expect(result.cache.type).toBe('redis'); + if (result.cache.type === 'redis') { + expect(result.cache.url).toBe('redis://localhost:6379'); + } + }); + + it('should accept Redis backend with host/port', () => { + const config: RedisCacheConfig = { + type: 'redis', + host: 'localhost', + port: 6379, + password: 'secret', + database: 0, + }; + + const result = StorageSchema.parse({ + cache: config, + database: { type: 'in-memory' }, + blob: testBlobConfig, + }); + expect(result.cache.type).toBe('redis'); + if (result.cache.type === 'redis') { + expect(result.cache.host).toBe('localhost'); + } + }); + + it('should reject Redis backend without URL or host', () => { + const config = { type: 'redis' }; + + const result = StorageSchema.safeParse({ + cache: config, + database: { type: 'in-memory' }, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.custom); + expect(result.error?.issues[0]?.message).toContain('Redis cache requires either'); + expect(result.error?.issues[0]?.path).toEqual(['cache', 'url']); + }); + }); + + describe('Backend Configuration - SQLite', () => { + it('should accept SQLite backend with path', () => { + const config: SqliteDatabaseConfig = { + type: 'sqlite', + path: '/tmp/db/dexto.db', + }; + + const result = StorageSchema.parse({ + cache: { type: 'in-memory' }, + database: config, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }); + expect(result.database.type).toBe('sqlite'); + if (result.database.type === 'sqlite') { + expect(result.database.path).toBe('/tmp/db/dexto.db'); + } + }); + }); + + describe('Backend Configuration - PostgreSQL', () => { + it('should accept PostgreSQL backend with URL', () => { + const config: PostgresDatabaseConfig = { + type: 'postgres', + url: 'postgresql://user:pass@localhost:5432/dexto', + }; + + const result = StorageSchema.parse({ + cache: { type: 'in-memory' }, + database: config, + blob: testBlobConfig, + }); + expect(result.database.type).toBe('postgres'); + if (result.database.type === 'postgres') { + expect(result.database.url).toBe('postgresql://user:pass@localhost:5432/dexto'); + } + }); + + it('should accept PostgreSQL backend with connection string', () => { + const config: PostgresDatabaseConfig = { + type: 'postgres', + connectionString: 'postgresql://user:pass@localhost:5432/dexto', + }; + + const result = StorageSchema.parse({ + cache: { type: 'in-memory' }, + database: config, + blob: testBlobConfig, + }); + expect(result.database.type).toBe('postgres'); + if (result.database.type === 'postgres') { + expect(result.database.connectionString).toBe( + 'postgresql://user:pass@localhost:5432/dexto' + ); + } + }); + + it('should accept PostgreSQL backend with host/port details', () => { + const config: PostgresDatabaseConfig = { + type: 'postgres', + host: 'localhost', + port: 5432, + database: 'dexto', + password: 'secret', + }; + + const result = StorageSchema.parse({ + cache: { type: 'in-memory' }, + database: config, + blob: testBlobConfig, + }); + expect(result.database.type).toBe('postgres'); + if (result.database.type === 'postgres') { + expect(result.database.host).toBe('localhost'); + expect(result.database.port).toBe(5432); + } + }); + + it('should reject PostgreSQL backend without connection info', () => { + const config = { type: 'postgres' }; + + const result = StorageSchema.safeParse({ + cache: { type: 'in-memory' }, + database: config, + blob: testBlobConfig, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.custom); + expect(result.error?.issues[0]?.message).toContain( + 'PostgreSQL database requires one of' + ); + expect(result.error?.issues[0]?.path).toEqual(['database', 'url']); + }); + }); + + describe('Discriminated Union Validation', () => { + it('should reject invalid backend type', () => { + const config = { type: 'invalid-backend' }; + + const result = StorageSchema.safeParse({ + cache: config, + database: { type: 'in-memory' }, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_union_discriminator); + expect(result.error?.issues[0]?.path).toEqual(['cache', 'type']); + }); + + it('should provide clear error messages for invalid discriminator', () => { + const config = { type: 'nosql' }; + + const result = StorageSchema.safeParse({ + cache: config, + database: { type: 'in-memory' }, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_union_discriminator); + expect(result.error?.issues[0]?.message).toContain('Invalid cache type'); + expect(result.error?.issues[0]?.path).toEqual(['cache', 'type']); + }); + }); + + describe('Connection Pool Options', () => { + it('should validate positive connection limits', () => { + // Negative connections should fail + let result = StorageSchema.safeParse({ + cache: { type: 'in-memory', maxConnections: -1 }, + database: { type: 'in-memory' }, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['cache', 'maxConnections']); + + // Zero connections should fail + result = StorageSchema.safeParse({ + cache: { type: 'in-memory', maxConnections: 0 }, + database: { type: 'in-memory' }, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['cache', 'maxConnections']); + + // Positive connections should succeed + const validResult = StorageSchema.parse({ + cache: { type: 'in-memory', maxConnections: 10 }, + database: { type: 'in-memory' }, + blob: testBlobConfig, + }); + expect(validResult.cache.maxConnections).toBe(10); + }); + + it('should validate positive timeout values', () => { + // Negative idle timeout should fail + let result = StorageSchema.safeParse({ + cache: { type: 'in-memory', idleTimeoutMillis: -1 }, + database: { type: 'in-memory' }, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['cache', 'idleTimeoutMillis']); + + // Zero connection timeout should fail + result = StorageSchema.safeParse({ + cache: { type: 'in-memory', connectionTimeoutMillis: 0 }, + database: { type: 'in-memory' }, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['cache', 'connectionTimeoutMillis']); + + // Positive timeout should succeed + const validResult = StorageSchema.parse({ + cache: { type: 'in-memory', idleTimeoutMillis: 5000 }, + database: { type: 'in-memory' }, + blob: testBlobConfig, + }); + expect(validResult.cache.idleTimeoutMillis).toBe(5000); + }); + }); + + describe('Strict Validation', () => { + it('should reject extra fields on backend configs', () => { + const configWithExtra = { + type: 'in-memory', + unknownField: 'should fail', + }; + + const result = StorageSchema.safeParse({ + cache: configWithExtra, + database: { type: 'in-memory' }, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + expect(result.error?.issues[0]?.path).toEqual(['cache']); + }); + + it('should reject extra fields on storage config', () => { + const configWithExtra = { + cache: { type: 'in-memory' }, + database: { type: 'in-memory' }, + blob: testBlobConfig, + unknownField: 'should fail', + }; + + const result = StorageSchema.safeParse(configWithExtra); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + + describe('Type Safety', () => { + it('should have correct type inference for different backends', () => { + const config: StorageConfig = { + cache: { type: 'redis', url: 'redis://localhost:6379' }, + database: { type: 'postgres', url: 'postgresql://localhost/dexto' }, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }; + + const result = StorageSchema.safeParse(config); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.cache.type).toBe('redis'); + expect(result.data.database.type).toBe('postgres'); + } + }); + + it('should handle cache config type unions correctly', () => { + const cacheConfigs: CacheConfig[] = [ + { type: 'in-memory' }, + { type: 'redis', url: 'redis://localhost:6379' }, + ]; + + cacheConfigs.forEach((cacheConfig) => { + const result = StorageSchema.parse({ + cache: cacheConfig, + database: { type: 'in-memory' }, + blob: testBlobConfig, + }); + expect(result.cache.type).toBe(cacheConfig.type); + }); + }); + + it('should handle database config type unions correctly', () => { + const dbConfigs: DatabaseConfig[] = [ + { type: 'in-memory' }, + { type: 'sqlite', path: '/tmp/db/test.db' }, + { type: 'postgres', url: 'postgresql://localhost/test' }, + ]; + + dbConfigs.forEach((dbConfig) => { + const result = StorageSchema.parse({ + cache: { type: 'in-memory' }, + database: dbConfig, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }); + expect(result.database.type).toBe(dbConfig.type); + }); + }); + }); + + describe('Real-world Scenarios', () => { + it('should handle typical development configuration', () => { + const devConfig: StorageConfig = { + cache: { type: 'in-memory' }, + database: { type: 'sqlite', path: './dev-db/dev.db' }, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }; + + const result = StorageSchema.safeParse(devConfig); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).toMatchObject(devConfig); + } + }); + + it('should handle production configuration with Redis cache', () => { + const prodConfig: StorageConfig = { + cache: { + type: 'redis', + url: 'redis://cache.example.com:6379', + maxConnections: 50, + idleTimeoutMillis: 30000, + }, + database: { + type: 'postgres', + url: 'postgresql://user:pass@db.example.com:5432/dexto', + maxConnections: 20, + connectionTimeoutMillis: 5000, + }, + blob: { type: 'local', storePath: '/var/dexto/blobs' }, + }; + + const result = StorageSchema.safeParse(prodConfig); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).toMatchObject(prodConfig); + } + }); + + it('should handle high-availability configuration', () => { + const haConfig: StorageConfig = { + cache: { + type: 'redis', + host: 'redis-cluster.example.com', + port: 6379, + password: 'cluster-secret', + maxConnections: 100, + }, + database: { + type: 'postgres', + host: 'postgres-primary.example.com', + port: 5432, + database: 'dexto_prod', + password: 'db-secret', + maxConnections: 50, + }, + blob: { type: 'local', storePath: '/var/dexto/blobs' }, + }; + + const result = StorageSchema.safeParse(haConfig); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).toMatchObject(haConfig); + } + }); + }); +}); diff --git a/dexto/packages/core/src/storage/schemas.ts b/dexto/packages/core/src/storage/schemas.ts new file mode 100644 index 00000000..3418db10 --- /dev/null +++ b/dexto/packages/core/src/storage/schemas.ts @@ -0,0 +1,61 @@ +import { z } from 'zod'; + +// Re-export all cache schemas and types +export { + CACHE_TYPES, + CacheConfigSchema, + type CacheType, + type CacheConfig, + type InMemoryCacheConfig, + type RedisCacheConfig, +} from './cache/schemas.js'; + +// Re-export all database schemas and types +export { + DATABASE_TYPES, + DatabaseConfigSchema, + type DatabaseType, + type DatabaseConfig, + type InMemoryDatabaseConfig, + type SqliteDatabaseConfig, + type PostgresDatabaseConfig, +} from './database/schemas.js'; + +// Re-export all blob schemas and types +export { + BLOB_STORE_TYPES, + BlobStoreConfigSchema, + InMemoryBlobStoreSchema, + LocalBlobStoreSchema, + type BlobStoreType, + type BlobStoreConfig, + type InMemoryBlobStoreConfig, + type LocalBlobStoreConfig, +} from './blob/schemas.js'; + +// Import for composition +import { CacheConfigSchema } from './cache/schemas.js'; +import { DatabaseConfigSchema } from './database/schemas.js'; +import { BlobStoreConfigSchema } from './blob/schemas.js'; + +/** + * Top-level storage configuration schema + * Composes cache, database, and blob store configurations + * + * Note: Blob config uses runtime validation via the provider registry, + * allowing custom providers to be registered at the CLI/server layer. + */ +export const StorageSchema = z + .object({ + cache: CacheConfigSchema.describe('Cache configuration (fast, ephemeral)'), + database: DatabaseConfigSchema.describe('Database configuration (persistent, reliable)'), + blob: BlobStoreConfigSchema.describe( + 'Blob store configuration (for large, unstructured data)' + ), + }) + .strict() + .describe('Storage configuration with cache, database, and blob store') + .brand<'ValidatedStorageConfig'>(); + +export type StorageConfig = z.input; +export type ValidatedStorageConfig = z.output; diff --git a/dexto/packages/core/src/storage/storage-manager.ts b/dexto/packages/core/src/storage/storage-manager.ts new file mode 100644 index 00000000..2b6fc628 --- /dev/null +++ b/dexto/packages/core/src/storage/storage-manager.ts @@ -0,0 +1,274 @@ +import type { Cache } from './cache/types.js'; +import type { Database } from './database/types.js'; +import type { BlobStore } from './blob/types.js'; +import type { ValidatedStorageConfig } from './schemas.js'; +// Import from index files to ensure providers are registered +import { createCache } from './cache/index.js'; +import { createDatabase } from './database/index.js'; +import { createBlobStore } from './blob/index.js'; +import { StorageError } from './errors.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; + +const HEALTH_CHECK_KEY = 'storage_manager_health_check'; + +/** + * Storage manager that initializes and manages storage backends. + * Handles cache, database, and blob backends with automatic fallbacks. + * + * Lifecycle: + * 1. new StorageManager(config) - Creates manager with config + * 2. await manager.initialize() - Creates store instances (without connecting) + * 3. await manager.connect() - Establishes connections to all stores + * 4. await manager.disconnect() - Closes all connections + */ +export class StorageManager { + private config: ValidatedStorageConfig; + private cache!: Cache; + private database!: Database; + private blobStore!: BlobStore; + private initialized = false; + private connected = false; + private logger: IDextoLogger; + + constructor(config: ValidatedStorageConfig, logger: IDextoLogger) { + this.config = config; + this.logger = logger.createChild(DextoLogComponent.STORAGE); + } + + /** + * Initialize storage instances without connecting. + * This allows configuration and inspection before establishing connections. + */ + async initialize(): Promise { + if (this.initialized) { + return; + } + + // Create store instances (schema provides in-memory defaults, CLI enrichment adds filesystem paths) + this.cache = await createCache(this.config.cache, this.logger); + this.database = await createDatabase(this.config.database, this.logger); + this.blobStore = createBlobStore(this.config.blob, this.logger); + + this.initialized = true; + } + + /** + * Connect all storage backends. + * Must call initialize() first, or use createStorageManager() helper. + */ + async connect(): Promise { + if (!this.initialized) { + throw StorageError.managerNotInitialized('connect'); + } + + if (this.connected) { + return; + } + + // Establish connections with rollback on partial failure + const connected: ('cache' | 'database' | 'blob')[] = []; + try { + await this.cache.connect(); + connected.push('cache'); + + await this.database.connect(); + connected.push('database'); + + await this.blobStore.connect(); + connected.push('blob'); + + this.connected = true; + } catch (error) { + // Rollback: disconnect any stores that were successfully connected + this.logger.warn( + `Storage connection failed, rolling back ${connected.length} connected stores` + ); + for (const store of connected.reverse()) { + try { + if (store === 'cache') await this.cache.disconnect(); + else if (store === 'database') await this.database.disconnect(); + else if (store === 'blob') await this.blobStore.disconnect(); + } catch (disconnectError) { + this.logger.error( + `Failed to rollback ${store} during connection failure: ${disconnectError}` + ); + } + } + throw error; + } + } + + async disconnect(): Promise { + if (!this.connected) return; + + // Disconnect all stores concurrently, continue even if some fail + const results = await Promise.allSettled([ + this.cache.disconnect(), + this.database.disconnect(), + this.blobStore.disconnect(), + ]); + + // Track errors from failed disconnections + const errors: Error[] = []; + const storeNames = ['cache', 'database', 'blob store']; + + results.forEach((result, index) => { + if (result.status === 'rejected') { + const storeName = storeNames[index]; + const error = result.reason; + this.logger.error(`Failed to disconnect ${storeName}: ${error}`); + errors.push(error instanceof Error ? error : new Error(String(error))); + } + }); + + this.connected = false; + + // If any disconnections failed, throw an aggregated error + if (errors.length > 0) { + throw StorageError.connectionFailed( + `Failed to disconnect ${errors.length} storage backend(s): ${errors.map((e) => e.message).join(', ')}` + ); + } + } + + isConnected(): boolean { + return ( + this.connected && + this.cache.isConnected() && + this.database.isConnected() && + this.blobStore.isConnected() + ); + } + + /** + * Get the cache store instance. + * @throws {DextoRuntimeError} if not connected + */ + getCache(): Cache { + if (!this.connected) { + throw StorageError.managerNotConnected('getCache'); + } + return this.cache; + } + + /** + * Get the database store instance. + * @throws {DextoRuntimeError} if not connected + */ + getDatabase(): Database { + if (!this.connected) { + throw StorageError.managerNotConnected('getDatabase'); + } + return this.database; + } + + /** + * Get the blob store instance. + * @throws {DextoRuntimeError} if not connected + */ + getBlobStore(): BlobStore { + if (!this.connected) { + throw StorageError.managerNotConnected('getBlobStore'); + } + return this.blobStore; + } + + // Utility methods + async getInfo(): Promise<{ + cache: { type: string; connected: boolean }; + database: { type: string; connected: boolean }; + blob: { type: string; connected: boolean }; + connected: boolean; + }> { + if (!this.connected) { + throw StorageError.managerNotConnected('getInfo'); + } + + return { + cache: { + type: this.cache.getStoreType(), + connected: this.cache.isConnected(), + }, + database: { + type: this.database.getStoreType(), + connected: this.database.isConnected(), + }, + blob: { + type: this.blobStore.getStoreType(), + connected: this.blobStore.isConnected(), + }, + connected: this.connected, + }; + } + + // Health check + async healthCheck(): Promise<{ + cache: boolean; + database: boolean; + blob: boolean; + overall: boolean; + }> { + if (!this.connected) { + throw StorageError.managerNotConnected('healthCheck'); + } + + let cacheHealthy = false; + let databaseHealthy = false; + let blobHealthy = false; + + try { + if (this.cache.isConnected()) { + await this.cache.set(HEALTH_CHECK_KEY, 'ok', 10); + const result = await this.cache.get(HEALTH_CHECK_KEY); + cacheHealthy = result === 'ok'; + await this.cache.delete(HEALTH_CHECK_KEY); + } + } catch (error) { + this.logger.warn(`Cache health check failed: ${error}`); + } + + try { + if (this.database.isConnected()) { + await this.database.set(HEALTH_CHECK_KEY, 'ok'); + const result = await this.database.get(HEALTH_CHECK_KEY); + databaseHealthy = result === 'ok'; + await this.database.delete(HEALTH_CHECK_KEY); + } + } catch (error) { + this.logger.warn(`Database health check failed: ${error}`); + } + + try { + if (this.blobStore.isConnected()) { + // For blob store, just check if it's connected (no data operations) + blobHealthy = this.blobStore.isConnected(); + } + } catch (error) { + this.logger.warn(`Blob store health check failed: ${error}`); + } + + return { + cache: cacheHealthy, + database: databaseHealthy, + blob: blobHealthy, + overall: cacheHealthy && databaseHealthy && blobHealthy, + }; + } +} + +/** + * Create and initialize a storage manager. + * This is a convenience helper that combines initialization and connection. + * Per-agent paths are provided via CLI enrichment layer before this point. + * @param config Storage configuration with explicit paths + */ +export async function createStorageManager( + config: ValidatedStorageConfig, + logger: IDextoLogger +): Promise { + const manager = new StorageManager(config, logger); + await manager.initialize(); + await manager.connect(); + return manager; +} diff --git a/dexto/packages/core/src/storage/types.ts b/dexto/packages/core/src/storage/types.ts new file mode 100644 index 00000000..b1424fc4 --- /dev/null +++ b/dexto/packages/core/src/storage/types.ts @@ -0,0 +1,6 @@ +/** + * Re-export simplified storage types + */ +export type { Cache } from './cache/types.js'; +export type { Database } from './database/types.js'; +export type { StorageConfig } from './schemas.js'; diff --git a/dexto/packages/core/src/systemPrompt/contributors.test.ts b/dexto/packages/core/src/systemPrompt/contributors.test.ts new file mode 100644 index 00000000..7abfec4a --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/contributors.test.ts @@ -0,0 +1,260 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest'; +import { FileContributor } from './contributors.js'; +import { writeFile, mkdir, rm } from 'fs/promises'; +import { join } from 'path'; +import { DynamicContributorContext } from './types.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { SystemPromptErrorCode } from './error-codes.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; + +const mockLogger = createMockLogger(); + +describe('FileContributor', () => { + const testDir = join(process.cwd(), 'test-files'); + const mockContext: DynamicContributorContext = { + mcpManager: {} as any, + }; + + beforeEach(async () => { + // Create test directory and files + await mkdir(testDir, { recursive: true }); + + // Create test files + await writeFile( + join(testDir, 'test1.md'), + '# Test Document 1\n\nThis is the first test document.' + ); + await writeFile(join(testDir, 'test2.txt'), 'This is a plain text file.\nSecond line.'); + await writeFile(join(testDir, 'large.md'), 'x'.repeat(200000)); // Large file for testing + await writeFile(join(testDir, 'invalid.json'), '{"key": "value"}'); // Invalid file type + }); + + afterEach(async () => { + // Clean up test files + await rm(testDir, { recursive: true, force: true }); + }); + + test('should read single markdown file with default options', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'test1.md')], + {}, + mockLogger + ); + const result = await contributor.getContent(mockContext); + + expect(result).toContain(''); + expect(result).toContain('test-files/test1.md'); + expect(result).toContain('# Test Document 1'); + expect(result).toContain('This is the first test document.'); + expect(result).toContain(''); + }); + + test('should read multiple files with separator', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'test1.md'), join(testDir, 'test2.txt')], + { + separator: '\n\n===\n\n', + }, + mockLogger + ); + const result = await contributor.getContent(mockContext); + + expect(result).toContain('# Test Document 1'); + expect(result).toContain('This is a plain text file.'); + expect(result).toContain('==='); + }); + + test('should handle missing files with skip mode', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'missing.md'), join(testDir, 'test1.md')], + { + errorHandling: 'skip', + }, + mockLogger + ); + const result = await contributor.getContent(mockContext); + + expect(result).toContain('# Test Document 1'); + expect(result).not.toContain('missing.md'); + }); + + test('should handle missing files with error mode', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'missing.md'), join(testDir, 'test1.md')], + { + errorHandling: 'error', + }, + mockLogger + ); + + const error = (await contributor + .getContent(mockContext) + .catch((e) => e)) as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(SystemPromptErrorCode.FILE_READ_FAILED); + expect(error.scope).toBe(ErrorScope.SYSTEM_PROMPT); + expect(error.type).toBe(ErrorType.SYSTEM); + }); + + test('should throw error for missing files with single file error mode', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'missing.md')], + { + errorHandling: 'error', + }, + mockLogger + ); + + const error = (await contributor + .getContent(mockContext) + .catch((e) => e)) as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(SystemPromptErrorCode.FILE_READ_FAILED); + expect(error.scope).toBe(ErrorScope.SYSTEM_PROMPT); + expect(error.type).toBe(ErrorType.SYSTEM); + }); + + test('should skip large files with skip mode', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'large.md'), join(testDir, 'test1.md')], + { + maxFileSize: 1000, + errorHandling: 'skip', + }, + mockLogger + ); + const result = await contributor.getContent(mockContext); + + expect(result).toContain('# Test Document 1'); + expect(result).not.toContain('large.md'); + }); + + test('should handle invalid file types with skip mode', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'invalid.json'), join(testDir, 'test1.md')], + { + errorHandling: 'skip', + }, + mockLogger + ); + const result = await contributor.getContent(mockContext); + + expect(result).toContain('# Test Document 1'); + expect(result).not.toContain('invalid.json'); + }); + + test('should throw error for invalid file types with error mode', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'invalid.json')], + { + errorHandling: 'error', + }, + mockLogger + ); + + await expect(contributor.getContent(mockContext)).rejects.toThrow( + 'is not a .md or .txt file' + ); + }); + + test('should exclude filenames when configured', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'test1.md')], + { + includeFilenames: false, + }, + mockLogger + ); + const result = await contributor.getContent(mockContext); + + expect(result).toContain('# Test Document 1'); + expect(result).not.toContain('test-files/test1.md'); + }); + + test('should include metadata when configured', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'test1.md')], + { + includeMetadata: true, + }, + mockLogger + ); + const result = await contributor.getContent(mockContext); + + expect(result).toContain('*File size:'); + expect(result).toContain('Modified:'); + }); + + test('should return empty context when no files can be loaded', async () => { + const contributor = new FileContributor( + 'test', + 0, + [join(testDir, 'missing.md')], + { + errorHandling: 'skip', + }, + mockLogger + ); + const result = await contributor.getContent(mockContext); + + expect(result).toBe('No files could be loaded'); + }); + + test('should read files using absolute paths from arbitrary directories', async () => { + const configDir = join(testDir, 'config'); + await mkdir(configDir, { recursive: true }); + + const absolutePath = join(configDir, 'config-file.md'); + await writeFile(absolutePath, '# Config File\n\nThis is a config file.'); + + const contributor = new FileContributor('test', 0, [absolutePath], {}, mockLogger); + + const result = await contributor.getContent(mockContext); + + expect(result).toContain(''); + expect(result).toContain('config-file.md'); + expect(result).toContain('# Config File'); + expect(result).toContain('This is a config file.'); + expect(result).toContain(''); + }); + + test('should read files from nested directories when given absolute paths', async () => { + const configDir = join(testDir, 'config'); + const docsDir = join(configDir, 'docs'); + await mkdir(docsDir, { recursive: true }); + + const nestedPath = join(docsDir, 'readme.md'); + await writeFile(nestedPath, '# Documentation\n\nThis is documentation.'); + + const contributor = new FileContributor('test', 0, [nestedPath], {}, mockLogger); + + const result = await contributor.getContent(mockContext); + + expect(result).toContain(''); + expect(result).toContain('readme.md'); + expect(result).toContain('# Documentation'); + expect(result).toContain('This is documentation.'); + expect(result).toContain(''); + }); +}); diff --git a/dexto/packages/core/src/systemPrompt/contributors.ts b/dexto/packages/core/src/systemPrompt/contributors.ts new file mode 100644 index 00000000..6f3ce5fd --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/contributors.ts @@ -0,0 +1,293 @@ +import { SystemPromptContributor, DynamicContributorContext } from './types.js'; +import { readFile, stat } from 'fs/promises'; +import { resolve, extname } from 'path'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { SystemPromptError } from './errors.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import type { MemoryManager } from '../memory/index.js'; +import type { PromptManager } from '../prompts/prompt-manager.js'; + +export class StaticContributor implements SystemPromptContributor { + constructor( + public id: string, + public priority: number, + private content: string + ) {} + + async getContent(_context: DynamicContributorContext): Promise { + return this.content; + } +} + +export class DynamicContributor implements SystemPromptContributor { + constructor( + public id: string, + public priority: number, + private promptGenerator: (context: DynamicContributorContext) => Promise + ) {} + + async getContent(context: DynamicContributorContext): Promise { + return this.promptGenerator(context); + } +} + +export interface FileContributorOptions { + includeFilenames?: boolean | undefined; + separator?: string | undefined; + errorHandling?: 'skip' | 'error' | undefined; + maxFileSize?: number | undefined; + includeMetadata?: boolean | undefined; + cache?: boolean | undefined; +} + +export class FileContributor implements SystemPromptContributor { + // Basic in-memory cache to avoid reading files on every prompt build + private cache: Map = new Map(); + private logger: IDextoLogger; + + constructor( + public id: string, + public priority: number, + private files: string[], + private options: FileContributorOptions = {}, + logger: IDextoLogger + ) { + this.logger = logger; + this.logger.debug(`[FileContributor] Created "${id}" with files: ${JSON.stringify(files)}`); + } + + async getContent(_context: DynamicContributorContext): Promise { + const { + includeFilenames = true, + separator = '\n\n---\n\n', + errorHandling = 'skip', + maxFileSize = 100000, + includeMetadata = false, + cache = true, + } = this.options; + + // If caching is enabled, check if we have cached content + if (cache) { + const cacheKey = JSON.stringify({ files: this.files, options: this.options }); + const cached = this.cache.get(cacheKey); + if (cached) { + this.logger.debug(`[FileContributor] Using cached content for "${this.id}"`); + return cached; + } + } + + const fileParts: string[] = []; + + for (const filePath of this.files) { + try { + const resolvedPath = resolve(filePath); + this.logger.debug( + `[FileContributor] Resolving path: ${filePath} → ${resolvedPath}` + ); + + // Check if file is .md or .txt + const ext = extname(resolvedPath).toLowerCase(); + if (ext !== '.md' && ext !== '.txt') { + if (errorHandling === 'error') { + throw SystemPromptError.invalidFileType(filePath, ['.md', '.txt']); + } + continue; + } + + // Check file size + const stats = await stat(resolvedPath); + if (stats.size > maxFileSize) { + if (errorHandling === 'error') { + throw SystemPromptError.fileTooLarge(filePath, stats.size, maxFileSize); + } + continue; + } + + // Read file content (always utf-8) + const content = await readFile(resolvedPath, { encoding: 'utf-8' }); + + // Build file part + let filePart = ''; + + if (includeFilenames) { + filePart += `## ${filePath}\n\n`; + } + + if (includeMetadata) { + filePart += `*File size: ${stats.size} bytes, Modified: ${stats.mtime.toISOString()}*\n\n`; + } + + filePart += content; + + fileParts.push(filePart); + } catch (error: unknown) { + if (errorHandling === 'error') { + // Preserve previously constructed structured errors + if (error instanceof DextoRuntimeError) { + throw error; + } + const reason = error instanceof Error ? error.message : String(error); + throw SystemPromptError.fileReadFailed(filePath, reason); + } + // 'skip' mode - do nothing, continue to next file + } + } + + if (fileParts.length === 0) { + return 'No files could be loaded'; + } + + const combinedContent = fileParts.join(separator); + const result = `\n${combinedContent}\n`; + + // Cache the result if caching is enabled + if (cache) { + const cacheKey = JSON.stringify({ files: this.files, options: this.options }); + this.cache.set(cacheKey, result); + this.logger.debug(`[FileContributor] Cached content for "${this.id}"`); + } + + return result; + } +} + +export interface MemoryContributorOptions { + /** Whether to include timestamps in memory display */ + includeTimestamps?: boolean | undefined; + /** Whether to include tags in memory display */ + includeTags?: boolean | undefined; + /** Maximum number of memories to include */ + limit?: number | undefined; + /** Only include pinned memories (for hybrid approach) */ + pinnedOnly?: boolean | undefined; +} + +/** + * MemoryContributor loads user memories from the database and formats them + * for inclusion in the system prompt. + * + * This enables memories to be automatically available in every conversation. + */ +export class MemoryContributor implements SystemPromptContributor { + private logger: IDextoLogger; + + constructor( + public id: string, + public priority: number, + private memoryManager: MemoryManager, + private options: MemoryContributorOptions = {}, + logger: IDextoLogger + ) { + this.logger = logger; + this.logger.debug( + `[MemoryContributor] Created "${id}" with options: ${JSON.stringify(options)}` + ); + } + + async getContent(_context: DynamicContributorContext): Promise { + const { + includeTimestamps = false, + includeTags = true, + limit, + pinnedOnly = false, + } = this.options; + + try { + // Fetch memories from the database + const memories = await this.memoryManager.list({ + ...(limit !== undefined && { limit }), + ...(pinnedOnly && { pinned: true }), + }); + + if (memories.length === 0) { + return ''; + } + + // Format memories for system prompt + const formattedMemories = memories.map((memory) => { + let formatted = `- ${memory.content}`; + + if (includeTags && memory.tags && memory.tags.length > 0) { + formatted += ` [Tags: ${memory.tags.join(', ')}]`; + } + + if (includeTimestamps) { + const date = new Date(memory.updatedAt).toLocaleDateString(); + formatted += ` (Updated: ${date})`; + } + + return formatted; + }); + + const header = '## User Memories'; + const memoryList = formattedMemories.join('\n'); + const result = `${header}\n${memoryList}`; + + this.logger.debug( + `[MemoryContributor] Loaded ${memories.length} memories into system prompt` + ); + return result; + } catch (error) { + this.logger.error( + `[MemoryContributor] Failed to load memories: ${error instanceof Error ? error.message : String(error)}` + ); + // Return empty string on error to not break system prompt generation + return ''; + } + } +} + +/** + * SkillsContributor lists available skills that the LLM can invoke via the invoke_skill tool. + * This enables the LLM to know what skills are available without hardcoding them. + */ +export class SkillsContributor implements SystemPromptContributor { + private logger: IDextoLogger; + + constructor( + public id: string, + public priority: number, + private promptManager: PromptManager, + logger: IDextoLogger + ) { + this.logger = logger; + this.logger.debug(`[SkillsContributor] Created "${id}"`); + } + + async getContent(_context: DynamicContributorContext): Promise { + try { + const skills = await this.promptManager.listAutoInvocablePrompts(); + const skillEntries = Object.entries(skills); + + if (skillEntries.length === 0) { + return ''; + } + + const skillsList = skillEntries + .map(([_key, info]) => { + const name = info.displayName || info.name; + const desc = info.description ? ` - ${info.description}` : ''; + return `- ${name}${desc}`; + }) + .join('\n'); + + const result = `## Available Skills + +You can invoke the following skills using the \`invoke_skill\` tool when they are relevant to the task: + +${skillsList} + +To use a skill, call invoke_skill with the skill name. The skill will provide specialized instructions for the task.`; + + this.logger.debug( + `[SkillsContributor] Listed ${skillEntries.length} skills in system prompt` + ); + return result; + } catch (error) { + this.logger.error( + `[SkillsContributor] Failed to list skills: ${error instanceof Error ? error.message : String(error)}` + ); + return ''; + } + } +} diff --git a/dexto/packages/core/src/systemPrompt/error-codes.ts b/dexto/packages/core/src/systemPrompt/error-codes.ts new file mode 100644 index 00000000..dbcc6fd6 --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/error-codes.ts @@ -0,0 +1,14 @@ +/** + * SystemPrompt-specific error codes + * Includes file processing and configuration errors + */ +export enum SystemPromptErrorCode { + // File processing + FILE_INVALID_TYPE = 'systemprompt_file_invalid_type', + FILE_TOO_LARGE = 'systemprompt_file_too_large', + FILE_READ_FAILED = 'systemprompt_file_read_failed', + + // Configuration + CONTRIBUTOR_SOURCE_UNKNOWN = 'systemprompt_contributor_source_unknown', + CONTRIBUTOR_CONFIG_INVALID = 'systemprompt_contributor_config_invalid', +} diff --git a/dexto/packages/core/src/systemPrompt/errors.ts b/dexto/packages/core/src/systemPrompt/errors.ts new file mode 100644 index 00000000..bbd71fd1 --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/errors.ts @@ -0,0 +1,75 @@ +import { DextoRuntimeError } from '@core/errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { SystemPromptErrorCode } from './error-codes.js'; +import { safeStringify } from '../utils/safe-stringify.js'; + +/** + * SystemPrompt error factory with typed methods for creating systemPrompt-specific errors + * Each method creates a properly typed DextoRuntimeError with SYSTEM_PROMPT scope + */ +export class SystemPromptError { + /** + * Invalid file type error + */ + static invalidFileType(filePath: string, allowedExtensions: string[]) { + return new DextoRuntimeError( + SystemPromptErrorCode.FILE_INVALID_TYPE, + ErrorScope.SYSTEM_PROMPT, + ErrorType.USER, + `File ${filePath} is not a ${allowedExtensions.join(' or ')} file`, + { filePath, allowedExtensions } + ); + } + + /** + * File too large error + */ + static fileTooLarge(filePath: string, fileSize: number, maxSize: number) { + return new DextoRuntimeError( + SystemPromptErrorCode.FILE_TOO_LARGE, + ErrorScope.SYSTEM_PROMPT, + ErrorType.USER, + `File ${filePath} exceeds maximum size of ${maxSize} bytes`, + { filePath, fileSize, maxSize } + ); + } + + /** + * File read failed error + */ + static fileReadFailed(filePath: string, reason: string) { + return new DextoRuntimeError( + SystemPromptErrorCode.FILE_READ_FAILED, + ErrorScope.SYSTEM_PROMPT, + ErrorType.SYSTEM, + `Failed to read file ${filePath}: ${reason}`, + { filePath, reason } + ); + } + + /** + * Unknown contributor source error + */ + static unknownContributorSource(source: string) { + return new DextoRuntimeError( + SystemPromptErrorCode.CONTRIBUTOR_SOURCE_UNKNOWN, + ErrorScope.SYSTEM_PROMPT, + ErrorType.USER, + `No generator registered for dynamic contributor source: ${source}`, + { source } + ); + } + + /** + * Invalid contributor config error (for exhaustive type checking) + */ + static invalidContributorConfig(config: unknown): DextoRuntimeError { + return new DextoRuntimeError( + SystemPromptErrorCode.CONTRIBUTOR_CONFIG_INVALID, + ErrorScope.SYSTEM_PROMPT, + ErrorType.USER, + `Invalid contributor config: ${safeStringify(config)}`, + { config } + ); + } +} diff --git a/dexto/packages/core/src/systemPrompt/in-built-prompts.ts b/dexto/packages/core/src/systemPrompt/in-built-prompts.ts new file mode 100644 index 00000000..c5635783 --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/in-built-prompts.ts @@ -0,0 +1,102 @@ +import { DynamicContributorContext } from './types.js'; + +/** + * Dynamic Prompt Generators + * + * This module contains functions for generating dynamic system prompts for the AI agent. + * Each function should return a string (or Promise) representing a prompt, possibly using the provided context. + * + * --- + * Guidelines for Adding Prompt Functions: + * - Place all dynamic prompt-generating functions in this file. + * - Also update the `registry.ts` file to register the new function. + * - Use XML tags to indicate the start and end of the dynamic prompt - they are known to improve performance + * - Each function should be named clearly to reflect its purpose (e.g., getCurrentDate, getEnvironmentInfo). + */ + +/** + * Returns the current date (without time to prevent KV-cache invalidation). + */ +export async function getCurrentDate(_context: DynamicContributorContext): Promise { + const date = new Date().toISOString().split('T')[0]; + return `Current date: ${date}`; +} + +/** + * Returns environment information to help agents understand their execution context. + * This is kept separate from date to optimize caching (env info rarely changes). + * + * Includes: + * - Working directory (cwd) + * - Platform (os) + * - Whether the cwd is a git repository + * - Default shell + * + * Note: This function uses dynamic imports for Node.js modules to maintain browser compatibility. + * In browser environments, it returns a placeholder message. + */ +export async function getEnvironmentInfo(_context: DynamicContributorContext): Promise { + // Check if we're in a Node.js environment + if (typeof process === 'undefined' || !process.cwd) { + return 'Environment info not available in browser context'; + } + + try { + // Dynamic imports for Node.js modules (browser-safe) + const [{ existsSync }, { platform }, { join }] = await Promise.all([ + import('fs'), + import('os'), + import('path'), + ]); + + const cwd = process.cwd(); + const os = platform(); + const isGitRepo = existsSync(join(cwd, '.git')); + const shell = process.env.SHELL || (os === 'win32' ? 'cmd.exe' : '/bin/sh'); + + return ` + ${cwd} + ${os} + ${isGitRepo} + ${shell} +`; + } catch { + return 'Environment info not available'; + } +} + +// TODO: This needs to be optimized to only fetch resources when needed. Currently this runs every time the prompt is generated. +export async function getResourceData(context: DynamicContributorContext): Promise { + const resources = await context.mcpManager.listAllResources(); + if (!resources || resources.length === 0) { + return ''; + } + const parts = await Promise.all( + resources.map(async (resource) => { + try { + const response = await context.mcpManager.readResource(resource.key); + const first = response?.contents?.[0]; + let content: string; + if (first && 'text' in first && first.text && typeof first.text === 'string') { + content = first.text; + } else if ( + first && + 'blob' in first && + first.blob && + typeof first.blob === 'string' + ) { + content = first.blob; + } else { + content = JSON.stringify(response, null, 2); + } + const label = resource.summary.name || resource.summary.uri; + return `${content}`; + } catch (error: any) { + return `Error loading resource: ${ + error.message || error + }`; + } + }) + ); + return `\n${parts.join('\n')}\n`; +} diff --git a/dexto/packages/core/src/systemPrompt/index.ts b/dexto/packages/core/src/systemPrompt/index.ts new file mode 100644 index 00000000..4253109a --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/index.ts @@ -0,0 +1,13 @@ +export * from './types.js'; +export * from './manager.js'; +export * from './registry.js'; +export * from './contributors.js'; +export * from './in-built-prompts.js'; +export { + type ContributorConfig, + type SystemPromptConfig, + type ValidatedContributorConfig, + type ValidatedSystemPromptConfig, + ContributorConfigSchema, + SystemPromptConfigSchema, +} from './schemas.js'; diff --git a/dexto/packages/core/src/systemPrompt/manager.test.ts b/dexto/packages/core/src/systemPrompt/manager.test.ts new file mode 100644 index 00000000..30cdb7b1 --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/manager.test.ts @@ -0,0 +1,666 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { SystemPromptManager } from './manager.js'; +import { SystemPromptConfigSchema } from './schemas.js'; +import type { DynamicContributorContext } from './types.js'; +import * as registry from './registry.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { SystemPromptErrorCode } from './error-codes.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import * as path from 'path'; + +// Mock the registry functions +vi.mock('./registry.js', () => ({ + getPromptGenerator: vi.fn(), + PROMPT_GENERATOR_SOURCES: ['date', 'env', 'resources'], +})); + +const mockGetPromptGenerator = vi.mocked(registry.getPromptGenerator); + +describe('SystemPromptManager', () => { + let mockContext: DynamicContributorContext; + let mockLogger: any; + let mockMemoryManager: any; + + beforeEach(() => { + vi.clearAllMocks(); + + mockLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), + } as any; + + mockMemoryManager = { + getMemories: vi.fn().mockResolvedValue([]), + create: vi.fn(), + update: vi.fn(), + delete: vi.fn(), + } as any; + + // Set up default mock generators to prevent "No generator registered" errors + mockGetPromptGenerator.mockImplementation((source) => { + const mockGenerators: Record = { + date: vi.fn().mockResolvedValue('Mock DateTime'), + env: vi.fn().mockResolvedValue('Mock Environment'), + resources: vi.fn().mockResolvedValue('Mock Resources'), + }; + return mockGenerators[source]; + }); + + mockContext = { + mcpManager: {} as any, // Mock MCPManager + }; + }); + + describe('Initialization', () => { + it('should initialize with string config and create static contributor', () => { + const config = SystemPromptConfigSchema.parse('You are a helpful assistant'); + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + + const contributors = manager.getContributors(); + expect(contributors).toHaveLength(1); + expect(contributors[0]?.id).toBe('inline'); + expect(contributors[0]?.priority).toBe(0); + }); + + it('should initialize with empty object config and apply defaults', () => { + const config = SystemPromptConfigSchema.parse({}); + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + + const contributors = manager.getContributors(); + expect(contributors).toHaveLength(2); // date and env are enabled by default + + // Should have date and env (resources is disabled by default) + expect(contributors[0]?.id).toBe('date'); // priority 10, enabled: true + expect(contributors[1]?.id).toBe('env'); // priority 15, enabled: true + }); + + it('should initialize with custom contributors config', () => { + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'main', + type: 'static', + priority: 0, + content: 'You are Dexto', + enabled: true, + }, + { + id: 'date', + type: 'dynamic', + priority: 10, + source: 'date', + enabled: true, + }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const contributors = manager.getContributors(); + + expect(contributors).toHaveLength(2); + expect(contributors[0]?.id).toBe('main'); + expect(contributors[1]?.id).toBe('date'); + }); + + it('should filter out disabled contributors', () => { + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'enabled', + type: 'static', + priority: 0, + content: 'Enabled contributor', + enabled: true, + }, + { + id: 'disabled', + type: 'static', + priority: 5, + content: 'Disabled contributor', + enabled: false, + }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const contributors = manager.getContributors(); + + expect(contributors).toHaveLength(1); + expect(contributors[0]?.id).toBe('enabled'); + }); + + it('should sort contributors by priority (lower number = higher priority)', () => { + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { id: 'low', type: 'static', priority: 20, content: 'Low priority' }, + { id: 'high', type: 'static', priority: 0, content: 'High priority' }, + { id: 'medium', type: 'static', priority: 10, content: 'Medium priority' }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const contributors = manager.getContributors(); + + expect(contributors).toHaveLength(3); + expect(contributors[0]?.id).toBe('high'); // priority 0 + expect(contributors[1]?.id).toBe('medium'); // priority 10 + expect(contributors[2]?.id).toBe('low'); // priority 20 + }); + }); + + describe('Static Contributors', () => { + it('should create static contributors with correct content', async () => { + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'greeting', + type: 'static', + priority: 0, + content: 'Hello, I am Dexto!', + }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const result = await manager.build(mockContext); + + expect(result).toBe('Hello, I am Dexto!'); + }); + + it('should handle multiline static content', async () => { + const multilineContent = `You are Dexto, an AI assistant. + +You can help with: +- Coding tasks +- Analysis +- General questions`; + + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'main', + type: 'static', + priority: 0, + content: multilineContent, + }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const result = await manager.build(mockContext); + + expect(result).toBe(multilineContent); + }); + }); + + describe('Dynamic Contributors', () => { + it('should create dynamic contributors and call generators', async () => { + const mockGenerator = vi.fn().mockResolvedValue('Current time: 2023-01-01'); + mockGetPromptGenerator.mockReturnValue(mockGenerator); + + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'date', + type: 'dynamic', + priority: 10, + source: 'date', + }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const result = await manager.build(mockContext); + + expect(mockGetPromptGenerator).toHaveBeenCalledWith('date'); + expect(mockGenerator).toHaveBeenCalledWith(mockContext); + expect(result).toBe('Current time: 2023-01-01'); + }); + + it('should throw error if generator is not found', () => { + mockGetPromptGenerator.mockReturnValue(undefined); + + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'unknownSource', + type: 'dynamic', + priority: 10, + source: 'date', // valid enum but mock returns undefined + }, + ], + }); + + const error = (() => { + try { + new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + return null; + } catch (e) { + return e; + } + })() as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(SystemPromptErrorCode.CONTRIBUTOR_SOURCE_UNKNOWN); + expect(error.scope).toBe(ErrorScope.SYSTEM_PROMPT); + expect(error.type).toBe(ErrorType.USER); + }); + + it('should handle multiple dynamic contributors', async () => { + const dateTimeGenerator = vi.fn().mockResolvedValue('Time: 2023-01-01'); + const resourcesGenerator = vi.fn().mockResolvedValue('Resources: file1.md, file2.md'); + + mockGetPromptGenerator.mockImplementation((source) => { + if (source === 'date') return dateTimeGenerator; + if (source === 'resources') return resourcesGenerator; + return undefined; + }); + + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { id: 'time', type: 'dynamic', priority: 10, source: 'date' }, + { id: 'files', type: 'dynamic', priority: 20, source: 'resources' }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const result = await manager.build(mockContext); + + expect(result).toBe('Time: 2023-01-01\nResources: file1.md, file2.md'); + expect(dateTimeGenerator).toHaveBeenCalledWith(mockContext); + expect(resourcesGenerator).toHaveBeenCalledWith(mockContext); + }); + }); + + describe('File Contributors', () => { + it('should create file contributors with correct configuration', () => { + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'docs', + type: 'file', + priority: 5, + files: [ + path.join(process.cwd(), 'README.md'), + path.join(process.cwd(), 'GUIDELINES.md'), + ], + options: { + includeFilenames: true, + separator: '\n\n---\n\n', + }, + }, + ], + }); + + const manager = new SystemPromptManager( + config, + '/custom/config/dir', + mockMemoryManager, + undefined, + mockLogger + ); + const contributors = manager.getContributors(); + + expect(contributors).toHaveLength(1); + expect(contributors[0]?.id).toBe('docs'); + expect(contributors[0]?.priority).toBe(5); + }); + + it('should use custom config directory', () => { + const customConfigDir = '/custom/project/path'; + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'docs', + type: 'file', + priority: 5, + files: [path.join(customConfigDir, 'context.md')], + }, + ], + }); + + const manager = new SystemPromptManager( + config, + customConfigDir, + mockMemoryManager, + undefined, + mockLogger + ); + + // The FileContributor should receive the custom config directory + expect(manager.getContributors()).toHaveLength(1); + }); + }); + + describe('Mixed Contributors', () => { + it('should handle mixed contributor types and build correctly', async () => { + const mockGenerator = vi.fn().mockResolvedValue('Dynamic content'); + mockGetPromptGenerator.mockReturnValue(mockGenerator); + + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'static', + type: 'static', + priority: 0, + content: 'Static content', + }, + { + id: 'dynamic', + type: 'dynamic', + priority: 10, + source: 'date', + }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const result = await manager.build(mockContext); + + expect(result).toBe('Static content\nDynamic content'); + }); + + it('should respect priority ordering with mixed types', async () => { + const mockGenerator = vi.fn().mockResolvedValue('Dynamic priority 5'); + mockGetPromptGenerator.mockReturnValue(mockGenerator); + + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'static-low', + type: 'static', + priority: 20, + content: 'Static priority 20', + }, + { id: 'dynamic-high', type: 'dynamic', priority: 5, source: 'date' }, + { + id: 'static-high', + type: 'static', + priority: 0, + content: 'Static priority 0', + }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const result = await manager.build(mockContext); + + // Should be ordered by priority: 0, 5, 20 + expect(result).toBe('Static priority 0\nDynamic priority 5\nStatic priority 20'); + }); + }); + + describe('Build Process', () => { + it('should join multiple contributors with newlines', async () => { + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { id: 'first', type: 'static', priority: 0, content: 'First line' }, + { id: 'second', type: 'static', priority: 10, content: 'Second line' }, + { id: 'third', type: 'static', priority: 20, content: 'Third line' }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const result = await manager.build(mockContext); + + expect(result).toBe('First line\nSecond line\nThird line'); + }); + + it('should handle empty contributor content', async () => { + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { id: 'empty', type: 'static', priority: 0, content: '' }, + { id: 'content', type: 'static', priority: 10, content: 'Has content' }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const result = await manager.build(mockContext); + + expect(result).toBe('\nHas content'); + }); + + it('should pass context correctly to all contributors', async () => { + const mockGenerator1 = vi.fn().mockResolvedValue('Gen1'); + const mockGenerator2 = vi.fn().mockResolvedValue('Gen2'); + + mockGetPromptGenerator.mockImplementation((source) => { + if (source === 'date') return mockGenerator1; + if (source === 'resources') return mockGenerator2; + return undefined; + }); + + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { id: 'gen1', type: 'dynamic', priority: 0, source: 'date' }, + { id: 'gen2', type: 'dynamic', priority: 10, source: 'resources' }, + ], + }); + + const customContext = { + mcpManager: {} as any, // Mock MCPManager + }; + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + await manager.build(customContext); + + expect(mockGenerator1).toHaveBeenCalledWith(customContext); + expect(mockGenerator2).toHaveBeenCalledWith(customContext); + }); + }); + + describe('Error Handling', () => { + it('should handle async errors in contributors gracefully', async () => { + const mockGenerator = vi.fn().mockRejectedValue(new Error('Generator failed')); + mockGetPromptGenerator.mockReturnValue(mockGenerator); + + const config = SystemPromptConfigSchema.parse({ + contributors: [{ id: 'failing', type: 'dynamic', priority: 0, source: 'date' }], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + + await expect(manager.build(mockContext)).rejects.toThrow('Generator failed'); + }); + + it('should use correct config directory default', () => { + const config = SystemPromptConfigSchema.parse('Simple prompt'); + + // Mock process.cwd() to test default behavior + const originalCwd = process.cwd; + process.cwd = vi.fn().mockReturnValue('/mocked/cwd'); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + expect(manager.getContributors()).toHaveLength(1); + + process.cwd = originalCwd; + }); + }); + + describe('Real-world Scenarios', () => { + it('should handle default configuration (empty object)', async () => { + const mockDateTimeGenerator = vi.fn().mockResolvedValue('2023-01-01 12:00:00'); + const mockEnvGenerator = vi.fn().mockResolvedValue('mock'); + const mockResourcesGenerator = vi.fn().mockResolvedValue('Available files: config.yml'); + + mockGetPromptGenerator.mockImplementation((source) => { + if (source === 'date') return mockDateTimeGenerator; + if (source === 'env') return mockEnvGenerator; + if (source === 'resources') return mockResourcesGenerator; + return undefined; + }); + + const config = SystemPromptConfigSchema.parse({}); + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + + // date and env should be enabled by default, resources is disabled + const contributors = manager.getContributors(); + expect(contributors).toHaveLength(2); + expect(contributors[0]?.id).toBe('date'); + expect(contributors[1]?.id).toBe('env'); + + const result = await manager.build(mockContext); + expect(result).toBe('2023-01-01 12:00:00\nmock'); + expect(mockDateTimeGenerator).toHaveBeenCalledWith(mockContext); + expect(mockEnvGenerator).toHaveBeenCalledWith(mockContext); + expect(mockResourcesGenerator).not.toHaveBeenCalled(); + }); + + it('should handle complex configuration with all contributor types', async () => { + const mockGenerator = vi.fn().mockResolvedValue('2023-01-01'); + mockGetPromptGenerator.mockReturnValue(mockGenerator); + + const config = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'intro', + type: 'static', + priority: 0, + content: 'You are Dexto, an advanced AI assistant.', + }, + { + id: 'context', + type: 'file', + priority: 5, + files: [path.join(process.cwd(), 'context.md')], + options: { includeFilenames: true }, + }, + { + id: 'datetime', + type: 'dynamic', + priority: 10, + source: 'date', + }, + ], + }); + + const manager = new SystemPromptManager( + config, + process.cwd(), + mockMemoryManager, + undefined, + mockLogger + ); + const contributors = manager.getContributors(); + + expect(contributors).toHaveLength(3); + expect(contributors[0]?.id).toBe('intro'); + expect(contributors[1]?.id).toBe('context'); + expect(contributors[2]?.id).toBe('datetime'); + }); + }); +}); diff --git a/dexto/packages/core/src/systemPrompt/manager.ts b/dexto/packages/core/src/systemPrompt/manager.ts new file mode 100644 index 00000000..e74749dc --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/manager.ts @@ -0,0 +1,146 @@ +import type { ValidatedSystemPromptConfig, ValidatedContributorConfig } from './schemas.js'; +import { StaticContributor, FileContributor, MemoryContributor } from './contributors.js'; +import { getPromptGenerator } from './registry.js'; +import type { MemoryManager, ValidatedMemoriesConfig } from '../memory/index.js'; + +import type { SystemPromptContributor, DynamicContributorContext } from './types.js'; +import { DynamicContributor } from './contributors.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import { SystemPromptError } from './errors.js'; + +/** + * SystemPromptManager orchestrates registration, loading, and composition + * of both static and dynamic system-prompt contributors. + */ +export class SystemPromptManager { + private contributors: SystemPromptContributor[]; + private configDir: string; + private memoryManager: MemoryManager; + private logger: IDextoLogger; + + // TODO: move config dir logic somewhere else + constructor( + config: ValidatedSystemPromptConfig, + configDir: string, + memoryManager: MemoryManager, + memoriesConfig: ValidatedMemoriesConfig | undefined, + logger: IDextoLogger + ) { + this.configDir = configDir; + this.memoryManager = memoryManager; + this.logger = logger.createChild(DextoLogComponent.SYSTEM_PROMPT); + this.logger.debug(`[SystemPromptManager] Initializing with configDir: ${configDir}`); + + // Filter enabled contributors and create contributor instances + const enabledContributors = config.contributors.filter((c) => c.enabled !== false); + + const contributors: SystemPromptContributor[] = enabledContributors.map((config) => + this.createContributor(config) + ); + + // Add memory contributor if enabled via top-level memories config + if (memoriesConfig?.enabled) { + this.logger.debug( + `[SystemPromptManager] Creating MemoryContributor with options: ${JSON.stringify(memoriesConfig)}` + ); + contributors.push( + new MemoryContributor( + 'memories', + memoriesConfig.priority, + this.memoryManager, + { + includeTimestamps: memoriesConfig.includeTimestamps, + includeTags: memoriesConfig.includeTags, + limit: memoriesConfig.limit, + pinnedOnly: memoriesConfig.pinnedOnly, + }, + this.logger + ) + ); + } + + this.contributors = contributors.sort((a, b) => a.priority - b.priority); // Lower priority number = higher priority + } + + private createContributor(config: ValidatedContributorConfig): SystemPromptContributor { + switch (config.type) { + case 'static': + return new StaticContributor(config.id, config.priority, config.content); + + case 'dynamic': { + const promptGenerator = getPromptGenerator(config.source); + if (!promptGenerator) { + throw SystemPromptError.unknownContributorSource(config.source); + } + return new DynamicContributor(config.id, config.priority, promptGenerator); + } + + case 'file': { + this.logger.debug( + `[SystemPromptManager] Creating FileContributor "${config.id}" with files: ${JSON.stringify(config.files)}` + ); + return new FileContributor( + config.id, + config.priority, + config.files, + config.options, + this.logger + ); + } + + default: { + // Exhaustive check - TypeScript will error if we miss a case + const _exhaustive: never = config; + throw SystemPromptError.invalidContributorConfig(_exhaustive); + } + } + } + + /** + * Build the full system prompt by invoking each contributor and concatenating. + */ + async build(ctx: DynamicContributorContext): Promise { + const parts = await Promise.all( + this.contributors.map(async (contributor) => { + const content = await contributor.getContent(ctx); + this.logger.debug( + `[SystemPrompt] Contributor "${contributor.id}" provided content: ${content.substring(0, 50)}${content.length > 50 ? '...' : ''}` + ); + return content; + }) + ); + return parts.join('\n'); + } + + /** + * Expose current list of contributors (for inspection or testing). + */ + getContributors(): SystemPromptContributor[] { + return this.contributors; + } + + /** + * Add a contributor dynamically after construction. + * The contributor will be inserted in priority order. + */ + addContributor(contributor: SystemPromptContributor): void { + this.contributors.push(contributor); + this.contributors.sort((a, b) => a.priority - b.priority); + this.logger.debug( + `Added contributor: ${contributor.id} (priority: ${contributor.priority})` + ); + } + + /** + * Remove a contributor by ID. + * Returns true if removed, false if not found. + */ + removeContributor(id: string): boolean { + const index = this.contributors.findIndex((c) => c.id === id); + if (index === -1) return false; + this.contributors.splice(index, 1); + this.logger.debug(`Removed contributor: ${id}`); + return true; + } +} diff --git a/dexto/packages/core/src/systemPrompt/registry.ts b/dexto/packages/core/src/systemPrompt/registry.ts new file mode 100644 index 00000000..3d61eebf --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/registry.ts @@ -0,0 +1,26 @@ +import * as handlers from './in-built-prompts.js'; +import { DynamicContributorContext } from './types.js'; + +/** + * This file contains the registry of all the functions that can generate dynamic prompt pieces at runtime. + */ +export type DynamicPromptGenerator = (context: DynamicContributorContext) => Promise; + +// Available dynamic prompt generator sources +export const PROMPT_GENERATOR_SOURCES = ['date', 'env', 'resources'] as const; + +export type PromptGeneratorSource = (typeof PROMPT_GENERATOR_SOURCES)[number]; + +// Registry mapping sources to their generator functions +export const PROMPT_GENERATOR_REGISTRY: Record = { + date: handlers.getCurrentDate, + env: handlers.getEnvironmentInfo, + resources: handlers.getResourceData, +}; + +// To fetch a prompt generator function from its source +export function getPromptGenerator( + source: PromptGeneratorSource +): DynamicPromptGenerator | undefined { + return PROMPT_GENERATOR_REGISTRY[source]; +} diff --git a/dexto/packages/core/src/systemPrompt/schemas.test.ts b/dexto/packages/core/src/systemPrompt/schemas.test.ts new file mode 100644 index 00000000..1400d062 --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/schemas.test.ts @@ -0,0 +1,363 @@ +import { describe, it, expect } from 'vitest'; +import * as path from 'path'; +import { + SystemPromptConfigSchema, + type SystemPromptConfig, + type ValidatedSystemPromptConfig, +} from './schemas.js'; + +describe('SystemPromptConfigSchema', () => { + describe('String Input Transform', () => { + it('should transform string to contributors object', () => { + const result = SystemPromptConfigSchema.parse('You are a helpful assistant'); + + expect(result.contributors).toHaveLength(1); + const contributor = result.contributors[0]; + expect(contributor).toEqual({ + id: 'inline', + type: 'static', + content: 'You are a helpful assistant', + priority: 0, + enabled: true, + }); + }); + + it('should handle empty string', () => { + const result = SystemPromptConfigSchema.parse(''); + + expect(result.contributors).toHaveLength(1); + const contributor = result.contributors[0]; + if (contributor?.type === 'static') { + expect(contributor.content).toBe(''); + } + }); + + it('should handle multiline string', () => { + const multilinePrompt = `You are Dexto, an AI assistant. + +You can help with: +- Coding tasks +- Analysis +- General questions`; + + const result = SystemPromptConfigSchema.parse(multilinePrompt); + + expect(result.contributors).toHaveLength(1); + const contributor = result.contributors[0]; + if (contributor?.type === 'static') { + expect((contributor! as any).content).toBe(multilinePrompt); + } + }); + }); + + describe('Object Input Validation', () => { + it('should apply default contributors for empty object', () => { + const result = SystemPromptConfigSchema.parse({}); + + expect(result.contributors).toHaveLength(3); + expect(result.contributors[0]).toEqual({ + id: 'date', + type: 'dynamic', + priority: 10, + source: 'date', + enabled: true, + }); + expect(result.contributors[1]).toEqual({ + id: 'env', + type: 'dynamic', + priority: 15, + source: 'env', + enabled: true, + }); + expect(result.contributors[2]).toEqual({ + id: 'resources', + type: 'dynamic', + priority: 20, + source: 'resources', + enabled: false, + }); + }); + + it('should allow overriding default contributors', () => { + const result = SystemPromptConfigSchema.parse({ + contributors: [ + { + id: 'custom', + type: 'static', + priority: 0, + content: 'Custom prompt', + enabled: true, + }, + ], + }); + + expect(result.contributors).toHaveLength(1); + expect(result.contributors[0]?.id).toBe('custom'); + }); + + it('should pass through valid contributors object', () => { + const contributorsConfig = { + contributors: [ + { + id: 'main', + type: 'static' as const, + priority: 0, + content: 'You are Dexto', + enabled: true, + }, + { + id: 'date', + type: 'dynamic' as const, + priority: 10, + source: 'date', + enabled: true, + }, + ], + }; + + const result = SystemPromptConfigSchema.parse(contributorsConfig); + expect(result).toEqual(contributorsConfig); + }); + }); + + describe('Contributor Type Validation', () => { + it('should validate static contributors', () => { + const validStatic = { + contributors: [{ id: 'test', type: 'static', priority: 0, content: 'hello world' }], + }; + const validResult = SystemPromptConfigSchema.parse(validStatic); + expect(validResult.contributors[0]?.type).toBe('static'); + + const invalidStatic = { + contributors: [ + { id: 'test', type: 'static', priority: 0 }, // Missing content + ], + }; + const result = SystemPromptConfigSchema.safeParse(invalidStatic); + expect(result.success).toBe(false); + // For union schemas, the actual error is in unionErrors[1] (second branch - the object branch) + // TODO: Fix typing - unionErrors not properly typed in Zod + const unionError = result.error?.issues[0] as any; + expect(unionError?.code).toBe('invalid_union'); + const objectErrors = unionError?.unionErrors?.[1]?.issues; + expect(objectErrors?.[0]?.path).toEqual(['contributors', 0, 'content']); + expect(objectErrors?.[0]?.code).toBe('invalid_type'); + }); + + it('should validate dynamic contributors', () => { + const validDynamic = { + contributors: [{ id: 'date', type: 'dynamic', priority: 10, source: 'date' }], + }; + const validResult = SystemPromptConfigSchema.parse(validDynamic); + expect(validResult.contributors[0]?.type).toBe('dynamic'); + + const invalidDynamic = { + contributors: [ + { id: 'date', type: 'dynamic', priority: 10 }, // Missing source + ], + }; + const result = SystemPromptConfigSchema.safeParse(invalidDynamic); + expect(result.success).toBe(false); + // For union schemas, the actual error is in unionErrors[1] (second branch - the object branch) + // TODO: Fix typing - unionErrors not properly typed in Zod + const unionError = result.error?.issues[0] as any; + expect(unionError?.code).toBe('invalid_union'); + const objectErrors = unionError?.unionErrors?.[1]?.issues; + expect(objectErrors?.[0]?.path).toEqual(['contributors', 0, 'source']); + expect(objectErrors?.[0]?.code).toBe('invalid_type'); + }); + + it('should validate dynamic contributor source enum', () => { + const validSources = ['date', 'env', 'resources']; + + for (const source of validSources) { + const validConfig = { + contributors: [{ id: 'test', type: 'dynamic', priority: 10, source }], + }; + const result = SystemPromptConfigSchema.parse(validConfig); + expect(result.contributors[0]?.type).toBe('dynamic'); + } + + const invalidSource = { + contributors: [ + { id: 'test', type: 'dynamic', priority: 10, source: 'invalidSource' }, // Invalid enum value + ], + }; + const result = SystemPromptConfigSchema.safeParse(invalidSource); + expect(result.success).toBe(false); + // For union schemas, the actual error is in unionErrors[1] (second branch - the object branch) + // TODO: Fix typing - unionErrors not properly typed in Zod + const unionError = result.error?.issues[0] as any; + expect(unionError?.code).toBe('invalid_union'); + const objectErrors = unionError?.unionErrors?.[1]?.issues; + expect(objectErrors?.[0]?.path).toEqual(['contributors', 0, 'source']); + expect(objectErrors?.[0]?.code).toBe('invalid_enum_value'); + }); + + it('should validate file contributors', () => { + const validFile = { + contributors: [ + { + id: 'docs', + type: 'file', + priority: 5, + files: [path.join(process.cwd(), 'README.md')], + }, + ], + }; + const validResult = SystemPromptConfigSchema.parse(validFile); + expect(validResult.contributors[0]?.type).toBe('file'); + + const invalidFile = { + contributors: [ + { id: 'docs', type: 'file', priority: 5, files: [] }, // Empty files array + ], + }; + const result = SystemPromptConfigSchema.safeParse(invalidFile); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['contributors', 0, 'files']); + + const relativePathFile = { + contributors: [ + { id: 'docs', type: 'file', priority: 5, files: ['relative/path.md'] }, + ], + }; + const relativeResult = SystemPromptConfigSchema.safeParse(relativePathFile); + expect(relativeResult.success).toBe(false); + }); + + it('should reject invalid contributor types', () => { + const result = SystemPromptConfigSchema.safeParse({ + contributors: [ + { id: 'invalid', type: 'invalid', priority: 0 }, // Invalid type + ], + }); + expect(result.success).toBe(false); + // For union schemas, the actual error is in unionErrors[1] (second branch - the object branch) + // TODO: Fix typing - unionErrors not properly typed in Zod + const unionError = result.error?.issues[0] as any; + expect(unionError?.code).toBe('invalid_union'); + const objectErrors = unionError?.unionErrors?.[1]?.issues; + expect(objectErrors?.[0]?.path).toEqual(['contributors', 0, 'type']); + expect(objectErrors?.[0]?.code).toBe('invalid_union_discriminator'); + }); + + it('should reject extra fields with strict validation', () => { + const result = SystemPromptConfigSchema.safeParse({ + contributors: [{ id: 'test', type: 'static', priority: 0, content: 'test' }], + unknownField: 'should fail', + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe('unrecognized_keys'); + }); + }); + + describe('Type Safety', () => { + it('should handle input and output types correctly', () => { + // Input can be string or object + const stringInput: SystemPromptConfig = 'Hello world'; + const objectInput: SystemPromptConfig = { + contributors: [{ id: 'test', type: 'static', priority: 0, content: 'test' }], + }; + + const stringResult = SystemPromptConfigSchema.parse(stringInput); + const objectResult = SystemPromptConfigSchema.parse(objectInput); + + // Both should produce ValidatedSystemPromptConfig (object only) + expect(stringResult.contributors).toBeDefined(); + expect(objectResult.contributors).toBeDefined(); + }); + + it('should produce consistent output type', () => { + const stringResult: ValidatedSystemPromptConfig = + SystemPromptConfigSchema.parse('test'); + const objectResult: ValidatedSystemPromptConfig = SystemPromptConfigSchema.parse({ + contributors: [{ id: 'test', type: 'static', priority: 0, content: 'test' }], + }); + + // Both results should have the same type structure + expect(typeof stringResult.contributors).toBe('object'); + expect(typeof objectResult.contributors).toBe('object'); + expect(Array.isArray(stringResult.contributors)).toBe(true); + expect(Array.isArray(objectResult.contributors)).toBe(true); + }); + }); + + describe('Real-world Scenarios', () => { + it('should handle simple string prompt', () => { + const result = SystemPromptConfigSchema.parse('You are a coding assistant'); + + expect(result.contributors).toHaveLength(1); + const contributor = result.contributors[0]; + expect(contributor?.type).toBe('static'); + if (contributor?.type === 'static') { + expect((contributor! as any).content).toBe('You are a coding assistant'); + } + }); + + it('should handle complex contributors configuration', () => { + const complexConfig = { + contributors: [ + { + id: 'main', + type: 'static' as const, + priority: 0, + content: 'You are Dexto, an advanced AI assistant.', + enabled: true, + }, + { + id: 'context', + type: 'file' as const, + priority: 5, + files: [ + path.join(process.cwd(), 'context.md'), + path.join(process.cwd(), 'guidelines.md'), + ], + enabled: true, + options: { + includeFilenames: true, + separator: '\n\n---\n\n', + }, + }, + { + id: 'date', + type: 'dynamic' as const, + priority: 10, + source: 'date', + enabled: true, + }, + ], + }; + + const result = SystemPromptConfigSchema.parse(complexConfig); + expect(result.contributors).toHaveLength(3); + expect(result.contributors.map((c) => c.id)).toEqual(['main', 'context', 'date']); + }); + + it('should handle template-style configuration', () => { + const templateConfig = { + contributors: [ + { + id: 'primary', + type: 'static' as const, + priority: 0, + content: + "You are a helpful AI assistant demonstrating Dexto's capabilities.", + }, + { + id: 'date', + type: 'dynamic' as const, + priority: 10, + source: 'date', + enabled: true, + }, + ], + }; + + const result = SystemPromptConfigSchema.parse(templateConfig); + expect(result.contributors).toHaveLength(2); + expect(result.contributors[0]?.id).toBe('primary'); + expect(result.contributors[1]?.id).toBe('date'); + }); + }); +}); diff --git a/dexto/packages/core/src/systemPrompt/schemas.ts b/dexto/packages/core/src/systemPrompt/schemas.ts new file mode 100644 index 00000000..9982fa37 --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/schemas.ts @@ -0,0 +1,163 @@ +import { z } from 'zod'; +import * as path from 'path'; +import { PROMPT_GENERATOR_SOURCES } from './registry.js'; + +// Define a base schema for common fields +const BaseContributorSchema = z + .object({ + id: z.string().describe('Unique identifier for the contributor'), + priority: z + .number() + .int() + .nonnegative() + .describe('Execution priority of the contributor (lower numbers run first)'), + enabled: z + .boolean() + .optional() + .default(true) + .describe('Whether this contributor is currently active'), + }) + .strict(); +// Schema for 'static' contributors - only includes relevant fields +const StaticContributorSchema = BaseContributorSchema.extend({ + type: z.literal('static'), + content: z.string().describe("Static content for the contributor (REQUIRED for 'static')"), + // No 'source' field here, as it's not relevant to static contributors +}).strict(); +// Schema for 'dynamic' contributors - only includes relevant fields +const DynamicContributorSchema = BaseContributorSchema.extend({ + type: z.literal('dynamic'), + source: z + .enum(PROMPT_GENERATOR_SOURCES) + .describe("Source identifier for dynamic content (REQUIRED for 'dynamic')"), + // No 'content' field here, as it's not relevant to dynamic contributors (source provides the content) +}).strict(); +// Schema for 'file' contributors - includes file-specific configuration +const FileContributorSchema = BaseContributorSchema.extend({ + type: z.literal('file'), + files: z + .array( + z.string().superRefine((filePath, ctx) => { + if (!path.isAbsolute(filePath)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + 'FileContributor paths must be absolute after template expansion (use ${{dexto.agent_dir}} or provide an absolute path).', + }); + } + }) + ) + .min(1) + .describe('Array of file paths to include as context (.md and .txt files)'), + options: z + .object({ + includeFilenames: z + .boolean() + .optional() + .default(true) + .describe('Whether to include the filename as a header for each file'), + separator: z + .string() + .optional() + .default('\n\n---\n\n') + .describe('Separator to use between multiple files'), + errorHandling: z + .enum(['skip', 'error']) + .optional() + .default('skip') + .describe( + 'How to handle missing or unreadable files: skip (ignore) or error (throw)' + ), + maxFileSize: z + .number() + .int() + .positive() + .optional() + .default(100000) + .describe('Maximum file size in bytes (default: 100KB)'), + includeMetadata: z + .boolean() + .optional() + .default(false) + .describe( + 'Whether to include file metadata (size, modification time) in the context' + ), + }) + .strict() + .optional() + .default({}), +}).strict(); + +export const ContributorConfigSchema = z + .discriminatedUnion( + 'type', // The field to discriminate on + [StaticContributorSchema, DynamicContributorSchema, FileContributorSchema], + { + // Optional: Custom error message for invalid discriminator + errorMap: (issue, ctx) => { + if (issue.code === z.ZodIssueCode.invalid_union_discriminator) { + return { + message: `Invalid contributor type. Expected 'static', 'dynamic', or 'file'. Note: memory contributors are now configured via the top-level 'memories' config.`, + }; + } + return { message: ctx.defaultError }; + }, + } + ) + .describe( + "Configuration for a system prompt contributor. Type 'static' requires 'content', type 'dynamic' requires 'source', type 'file' requires 'files'." + ); +// Input type for user-facing API (pre-parsing) + +export type ContributorConfig = z.input; +// Validated type for internal use (post-parsing) +export type ValidatedContributorConfig = z.output; + +export const SystemPromptContributorsSchema = z + .object({ + contributors: z + .array(ContributorConfigSchema) + .min(1) + .default([ + { + id: 'date', + type: 'dynamic', + priority: 10, + source: 'date', + enabled: true, + }, + { + id: 'env', + type: 'dynamic', + priority: 15, + source: 'env', + enabled: true, + }, + { + id: 'resources', + type: 'dynamic', + priority: 20, + source: 'resources', + enabled: false, + }, + ] as const) + .describe('An array of contributor configurations that make up the system prompt'), + }) + .strict(); + +// Add the union with transform - handles string | object input +export const SystemPromptConfigSchema = z + .union([ + z.string().transform((str) => ({ + contributors: [ + { id: 'inline', type: 'static' as const, content: str, priority: 0, enabled: true }, + ], + })), + SystemPromptContributorsSchema, + ]) + .describe('Plain string or structured contributors object') + .brand<'ValidatedSystemPromptConfig'>(); + +// Type definitions +export type SystemPromptConfig = z.input; // string | object (user input) +export type ValidatedSystemPromptConfig = z.output; // object only (parsed output) diff --git a/dexto/packages/core/src/systemPrompt/types.ts b/dexto/packages/core/src/systemPrompt/types.ts new file mode 100644 index 00000000..782eb43b --- /dev/null +++ b/dexto/packages/core/src/systemPrompt/types.ts @@ -0,0 +1,13 @@ +import { MCPManager } from '../mcp/manager.js'; + +// Context passed to dynamic contributors +export interface DynamicContributorContext { + mcpManager: MCPManager; +} + +// Interface for all system prompt contributors +export interface SystemPromptContributor { + id: string; + priority: number; + getContent(context: DynamicContributorContext): Promise; +} diff --git a/dexto/packages/core/src/telemetry/README.md b/dexto/packages/core/src/telemetry/README.md new file mode 100644 index 00000000..d6da6225 --- /dev/null +++ b/dexto/packages/core/src/telemetry/README.md @@ -0,0 +1,199 @@ +# Telemetry Module + +OpenTelemetry distributed tracing for Dexto agent operations. + +## What It Does + +- **Traces execution flow** across DextoAgent, LLM services, and tool operations +- **Captures token usage** for all LLM calls (input/output/total tokens) +- **Exports to OTLP-compatible backends** (Jaeger, Grafana, etc.) +- **Zero overhead when disabled** - all instrumentation is opt-in + +## Architecture + +### Decorator-Based Instrumentation + +Uses `@InstrumentClass` decorator on critical execution paths: + +- `DextoAgent` - Top-level orchestrator +- `VercelLLMService` - LLM operations (all providers via Vercel AI SDK) +- `ToolManager` - Tool execution + +**Not decorated** (following selective instrumentation strategy): +- Low-level services (MCPManager, SessionManager, PluginManager) +- Storage/memory operations (ResourceManager, MemoryManager) + +### Initialization + +Telemetry is initialized in `createAgentServices()` **before** any decorated classes are instantiated: + +```typescript +// packages/core/src/utils/service-initializer.ts +if (config.telemetry?.enabled) { + await Telemetry.init(config.telemetry); +} +``` + +### Agent Switching + +For sequential agent switching, telemetry is shut down before creating the new agent: + +```typescript +// packages/cli/src/api/server.ts +await Telemetry.shutdownGlobal(); // Old telemetry +newAgent = await getDexto().createAgent(agentId); // Fresh telemetry +``` + +## Configuration + +Enable in your agent config: + +```yaml +# agents/my-agent.yml +telemetry: + enabled: true + serviceName: my-dexto-agent + tracerName: dexto-tracer + export: + type: otlp + protocol: http + endpoint: http://localhost:4318/v1/traces +``` + +## Testing with Jaeger + +### 1. Start Jaeger + +```bash +docker run -d \ + --name jaeger \ + -p 16686:16686 \ + -p 4318:4318 \ + jaegertracing/all-in-one:latest +``` + +**Ports:** +- `16686` - Jaeger UI (web interface) +- `4318` - OTLP HTTP receiver (where Dexto sends traces) + +### 2. Enable Telemetry + +Telemetry is already enabled in `agents/default-agent.yml`. To disable, set `enabled: false`. + +### 3. Run Dexto webUI + +```bash +# Run in CLI mode +pnpm run dev +``` + +### 4. Generate Traces + +Send messages through CLI or WebUI to generate traces. + +### 5. View Traces + +1. Open Jaeger UI: http://localhost:16686 +2. Select service: `dexto-default-agent` +3. Click "Find Traces" +4. Select an operation: `agent.run` + +### 6. Verify Trace Structure + +Click on a trace to see the span hierarchy: + +``` +agent.run (20.95s total) + ├─ agent.maybeGenerateTitle (14.99ms) + └─ llm.vercel.completeTask (20.93s) + └─ llm.vercel.streamText (20.92s) + ├─ POST https://api.openai.com/... (10.01s) ← HTTP auto-instrumentation + └─ POST https://api.openai.com/... (10.79s) ← HTTP auto-instrumentation +``` + +**What to verify:** +- ✅ Span names use correct prefixes (`agent.`, `llm.vercel.`) +- ✅ Span hierarchy shows parent-child relationships +- ✅ HTTP auto-instrumentation captures API calls +- ✅ Token usage attributes: `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` +- ✅ No errors in console logs + +### 7. Cleanup + +```bash +docker stop jaeger +docker rm jaeger +``` + +## Module Structure + +``` +telemetry/ +├── README.md # This file +├── telemetry.ts # Core Telemetry class, SDK initialization +├── decorators.ts # @InstrumentClass decorator implementation +├── schemas.ts # Zod schemas for telemetry config +├── types.ts # TypeScript types for spans and traces +├── exporters.ts # CompositeExporter for multi-destination support +└── utils.ts # Helper functions +``` + +## Key Files + +### `telemetry.ts` +- `Telemetry.init(config)` - Initialize OpenTelemetry SDK +- `Telemetry.shutdownGlobal()` - Shutdown for agent switching +- `Telemetry.get()` - Get initialized instance + +### `decorators.ts` +- `@InstrumentClass(options)` - Decorator for automatic tracing +- `withSpan(spanName, fn, options)` - Manual span creation + +### `exporters.ts` +- `CompositeExporter` - Multi-destination exporting with recursive telemetry filtering + +## Adding Telemetry to New Modules + +Use the `@InstrumentClass` decorator on classes in critical execution paths: + +```typescript +import { InstrumentClass } from '../telemetry/decorators.js'; + +@InstrumentClass({ + prefix: 'mymodule', // Span prefix: mymodule.methodName + excludeMethods: ['helper'] // Methods to skip +}) +export class MyModule { + async process(data: string): Promise { + // Span automatically created: "mymodule.process" + // Add custom attributes to active span: + const span = trace.getActiveSpan(); + if (span) { + span.setAttribute('data.length', data.length); + } + } +} +``` + +## Troubleshooting + +**No traces appearing in Jaeger?** +1. Verify Jaeger is running: `docker ps | grep jaeger` +2. Check endpoint in agent config: `http://localhost:4318/v1/traces` +3. Check console for "Telemetry initialized" log +4. Verify `enabled: true` in telemetry config + +**Only seeing HTTP GET/POST spans?** +- These are from OpenTelemetry's automatic HTTP instrumentation (expected!) +- Filter by Operation: `agent.run` to see decorated spans +- Click into a trace to see the full hierarchy + +**Build errors?** +- Run `pnpm install` if dependencies are missing +- Ensure you're on the `telemetry` branch + +## Further Documentation + +- Full feature plan: `/feature-plans/telemetry.md` +- Configuration options: See `schemas.ts` +- OpenTelemetry docs: https://opentelemetry.io/docs/ diff --git a/dexto/packages/core/src/telemetry/decorators.ts b/dexto/packages/core/src/telemetry/decorators.ts new file mode 100644 index 00000000..5b4173d2 --- /dev/null +++ b/dexto/packages/core/src/telemetry/decorators.ts @@ -0,0 +1,271 @@ +import { + trace, + context, + SpanStatusCode, + SpanKind, + propagation, + SpanOptions, + type BaggageEntry, +} from '@opentelemetry/api'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { hasActiveTelemetry, getBaggageValues } from './utils.js'; +import { safeStringify } from '../utils/safe-stringify.js'; + +// Decorator factory that takes optional spanName +export function withSpan(options: { + spanName?: string; + skipIfNoTelemetry?: boolean; + spanKind?: SpanKind; + tracerName?: string; +}): any { + return function ( + _target: unknown, + propertyKey: string | symbol, + descriptor?: PropertyDescriptor | number + ) { + if (!descriptor || typeof descriptor === 'number') return; + + const originalMethod = descriptor.value as Function; + const methodName = String(propertyKey); + + descriptor.value = function (this: unknown, ...args: unknown[]) { + // Try to get logger from instance for DI pattern (optional) + const logger = (this as any)?.logger as IDextoLogger | undefined; + + // Skip if no telemetry is available and skipIfNoTelemetry is true + // Guard against Telemetry.get() throwing if globalThis.__TELEMETRY__ is not yet defined + if ( + options?.skipIfNoTelemetry && + (!globalThis.__TELEMETRY__ || !hasActiveTelemetry(logger)) + ) { + return originalMethod.apply(this, args); + } + const tracer = trace.getTracer(options?.tracerName ?? 'dexto'); + + // Determine span name and kind + let spanName: string = methodName; // Default spanName + let spanKind: SpanKind | undefined; + + if (options) { + // options is always an object here due to decorator factory + spanName = options.spanName ?? methodName; + if (options.spanKind !== undefined) { + spanKind = options.spanKind; + } + } + + // Start the span with optional kind + const spanOptions: SpanOptions = {}; + if (spanKind !== undefined) { + spanOptions.kind = spanKind; + } + const span = tracer.startSpan(spanName, spanOptions); + let ctx = trace.setSpan(context.active(), span); + + // Record input arguments as span attributes (sanitized and truncated) + args.forEach((arg, index) => { + span.setAttribute(`${spanName}.argument.${index}`, safeStringify(arg, 8192)); + }); + + // Extract baggage values from the current context (may include values set by parent spans) + const { requestId, componentName, runId, threadId, resourceId, sessionId } = + getBaggageValues(ctx); + + // Add all baggage values to span attributes + // Set both direct attributes and baggage-prefixed versions for storage schema fallback + if (sessionId) { + span.setAttribute('sessionId', sessionId); + span.setAttribute('baggage.sessionId', sessionId); // Fallback for storage + } + + if (requestId) { + span.setAttribute('http.request_id', requestId); + span.setAttribute('baggage.http.request_id', requestId); + } + + if (threadId) { + span.setAttribute('threadId', threadId); + span.setAttribute('baggage.threadId', threadId); + } + + if (resourceId) { + span.setAttribute('resourceId', resourceId); + span.setAttribute('baggage.resourceId', resourceId); + } + + if (runId !== undefined) { + span.setAttribute('runId', String(runId)); + span.setAttribute('baggage.runId', String(runId)); + } + + if (componentName) { + span.setAttribute('componentName', componentName); + span.setAttribute('baggage.componentName', componentName); + } else if (this && typeof this === 'object') { + const contextObj = this as { + name?: string; + runId?: string; + constructor?: { name?: string }; + }; + // Prefer instance.name, fallback to constructor.name + const inferredName = contextObj.name ?? contextObj.constructor?.name; + if (inferredName) { + span.setAttribute('componentName', inferredName); + } + if (contextObj.runId) { + span.setAttribute('runId', contextObj.runId); + span.setAttribute('baggage.runId', contextObj.runId); + } + + // Merge with existing baggage to preserve parent context values + const existingBaggage = propagation.getBaggage(ctx); + const baggageEntries: Record = {}; + + // Copy all existing baggage entries to preserve custom baggage + if (existingBaggage) { + existingBaggage.getAllEntries().forEach(([key, entry]) => { + baggageEntries[key] = entry; + }); + } + + // Preserve existing baggage values and metadata + if (sessionId !== undefined) { + baggageEntries.sessionId = { + ...baggageEntries.sessionId, + value: String(sessionId), + }; + } + if (requestId !== undefined) { + baggageEntries['http.request_id'] = { + ...baggageEntries['http.request_id'], + value: String(requestId), + }; + } + if (threadId !== undefined) { + baggageEntries.threadId = { + ...baggageEntries.threadId, + value: String(threadId), + }; + } + if (resourceId !== undefined) { + baggageEntries.resourceId = { + ...baggageEntries.resourceId, + value: String(resourceId), + }; + } + + // Add new component-specific baggage values + if (inferredName !== undefined) { + baggageEntries.componentName = { + ...baggageEntries.componentName, + value: String(inferredName), + }; + } + if (contextObj.runId !== undefined) { + baggageEntries.runId = { + ...baggageEntries.runId, + value: String(contextObj.runId), + }; + } + + if (Object.keys(baggageEntries).length > 0) { + ctx = propagation.setBaggage(ctx, propagation.createBaggage(baggageEntries)); + } + } + + let result: unknown; + try { + // Call the original method within the context + result = context.with(ctx, () => originalMethod.apply(this, args)); + + // Handle promises + if (result instanceof Promise) { + return result + .then((resolvedValue) => { + span.setAttribute( + `${spanName}.result`, + safeStringify(resolvedValue, 8192) + ); + return resolvedValue; + }) + .catch((error) => { + span.recordException(error); + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error?.toString(), + }); + throw error; + }) + .finally(() => { + span.end(); + }); + } + + // Record result for non-promise returns (sanitized and truncated) + span.setAttribute(`${spanName}.result`, safeStringify(result, 8192)); + // Return regular results + return result; + } catch (error) { + // Try to use instance logger if available (DI pattern) + const logger = (this as any)?.logger as IDextoLogger | undefined; + logger?.error( + `withSpan: Error in method '${methodName}': ${error instanceof Error ? error.message : String(error)}`, + { error } + ); + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : 'Unknown error', + }); + if (error instanceof Error) { + span.recordException(error); + } + throw error; + } finally { + // End span for non-promise returns + if (!(result instanceof Promise)) { + span.end(); + } + } + }; + + return descriptor; + }; +} + +// class-telemetry.decorator.ts +export function InstrumentClass(options?: { + prefix?: string; + spanKind?: SpanKind; + excludeMethods?: string[]; + methodFilter?: (methodName: string) => boolean; + tracerName?: string; +}) { + return function (target: T) { + const methods = Object.getOwnPropertyNames(target.prototype); + methods.forEach((method) => { + // Skip excluded methods + if (options?.excludeMethods?.includes(method) || method === 'constructor') { + return; + } + // Apply method filter if provided + if (options?.methodFilter && !options.methodFilter(method)) return; + + const descriptor = Object.getOwnPropertyDescriptor(target.prototype, method); + if (descriptor && typeof descriptor.value === 'function') { + Object.defineProperty( + target.prototype, + method, + withSpan({ + spanName: options?.prefix ? `${options.prefix}.${method}` : method, + skipIfNoTelemetry: true, + spanKind: options?.spanKind || SpanKind.INTERNAL, + ...(options?.tracerName !== undefined && { + tracerName: options.tracerName, + }), + })(target, method, descriptor) + ); + } + }); + return target; + }; +} diff --git a/dexto/packages/core/src/telemetry/error-codes.ts b/dexto/packages/core/src/telemetry/error-codes.ts new file mode 100644 index 00000000..b3459632 --- /dev/null +++ b/dexto/packages/core/src/telemetry/error-codes.ts @@ -0,0 +1,19 @@ +/** + * Telemetry-specific error codes + * Covers initialization, dependencies, and export operations + */ +export enum TelemetryErrorCode { + // Initialization errors + INITIALIZATION_FAILED = 'telemetry_initialization_failed', + NOT_INITIALIZED = 'telemetry_not_initialized', + + // Dependency errors + DEPENDENCY_NOT_INSTALLED = 'telemetry_dependency_not_installed', + EXPORTER_DEPENDENCY_NOT_INSTALLED = 'telemetry_exporter_dependency_not_installed', + + // Configuration errors + INVALID_CONFIG = 'telemetry_invalid_config', + + // Shutdown errors + SHUTDOWN_FAILED = 'telemetry_shutdown_failed', +} diff --git a/dexto/packages/core/src/telemetry/errors.ts b/dexto/packages/core/src/telemetry/errors.ts new file mode 100644 index 00000000..0cd898d2 --- /dev/null +++ b/dexto/packages/core/src/telemetry/errors.ts @@ -0,0 +1,91 @@ +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import { TelemetryErrorCode } from './error-codes.js'; + +/** + * Telemetry error factory with typed methods for creating telemetry-specific errors + * Each method creates a properly typed error with TELEMETRY scope + */ +export class TelemetryError { + /** + * Required OpenTelemetry dependencies not installed + */ + static dependencyNotInstalled(packages: string[]): DextoRuntimeError { + return new DextoRuntimeError( + TelemetryErrorCode.DEPENDENCY_NOT_INSTALLED, + ErrorScope.TELEMETRY, + ErrorType.USER, + 'Telemetry is enabled but required OpenTelemetry packages are not installed.', + { + packages, + hint: `Install with: npm install ${packages.join(' ')}`, + recovery: 'Or disable telemetry by setting enabled: false in your configuration.', + } + ); + } + + /** + * Specific exporter dependency not installed (gRPC or HTTP) + */ + static exporterDependencyNotInstalled( + exporterType: 'grpc' | 'http', + packageName: string + ): DextoRuntimeError { + return new DextoRuntimeError( + TelemetryErrorCode.EXPORTER_DEPENDENCY_NOT_INSTALLED, + ErrorScope.TELEMETRY, + ErrorType.USER, + `OTLP ${exporterType.toUpperCase()} exporter configured but '${packageName}' is not installed.`, + { + exporterType, + packageName, + hint: `Install with: npm install ${packageName}`, + } + ); + } + + /** + * Telemetry initialization failed + */ + static initializationFailed(reason: string, originalError?: unknown): DextoRuntimeError { + return new DextoRuntimeError( + TelemetryErrorCode.INITIALIZATION_FAILED, + ErrorScope.TELEMETRY, + ErrorType.SYSTEM, + `Failed to initialize telemetry: ${reason}`, + { + reason, + originalError: + originalError instanceof Error ? originalError.message : String(originalError), + } + ); + } + + /** + * Telemetry not initialized when expected + */ + static notInitialized(): DextoRuntimeError { + return new DextoRuntimeError( + TelemetryErrorCode.NOT_INITIALIZED, + ErrorScope.TELEMETRY, + ErrorType.USER, + 'Telemetry not initialized. Call Telemetry.init() first.', + { + hint: 'Ensure telemetry is initialized before accessing the global instance.', + } + ); + } + + /** + * Telemetry shutdown failed (non-blocking warning) + */ + static shutdownFailed(reason: string): DextoRuntimeError { + return new DextoRuntimeError( + TelemetryErrorCode.SHUTDOWN_FAILED, + ErrorScope.TELEMETRY, + ErrorType.SYSTEM, + `Telemetry shutdown failed: ${reason}`, + { reason } + ); + } +} diff --git a/dexto/packages/core/src/telemetry/exporters.ts b/dexto/packages/core/src/telemetry/exporters.ts new file mode 100644 index 00000000..241adc6b --- /dev/null +++ b/dexto/packages/core/src/telemetry/exporters.ts @@ -0,0 +1,131 @@ +import { ExportResultCode } from '@opentelemetry/core'; +import type { ExportResult } from '@opentelemetry/core'; +import type { ReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-base'; + +/** + * Normalizes URL paths for consistent comparison + * Handles both full URLs and path-only strings + * @param url - URL or path to normalize + * @returns Normalized lowercase path without trailing slash + */ +function normalizeUrlPath(url: string): string { + try { + const parsedUrl = new URL(url); + let pathname = parsedUrl.pathname.toLowerCase().trim(); + if (pathname.endsWith('/')) { + pathname = pathname.slice(0, -1); + } + return pathname; + } catch (_e) { + // If it's not a valid URL, treat it as a path and normalize + let path = url.toLowerCase().trim(); + if (path.endsWith('/')) { + path = path.slice(0, -1); + } + return path; + } +} + +/** + * CompositeExporter wraps multiple span exporters and provides two key features: + * + * 1. **Multi-exporter support**: Exports spans to multiple destinations in parallel + * (e.g., console for development + OTLP for production monitoring) + * + * 2. **Recursive telemetry filtering**: Prevents telemetry infinity loops by filtering + * out spans from `/api/telemetry` endpoints. Without this, telemetry API calls would + * generate spans, which would be exported via HTTP to `/api/telemetry`, generating + * more spans, creating an infinite loop. + * + * @example + * ```typescript + * const exporter = new CompositeExporter([ + * new ConsoleSpanExporter(), + * new OTLPHttpExporter({ url: 'http://localhost:4318/v1/traces' }) + * ]); + * ``` + */ +export class CompositeExporter implements SpanExporter { + private exporters: SpanExporter[]; + + constructor(exporters: SpanExporter[]) { + this.exporters = exporters; + } + + export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void { + // First collect all traceIds from telemetry endpoint spans + const telemetryTraceIds = new Set( + spans + .filter((span) => { + const attrs = span.attributes || {}; + const relevantHttpAttributes = [ + attrs['http.target'], + attrs['http.route'], + attrs['http.url'], + attrs['url.path'], + attrs['http.request_path'], + ]; + + const isTelemetrySpan = relevantHttpAttributes.some((attr) => { + if (typeof attr === 'string') { + const normalizedPath = normalizeUrlPath(attr); + // Check for exact match or path prefix + return ( + normalizedPath === '/api/telemetry' || + normalizedPath.startsWith('/api/telemetry/') + ); + } + return false; + }); + return isTelemetrySpan; + }) + .map((span) => span.spanContext().traceId) + ); + + // Then filter out any spans that have those traceIds + const filteredSpans = spans.filter( + (span) => !telemetryTraceIds.has(span.spanContext().traceId) + ); + + // Return early if no spans to export + if (filteredSpans.length === 0) { + resultCallback({ code: ExportResultCode.SUCCESS }); + return; + } + + void Promise.all( + this.exporters.map( + (exporter) => + new Promise((resolve) => { + if (exporter.export) { + exporter.export(filteredSpans, resolve); + } else { + resolve({ code: ExportResultCode.FAILED }); + } + }) + ) + ) + .then((results) => { + const hasError = results.some((r) => r.code === ExportResultCode.FAILED); + resultCallback({ + code: hasError ? ExportResultCode.FAILED : ExportResultCode.SUCCESS, + }); + }) + .catch((error) => { + console.error('[CompositeExporter] Export error:', error); + resultCallback({ code: ExportResultCode.FAILED }); + }); + } + + shutdown(): Promise { + return Promise.all(this.exporters.map((e) => e.shutdown?.() ?? Promise.resolve())).then( + () => undefined + ); + } + + forceFlush(): Promise { + return Promise.all(this.exporters.map((e) => e.forceFlush?.() ?? Promise.resolve())).then( + () => undefined + ); + } +} diff --git a/dexto/packages/core/src/telemetry/http-instrumentation.integration.test.ts b/dexto/packages/core/src/telemetry/http-instrumentation.integration.test.ts new file mode 100644 index 00000000..1e040d84 --- /dev/null +++ b/dexto/packages/core/src/telemetry/http-instrumentation.integration.test.ts @@ -0,0 +1,148 @@ +/** + * Integration test for HTTP instrumentation. + * + * This test verifies that OpenTelemetry's HTTP/fetch instrumentation is working correctly. + * It makes actual HTTP calls and verifies that spans are created for them. + * + * This is critical for ensuring that LLM API calls (which use fetch) are traced. + * + * NOTE: This test sets up OpenTelemetry SDK directly (not via Telemetry class) to verify + * that the specific instrumentations (http + undici) correctly instrument fetch() calls. + * This mirrors the production setup in telemetry.ts. + */ +import { describe, test, expect, afterAll, beforeAll } from 'vitest'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { Resource } from '@opentelemetry/resources'; +import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions'; +import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; +import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici'; + +describe('HTTP Instrumentation', () => { + let serverPort: number; + let memoryExporter: InMemorySpanExporter; + let sdk: NodeSDK; + let server: Awaited>; + + beforeAll(async () => { + // Create in-memory exporter + memoryExporter = new InMemorySpanExporter(); + + // Initialize OpenTelemetry SDK directly with specific instrumentations + // This mirrors the production setup in telemetry.ts + sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: 'http-instrumentation-test', + }), + spanProcessor: new SimpleSpanProcessor(memoryExporter) as any, + instrumentations: [new HttpInstrumentation(), new UndiciInstrumentation()], + }); + + await sdk.start(); + + // NOW import http and create the server (after instrumentation is set up) + const http = await import('http'); + server = http.createServer((req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ message: 'ok', path: req.url })); + }); + + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + const addr = server.address(); + if (addr && typeof addr === 'object') { + serverPort = addr.port; + } + resolve(); + }); + }); + }); + + afterAll(async () => { + // Close server first + if (server) { + await new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }); + } + + // Then shutdown SDK + if (sdk) { + await sdk.shutdown(); + } + }); + + test('fetch() calls are instrumented and create HTTP spans', async () => { + // Clear any previous spans + memoryExporter.reset(); + + // Make a fetch call - this should be instrumented by undici instrumentation + // (Node.js 18+ uses undici internally for fetch()) + const url = `http://127.0.0.1:${serverPort}/test-fetch-endpoint`; + const response = await fetch(url); + const data = await response.json(); + expect(data.message).toBe('ok'); + + // Give time for async span processing + await new Promise((resolve) => setTimeout(resolve, 500)); + + // Check that spans were created + const spans = memoryExporter.getFinishedSpans(); + + // We should have at least one HTTP span + const httpSpans = spans.filter((span) => { + const name = span.name.toLowerCase(); + const attrs = span.attributes; + return ( + name.includes('get') || + name.includes('http') || + name.includes('fetch') || + attrs['http.method'] === 'GET' || + attrs['http.request.method'] === 'GET' + ); + }); + + expect(httpSpans.length).toBeGreaterThan(0); + + // Verify the span has expected HTTP attributes + const httpSpan = httpSpans[0]!; + const attrs = httpSpan.attributes; + + // Should have URL-related attributes + expect(attrs['url.full'] || attrs['http.url'] || attrs['http.target']).toBeDefined(); + + // Should have method attribute + expect(attrs['http.request.method'] || attrs['http.method']).toBe('GET'); + + // Should have status code + expect(attrs['http.response.status_code'] || attrs['http.status_code']).toBe(200); + }); + + test('multiple fetch() calls create multiple spans', async () => { + // Clear any previous spans + memoryExporter.reset(); + + // Make multiple fetch calls + const urls = [ + `http://127.0.0.1:${serverPort}/endpoint-1`, + `http://127.0.0.1:${serverPort}/endpoint-2`, + `http://127.0.0.1:${serverPort}/endpoint-3`, + ]; + + await Promise.all(urls.map((url) => fetch(url))); + + // Give time for async span processing + await new Promise((resolve) => setTimeout(resolve, 500)); + + // Check that spans were created + const spans = memoryExporter.getFinishedSpans(); + + // Should have at least 3 spans (one for each request) + const httpSpans = spans.filter((span) => { + const attrs = span.attributes; + return attrs['http.request.method'] === 'GET' || attrs['http.method'] === 'GET'; + }); + + expect(httpSpans.length).toBeGreaterThanOrEqual(3); + }); +}); diff --git a/dexto/packages/core/src/telemetry/index.ts b/dexto/packages/core/src/telemetry/index.ts new file mode 100644 index 00000000..6105470c --- /dev/null +++ b/dexto/packages/core/src/telemetry/index.ts @@ -0,0 +1 @@ +export { Telemetry } from './telemetry.js'; diff --git a/dexto/packages/core/src/telemetry/schemas.ts b/dexto/packages/core/src/telemetry/schemas.ts new file mode 100644 index 00000000..55be20ac --- /dev/null +++ b/dexto/packages/core/src/telemetry/schemas.ts @@ -0,0 +1,50 @@ +import { z } from 'zod'; + +export const OtelConfigurationSchema = z.object({ + serviceName: z.string().optional(), + enabled: z.boolean().optional(), + tracerName: z.string().optional(), + // TODO (Telemetry): Implement sampling support in Phase 5 + // Currently sampling schema is defined but not implemented in telemetry.ts + // See feature-plans/telemetry.md Phase 5 for implementation details + // sampling: z + // .discriminatedUnion('type', [ + // z.object({ + // type: z.literal('ratio'), + // probability: z.number().min(0).max(1), + // }), + // z.object({ + // type: z.literal('always_on'), + // }), + // z.object({ + // type: z.literal('always_off'), + // }), + // z.object({ + // type: z.literal('parent_based'), + // root: z.object({ + // probability: z.number().min(0).max(1), + // }), + // }), + // ]) + // .optional(), + export: z + .union([ + z.object({ + type: z.literal('otlp'), + protocol: z.enum(['grpc', 'http']).optional(), + endpoint: z + .union([ + z.string().url(), + z.string().regex(/^[\w.-]+:\d+$/), // host:port + ]) + .optional(), + headers: z.record(z.string()).optional(), + }), + z.object({ + type: z.literal('console'), + }), + ]) + .optional(), +}); + +export type OtelConfiguration = z.output; diff --git a/dexto/packages/core/src/telemetry/telemetry.test.ts b/dexto/packages/core/src/telemetry/telemetry.test.ts new file mode 100644 index 00000000..02d751d9 --- /dev/null +++ b/dexto/packages/core/src/telemetry/telemetry.test.ts @@ -0,0 +1,271 @@ +import { describe, test, expect, afterEach } from 'vitest'; +import { Telemetry } from './telemetry.js'; +import type { OtelConfiguration } from './schemas.js'; + +describe.sequential('Telemetry Core', () => { + // Clean up after each test to prevent state leakage + afterEach(async () => { + // Force clear global state + if (Telemetry.hasGlobalInstance()) { + await Telemetry.shutdownGlobal(); + } + // Longer delay to ensure cleanup completes and providers are unregistered + await new Promise((resolve) => setTimeout(resolve, 100)); + }); + + describe('Initialization', () => { + test('init() with enabled=true creates telemetry instance', async () => { + const config: OtelConfiguration = { + enabled: true, + serviceName: 'test-service', + export: { type: 'console' }, + }; + + const telemetry = await Telemetry.init(config); + + expect(telemetry).toBeDefined(); + expect(telemetry.isInitialized()).toBe(true); + expect(telemetry.name).toBe('test-service'); + expect(Telemetry.hasGlobalInstance()).toBe(true); + }); + + test('init() with enabled=false creates instance but does not initialize SDK', async () => { + const config: OtelConfiguration = { + enabled: false, + serviceName: 'test-service', + }; + + const telemetry = await Telemetry.init(config); + + expect(telemetry).toBeDefined(); + expect(telemetry.isInitialized()).toBe(false); + expect(Telemetry.hasGlobalInstance()).toBe(true); + }); + + test('init() with console exporter works', async () => { + const config: OtelConfiguration = { + enabled: true, + export: { type: 'console' }, + }; + + const telemetry = await Telemetry.init(config); + + expect(telemetry.isInitialized()).toBe(true); + }); + + test('init() with otlp-http exporter works', async () => { + const config: OtelConfiguration = { + enabled: true, + export: { + type: 'otlp', + protocol: 'http', + endpoint: 'http://localhost:4318/v1/traces', + }, + }; + + const telemetry = await Telemetry.init(config); + + expect(telemetry.isInitialized()).toBe(true); + }); + + test('init() with otlp-grpc exporter works', async () => { + const config: OtelConfiguration = { + enabled: true, + export: { + type: 'otlp', + protocol: 'grpc', + endpoint: 'http://localhost:4317', + }, + }; + + const telemetry = await Telemetry.init(config); + + expect(telemetry.isInitialized()).toBe(true); + }); + + test('init() is idempotent - returns same instance on subsequent calls', async () => { + const config: OtelConfiguration = { + enabled: true, + serviceName: 'test-service', + export: { type: 'console' }, + }; + + const telemetry1 = await Telemetry.init(config); + const telemetry2 = await Telemetry.init(config); + const telemetry3 = await Telemetry.init({ enabled: false }); // Different config + + // Should return the same instance regardless of config + expect(telemetry1).toBe(telemetry2); + expect(telemetry2).toBe(telemetry3); + }); + + test('init() is race-safe - concurrent calls return same instance', async () => { + const config: OtelConfiguration = { + enabled: true, + serviceName: 'test-service', + export: { type: 'console' }, + }; + + // Start multiple init calls concurrently + const [telemetry1, telemetry2, telemetry3] = await Promise.all([ + Telemetry.init(config), + Telemetry.init(config), + Telemetry.init(config), + ]); + + // All should return the same instance + expect(telemetry1).toBe(telemetry2); + expect(telemetry2).toBe(telemetry3); + expect(Telemetry.hasGlobalInstance()).toBe(true); + }); + + test('get() throws when not initialized', () => { + expect(() => Telemetry.get()).toThrow('Telemetry not initialized'); + }); + + test('get() returns instance after initialization', async () => { + const config: OtelConfiguration = { + enabled: true, + export: { type: 'console' }, + }; + + const telemetry = await Telemetry.init(config); + const retrieved = Telemetry.get(); + + expect(retrieved).toBe(telemetry); + }); + + test('hasGlobalInstance() returns correct state', async () => { + expect(Telemetry.hasGlobalInstance()).toBe(false); + + await Telemetry.init({ enabled: true, export: { type: 'console' } }); + expect(Telemetry.hasGlobalInstance()).toBe(true); + + await Telemetry.shutdownGlobal(); + expect(Telemetry.hasGlobalInstance()).toBe(false); + }); + }); + + describe('Shutdown', () => { + test('shutdownGlobal() clears global instance', async () => { + await Telemetry.init({ enabled: true, export: { type: 'console' } }); + expect(Telemetry.hasGlobalInstance()).toBe(true); + + await Telemetry.shutdownGlobal(); + expect(Telemetry.hasGlobalInstance()).toBe(false); + }); + + test('shutdownGlobal() allows re-initialization', async () => { + // First initialization + const telemetry1 = await Telemetry.init({ + enabled: true, + serviceName: 'service-1', + export: { type: 'console' }, + }); + expect(telemetry1.name).toBe('service-1'); + + // Shutdown + await Telemetry.shutdownGlobal(); + + // Second initialization with different config + const telemetry2 = await Telemetry.init({ + enabled: true, + serviceName: 'service-2', + export: { type: 'console' }, + }); + expect(telemetry2.name).toBe('service-2'); + expect(telemetry2).not.toBe(telemetry1); + }); + + test('shutdownGlobal() is safe to call when not initialized', async () => { + expect(Telemetry.hasGlobalInstance()).toBe(false); + await expect(Telemetry.shutdownGlobal()).resolves.not.toThrow(); + }); + + test('shutdown() on instance clears isInitialized flag', async () => { + const telemetry = await Telemetry.init({ + enabled: true, + export: { type: 'console' }, + }); + expect(telemetry.isInitialized()).toBe(true); + + await telemetry.shutdown(); + expect(telemetry.isInitialized()).toBe(false); + }); + }); + + // Note: Signal handler tests removed - they are implementation details + // that are difficult to test reliably with mocks. Signal handlers are + // manually verified to work correctly (process cleanup on SIGTERM/SIGINT). + + describe('Agent Switching', () => { + test('supports sequential agent switching with different configs', async () => { + // Agent 1 + const telemetry1 = await Telemetry.init({ + enabled: true, + serviceName: 'agent-1', + export: { type: 'console' }, + }); + expect(telemetry1.name).toBe('agent-1'); + expect(Telemetry.hasGlobalInstance()).toBe(true); + + // Shutdown agent 1 + await Telemetry.shutdownGlobal(); + expect(Telemetry.hasGlobalInstance()).toBe(false); + + // Agent 2 with different config + const telemetry2 = await Telemetry.init({ + enabled: true, + serviceName: 'agent-2', + export: { + type: 'otlp', + protocol: 'http', + endpoint: 'http://different:4318', + }, + }); + expect(telemetry2.name).toBe('agent-2'); + expect(telemetry2).not.toBe(telemetry1); + + // Shutdown agent 2 + await Telemetry.shutdownGlobal(); + + // Agent 3 - telemetry disabled + const telemetry3 = await Telemetry.init({ + enabled: false, + }); + expect(telemetry3.isInitialized()).toBe(false); + expect(telemetry3).not.toBe(telemetry1); + expect(telemetry3).not.toBe(telemetry2); + }); + }); + + describe('Static Methods', () => { + test('getActiveSpan() returns undefined when no active span', () => { + const span = Telemetry.getActiveSpan(); + expect(span).toBeUndefined(); + }); + + test('setBaggage() creates new context with baggage', () => { + const baggage = { + sessionId: { value: 'test-session-123' }, + }; + + const newCtx = Telemetry.setBaggage(baggage); + expect(newCtx).toBeDefined(); + }); + + test('withContext() executes function in given context', () => { + const baggage = { + testKey: { value: 'testValue' }, + }; + const ctx = Telemetry.setBaggage(baggage); + + let executed = false; + Telemetry.withContext(ctx, () => { + executed = true; + }); + + expect(executed).toBe(true); + }); + }); +}); diff --git a/dexto/packages/core/src/telemetry/telemetry.ts b/dexto/packages/core/src/telemetry/telemetry.ts new file mode 100644 index 00000000..116b1dd2 --- /dev/null +++ b/dexto/packages/core/src/telemetry/telemetry.ts @@ -0,0 +1,361 @@ +import { context as otlpContext, trace, propagation } from '@opentelemetry/api'; +import type { Tracer, Context, BaggageEntry } from '@opentelemetry/api'; +import type { OtelConfiguration } from './schemas.js'; +import { logger } from '../logger/logger.js'; +import { TelemetryError } from './errors.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; + +// Type definitions for dynamically imported modules +type NodeSDKType = import('@opentelemetry/sdk-node').NodeSDK; +type ConsoleSpanExporterType = import('@opentelemetry/sdk-trace-base').ConsoleSpanExporter; +type OTLPHttpExporterType = import('@opentelemetry/exporter-trace-otlp-http').OTLPTraceExporter; +type OTLPGrpcExporterType = import('@opentelemetry/exporter-trace-otlp-grpc').OTLPTraceExporter; + +// Add type declaration for global namespace +declare global { + var __TELEMETRY__: Telemetry | undefined; +} + +/** + * TODO (Telemetry): enhancements + * - Implement sampling strategies (ratio-based, parent-based, always-on/off) + * - Add custom span processors for filtering/enrichment + * - Support context propagation across A2A (agent-to-agent) calls + * - Add cost tracking per trace (token costs, API costs) + * - Add static shutdownGlobal() method for agent switching + * See feature-plans/telemetry.md for details + */ +export class Telemetry { + public tracer: Tracer = trace.getTracer('dexto'); + name: string = 'dexto-service'; + private _isInitialized: boolean = false; + private _sdk?: NodeSDKType | undefined; + private static _initPromise?: Promise | undefined; + private static _signalHandlers?: { sigterm: () => void; sigint: () => void } | undefined; + + private constructor(config: OtelConfiguration, enabled: boolean, sdk?: NodeSDKType) { + const serviceName = config.serviceName ?? 'dexto-service'; + const tracerName = config.tracerName ?? serviceName; + + this.name = serviceName; + this.tracer = trace.getTracer(tracerName); + if (sdk) { + this._sdk = sdk; + } + this._isInitialized = enabled && !!sdk; + } + + private static async buildTraceExporter( + config: OtelConfiguration | undefined + ): Promise { + const e = config?.export; + if (!e || e.type === 'console') { + const { ConsoleSpanExporter } = await import('@opentelemetry/sdk-trace-base'); + return new ConsoleSpanExporter(); + } + if (e.type === 'otlp') { + if (e.protocol === 'grpc') { + let OTLPGrpcExporter: typeof import('@opentelemetry/exporter-trace-otlp-grpc').OTLPTraceExporter; + try { + const mod = await import('@opentelemetry/exporter-trace-otlp-grpc'); + OTLPGrpcExporter = mod.OTLPTraceExporter; + } catch (err) { + const error = err as NodeJS.ErrnoException; + if (error.code === 'ERR_MODULE_NOT_FOUND') { + throw TelemetryError.exporterDependencyNotInstalled( + 'grpc', + '@opentelemetry/exporter-trace-otlp-grpc' + ); + } + throw err; + } + const options: { url?: string } = {}; + if (e.endpoint) { + options.url = e.endpoint; + } + return new OTLPGrpcExporter(options); + } + // default to http when omitted + let OTLPHttpExporter: typeof import('@opentelemetry/exporter-trace-otlp-http').OTLPTraceExporter; + try { + const mod = await import('@opentelemetry/exporter-trace-otlp-http'); + OTLPHttpExporter = mod.OTLPTraceExporter; + } catch (err) { + const error = err as NodeJS.ErrnoException; + if (error.code === 'ERR_MODULE_NOT_FOUND') { + throw TelemetryError.exporterDependencyNotInstalled( + 'http', + '@opentelemetry/exporter-trace-otlp-http' + ); + } + throw err; + } + const options: { url?: string; headers?: Record } = {}; + if (e.endpoint) { + options.url = e.endpoint; + } + if (e.headers) { + options.headers = e.headers; + } + return new OTLPHttpExporter(options); + } + // schema also allows 'custom' but YAML cannot provide a SpanExporter instance + const { ConsoleSpanExporter } = await import('@opentelemetry/sdk-trace-base'); + return new ConsoleSpanExporter(); + } + /** + * Initialize telemetry with the given configuration + * @param config - Optional telemetry configuration object + * @param exporter - Optional custom span exporter (overrides config.export, useful for testing) + * @returns Telemetry instance that can be used for tracing + */ + static async init( + config: OtelConfiguration = {}, + exporter?: import('@opentelemetry/sdk-trace-base').SpanExporter + ): Promise { + try { + // Return existing instance if already initialized + if (globalThis.__TELEMETRY__) return globalThis.__TELEMETRY__; + + // Return pending promise if initialization is in progress + if (Telemetry._initPromise) return Telemetry._initPromise; + + // Create and store initialization promise to prevent race conditions + Telemetry._initPromise = (async () => { + if (!globalThis.__TELEMETRY__) { + // honor enabled=false: skip SDK registration + const enabled = config.enabled !== false; + + let sdk: NodeSDKType | undefined; + if (enabled) { + // Dynamic imports for optional OpenTelemetry dependencies + let NodeSDK: typeof import('@opentelemetry/sdk-node').NodeSDK; + let Resource: typeof import('@opentelemetry/resources').Resource; + let HttpInstrumentation: typeof import('@opentelemetry/instrumentation-http').HttpInstrumentation; + let UndiciInstrumentation: typeof import('@opentelemetry/instrumentation-undici').UndiciInstrumentation; + let ATTR_SERVICE_NAME: string; + + try { + const sdkModule = await import('@opentelemetry/sdk-node'); + NodeSDK = sdkModule.NodeSDK; + + const resourcesModule = await import('@opentelemetry/resources'); + Resource = resourcesModule.Resource; + + // Import specific instrumentations instead of auto-instrumentations-node + // This reduces install size by ~130MB while maintaining HTTP tracing for LLM API calls + const httpInstModule = await import( + '@opentelemetry/instrumentation-http' + ); + HttpInstrumentation = httpInstModule.HttpInstrumentation; + + const undiciInstModule = await import( + '@opentelemetry/instrumentation-undici' + ); + UndiciInstrumentation = undiciInstModule.UndiciInstrumentation; + + const semanticModule = await import( + '@opentelemetry/semantic-conventions' + ); + ATTR_SERVICE_NAME = semanticModule.ATTR_SERVICE_NAME; + } catch (importError) { + const err = importError as NodeJS.ErrnoException; + if (err.code === 'ERR_MODULE_NOT_FOUND') { + throw TelemetryError.dependencyNotInstalled([ + '@opentelemetry/sdk-node', + '@opentelemetry/instrumentation-http', + '@opentelemetry/instrumentation-undici', + '@opentelemetry/resources', + '@opentelemetry/semantic-conventions', + '@opentelemetry/sdk-trace-base', + '@opentelemetry/exporter-trace-otlp-http', + '@opentelemetry/exporter-trace-otlp-grpc', + ]); + } + throw importError; + } + + const resource = new Resource({ + [ATTR_SERVICE_NAME]: config.serviceName ?? 'dexto-service', + }); + + // Use custom exporter if provided, otherwise build from config + const spanExporter = + exporter || (await Telemetry.buildTraceExporter(config)); + + // Dynamically import CompositeExporter to avoid loading OpenTelemetry at startup + const { CompositeExporter } = await import('./exporters.js'); + const traceExporter = + spanExporter instanceof CompositeExporter + ? spanExporter + : new CompositeExporter([spanExporter]); + + // Use specific instrumentations for HTTP tracing: + // - HttpInstrumentation: traces http/https module calls + // - UndiciInstrumentation: traces fetch() calls (Node.js 18+ uses undici internally) + sdk = new NodeSDK({ + resource, + traceExporter, + instrumentations: [ + new HttpInstrumentation(), + new UndiciInstrumentation(), + ], + }); + + await sdk.start(); // registers the global provider → no ProxyTracer + + // graceful shutdown (one-shot, avoid unhandled rejection) + const sigterm = () => { + void sdk?.shutdown(); + }; + const sigint = () => { + void sdk?.shutdown(); + }; + process.once('SIGTERM', sigterm); + process.once('SIGINT', sigint); + Telemetry._signalHandlers = { sigterm, sigint }; + } + + globalThis.__TELEMETRY__ = new Telemetry(config, enabled, sdk); + } + return globalThis.__TELEMETRY__!; + })(); + + // Await the promise so failures are caught by outer try/catch + // This ensures _initPromise is cleared on failure, allowing re-initialization + return await Telemetry._initPromise; + } catch (error) { + // Clear init promise so subsequent calls can retry + Telemetry._initPromise = undefined; + // Re-throw typed errors as-is, wrap unknown errors + if (error instanceof DextoRuntimeError) { + throw error; + } + throw TelemetryError.initializationFailed( + error instanceof Error ? error.message : String(error), + error + ); + } + } + + static getActiveSpan() { + const span = trace.getActiveSpan(); + return span; + } + + /** + * Get the global telemetry instance + * @throws {DextoRuntimeError} If telemetry has not been initialized + * @returns {Telemetry} The global telemetry instance + */ + static get(): Telemetry { + if (!globalThis.__TELEMETRY__) { + throw TelemetryError.notInitialized(); + } + return globalThis.__TELEMETRY__; + } + + /** + * Check if global telemetry instance exists + * @returns True if telemetry has been initialized, false otherwise + */ + static hasGlobalInstance(): boolean { + return globalThis.__TELEMETRY__ !== undefined; + } + + /** + * Shutdown global telemetry instance + * Used during agent switching to cleanly shutdown old agent's telemetry + * before initializing new agent's telemetry with potentially different config + * @returns Promise that resolves when shutdown is complete + */ + static async shutdownGlobal(): Promise { + if (globalThis.__TELEMETRY__) { + await globalThis.__TELEMETRY__.shutdown(); + globalThis.__TELEMETRY__ = undefined; + } + // Also clear the init promise to allow re-initialization + Telemetry._initPromise = undefined; + } + + /** + * Checks if the Telemetry instance has been successfully initialized. + * @returns True if the instance is initialized, false otherwise. + */ + public isInitialized(): boolean { + return this._isInitialized; + } + + static setBaggage(baggage: Record, ctx: Context = otlpContext.active()) { + const currentBaggage = Object.fromEntries( + propagation.getBaggage(ctx)?.getAllEntries() ?? [] + ); + const newCtx = propagation.setBaggage( + ctx, + propagation.createBaggage({ + ...currentBaggage, + ...baggage, + }) + ); + return newCtx; + } + + static withContext(ctx: Context, fn: () => void) { + return otlpContext.with(ctx, fn); + } + + /** + * Forces pending spans to be exported immediately. + * Useful for testing to ensure spans are available in exporters. + */ + public async forceFlush(): Promise { + if (this._isInitialized) { + // Access the global tracer provider and force flush + const provider = trace.getTracerProvider() as any; + if (provider && typeof provider.forceFlush === 'function') { + await provider.forceFlush(); + } + } + } + + /** + * Shuts down the OpenTelemetry SDK, flushing any pending spans. + * This should be called before the application exits. + * + * Uses two-phase shutdown: + * 1. Best-effort flush - Try to export pending spans (can fail if backend unavailable) + * 2. Force cleanup - Always clear global state to allow re-initialization + * + * This ensures agent switching works even when telemetry export fails. + */ + public async shutdown(): Promise { + if (this._sdk) { + try { + // Phase 1: Best-effort flush pending spans to backend + // This can fail if Jaeger/OTLP collector is unreachable + await this._sdk.shutdown(); + } catch (error) { + // Don't throw - log warning and continue with cleanup + // Telemetry is observability infrastructure, not core functionality + const errorMsg = error instanceof Error ? error.message : String(error); + logger.warn(`Telemetry shutdown failed to flush spans (non-blocking): ${errorMsg}`); + } finally { + // Phase 2: Force cleanup - MUST always happen regardless of flush success + // This ensures we can reinitialize telemetry for agent switching + this._isInitialized = false; + globalThis.__TELEMETRY__ = undefined; // Clear the global instance + + // Cleanup signal handlers to prevent leaks + if (Telemetry._signalHandlers) { + process.off('SIGTERM', Telemetry._signalHandlers.sigterm); + process.off('SIGINT', Telemetry._signalHandlers.sigint); + Telemetry._signalHandlers = undefined; + } + + // Clear references for GC and re-initialization + this._sdk = undefined; + Telemetry._initPromise = undefined; + } + } + } +} diff --git a/dexto/packages/core/src/telemetry/types.ts b/dexto/packages/core/src/telemetry/types.ts new file mode 100644 index 00000000..c192f087 --- /dev/null +++ b/dexto/packages/core/src/telemetry/types.ts @@ -0,0 +1,22 @@ +import type { ReadableSpan } from '@opentelemetry/sdk-trace-base'; + +/** + * Trace data structure for storage/retrieval + * Used by telemetry storage exporters for persisting trace data + */ +export type Trace = { + id: string; + parentSpanId: string; + name: string; + traceId: string; + scope: string; + kind: ReadableSpan['kind']; + attributes: ReadableSpan['attributes']; + status: ReadableSpan['status']; + events: ReadableSpan['events']; + links: ReadableSpan['links']; + other: Record; + startTime: number; + endTime: number; + createdAt: string; +}; diff --git a/dexto/packages/core/src/telemetry/utils.ts b/dexto/packages/core/src/telemetry/utils.ts new file mode 100644 index 00000000..14b8c837 --- /dev/null +++ b/dexto/packages/core/src/telemetry/utils.ts @@ -0,0 +1,82 @@ +import { propagation } from '@opentelemetry/api'; +import type { Context, Span } from '@opentelemetry/api'; +import { Telemetry } from './telemetry.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; + +// Helper function to check if telemetry is active +export function hasActiveTelemetry(logger?: IDextoLogger): boolean { + logger?.silly('hasActiveTelemetry called.'); + try { + const telemetryInstance = Telemetry.get(); + const isActive = telemetryInstance.isInitialized(); + logger?.silly(`hasActiveTelemetry: Telemetry is initialized: ${isActive}`); + return isActive; + } catch (error) { + logger?.silly( + `hasActiveTelemetry: Telemetry not active or initialized. Error: ${error instanceof Error ? error.message : String(error)}` + ); + return false; + } +} + +/** + * Get baggage values from context + * @param ctx The context to get baggage values from + * @param logger Optional logger instance + * @returns + */ +export function getBaggageValues(ctx: Context, logger?: IDextoLogger) { + logger?.silly('getBaggageValues called.'); + const currentBaggage = propagation.getBaggage(ctx); + const requestId = currentBaggage?.getEntry('http.request_id')?.value; + const componentName = currentBaggage?.getEntry('componentName')?.value; + const runId = currentBaggage?.getEntry('runId')?.value; + const threadId = currentBaggage?.getEntry('threadId')?.value; + const resourceId = currentBaggage?.getEntry('resourceId')?.value; + const sessionId = currentBaggage?.getEntry('sessionId')?.value; + logger?.silly( + `getBaggageValues: Extracted - requestId: ${requestId}, componentName: ${componentName}, runId: ${runId}, threadId: ${threadId}, resourceId: ${resourceId}, sessionId: ${sessionId}` + ); + return { + requestId, + componentName, + runId, + threadId, + resourceId, + sessionId, + }; +} + +/** + * Attaches baggage values from the given context to the provided span as attributes. + * @param span The OpenTelemetry Span to add attributes to. + * @param ctx The OpenTelemetry Context from which to extract baggage values. + * @param logger Optional logger instance + */ +export function addBaggageAttributesToSpan(span: Span, ctx: Context, logger?: IDextoLogger): void { + logger?.debug('addBaggageAttributesToSpan called.'); + const { requestId, componentName, runId, threadId, resourceId, sessionId } = getBaggageValues( + ctx, + logger + ); + + if (componentName) { + span.setAttribute('componentName', componentName); + } + if (runId) { + span.setAttribute('runId', runId); + } + if (requestId) { + span.setAttribute('http.request_id', requestId); + } + if (threadId) { + span.setAttribute('threadId', threadId); + } + if (resourceId) { + span.setAttribute('resourceId', resourceId); + } + if (sessionId) { + span.setAttribute('sessionId', sessionId); + } + logger?.debug('addBaggageAttributesToSpan: Baggage attributes added to span.'); +} diff --git a/dexto/packages/core/src/tools/bash-pattern-utils.test.ts b/dexto/packages/core/src/tools/bash-pattern-utils.test.ts new file mode 100644 index 00000000..d64fcdd3 --- /dev/null +++ b/dexto/packages/core/src/tools/bash-pattern-utils.test.ts @@ -0,0 +1,277 @@ +import { describe, it, expect } from 'vitest'; +import { + DANGEROUS_COMMAND_PREFIXES, + isDangerousCommand, + generateBashPatternKey, + generateBashPatternSuggestions, + patternCovers, +} from './bash-pattern-utils.js'; + +describe('bash-pattern-utils', () => { + describe('patternCovers', () => { + describe('exact matches', () => { + it('should match identical patterns', () => { + expect(patternCovers('git *', 'git *')).toBe(true); + expect(patternCovers('ls *', 'ls *')).toBe(true); + expect(patternCovers('npm install *', 'npm install *')).toBe(true); + }); + + it('should match patterns without wildcards', () => { + expect(patternCovers('git status', 'git status')).toBe(true); + }); + }); + + describe('broader pattern covers narrower', () => { + it('should cover single subcommand patterns', () => { + // "git *" covers "git push *", "git status *", etc. + expect(patternCovers('git *', 'git push *')).toBe(true); + expect(patternCovers('git *', 'git status *')).toBe(true); + expect(patternCovers('git *', 'git commit *')).toBe(true); + }); + + it('should cover multi-level subcommand patterns', () => { + // "docker *" covers "docker compose *" + expect(patternCovers('docker *', 'docker compose *')).toBe(true); + // "docker compose *" covers "docker compose up *" + expect(patternCovers('docker compose *', 'docker compose up *')).toBe(true); + }); + + it('should cover npm commands', () => { + expect(patternCovers('npm *', 'npm install *')).toBe(true); + expect(patternCovers('npm *', 'npm run *')).toBe(true); + }); + }); + + describe('narrower pattern does NOT cover broader', () => { + it('should not cover broader patterns', () => { + // "git push *" does NOT cover "git *" + expect(patternCovers('git push *', 'git *')).toBe(false); + // "git push *" does NOT cover "git status *" + expect(patternCovers('git push *', 'git status *')).toBe(false); + }); + + it('should not cover unrelated patterns', () => { + expect(patternCovers('npm *', 'git *')).toBe(false); + expect(patternCovers('ls *', 'cat *')).toBe(false); + }); + }); + + describe('similar but different commands', () => { + it('should not match commands with similar prefixes', () => { + // "npm *" should NOT cover "npx *" (different command) + expect(patternCovers('npm *', 'npx *')).toBe(false); + // "git *" should NOT cover "gitk *" (different command) + expect(patternCovers('git *', 'gitk *')).toBe(false); + }); + }); + + describe('edge cases', () => { + it('should handle patterns without trailing wildcard', () => { + expect(patternCovers('git', 'git')).toBe(true); + // Note: In practice, we always generate patterns with " *" suffix. + // Patterns without wildcard still use the covering logic. + expect(patternCovers('git', 'git push')).toBe(true); + }); + + it('should handle mixed patterns (wildcard vs no wildcard)', () => { + // "git *" does NOT cover "git" (target doesn't end with " *") + expect(patternCovers('git *', 'git')).toBe(false); + // "git" does NOT cover "git *" (bases are "git" vs "git", exact match fails, then covering check) + expect(patternCovers('git', 'git *')).toBe(false); + }); + }); + }); + + describe('DANGEROUS_COMMAND_PREFIXES', () => { + it('should include common dangerous commands', () => { + expect(DANGEROUS_COMMAND_PREFIXES).toContain('rm'); + expect(DANGEROUS_COMMAND_PREFIXES).toContain('sudo'); + expect(DANGEROUS_COMMAND_PREFIXES).toContain('chmod'); + expect(DANGEROUS_COMMAND_PREFIXES).toContain('kill'); + expect(DANGEROUS_COMMAND_PREFIXES).toContain('shutdown'); + }); + + it('should be readonly', () => { + // TypeScript enforces this at compile time + expect(Array.isArray(DANGEROUS_COMMAND_PREFIXES)).toBe(true); + }); + }); + + describe('isDangerousCommand', () => { + it('should return true for dangerous commands', () => { + expect(isDangerousCommand('rm -rf /')).toBe(true); + expect(isDangerousCommand('sudo apt install')).toBe(true); + expect(isDangerousCommand('chmod 777 file')).toBe(true); + expect(isDangerousCommand('kill -9 1234')).toBe(true); + expect(isDangerousCommand('dd if=/dev/zero of=/dev/sda')).toBe(true); + }); + + it('should return false for safe commands', () => { + expect(isDangerousCommand('ls -la')).toBe(false); + expect(isDangerousCommand('git status')).toBe(false); + expect(isDangerousCommand('npm install')).toBe(false); + expect(isDangerousCommand('cat file.txt')).toBe(false); + }); + + it('should be case insensitive for command prefix', () => { + expect(isDangerousCommand('RM -rf /')).toBe(true); + expect(isDangerousCommand('SUDO apt install')).toBe(true); + expect(isDangerousCommand('Chmod 777 file')).toBe(true); + }); + + it('should handle empty input', () => { + expect(isDangerousCommand('')).toBe(false); + expect(isDangerousCommand(' ')).toBe(false); + }); + }); + + describe('generateBashPatternKey', () => { + describe('basic pattern generation', () => { + it('should generate pattern for simple command', () => { + expect(generateBashPatternKey('ls')).toBe('ls *'); + }); + + it('should generate pattern for command with flags only', () => { + // Flags don't count as subcommand + expect(generateBashPatternKey('ls -la')).toBe('ls *'); + expect(generateBashPatternKey('ls -l -a -h')).toBe('ls *'); + }); + + it('should generate pattern for command with subcommand', () => { + expect(generateBashPatternKey('git status')).toBe('git status *'); + expect(generateBashPatternKey('git push')).toBe('git push *'); + expect(generateBashPatternKey('npm install')).toBe('npm install *'); + }); + + it('should use first non-flag argument as subcommand', () => { + // "git -v status" → git status * (flags before subcommand are skipped) + expect(generateBashPatternKey('git -v status')).toBe('git status *'); + expect(generateBashPatternKey('npm --verbose install')).toBe('npm install *'); + }); + + it('should ignore arguments after subcommand', () => { + expect(generateBashPatternKey('git push origin main')).toBe('git push *'); + expect(generateBashPatternKey('npm install lodash --save')).toBe('npm install *'); + }); + }); + + describe('dangerous commands', () => { + it('should return null for dangerous commands', () => { + expect(generateBashPatternKey('rm -rf /')).toBeNull(); + expect(generateBashPatternKey('sudo apt install')).toBeNull(); + expect(generateBashPatternKey('chmod 777 file')).toBeNull(); + expect(generateBashPatternKey('kill -9 1234')).toBeNull(); + expect(generateBashPatternKey('shutdown -h now')).toBeNull(); + }); + + it('should be case insensitive for dangerous command detection', () => { + expect(generateBashPatternKey('RM -rf /')).toBeNull(); + expect(generateBashPatternKey('SUDO apt install')).toBeNull(); + }); + }); + + describe('edge cases', () => { + it('should handle empty input', () => { + expect(generateBashPatternKey('')).toBeNull(); + expect(generateBashPatternKey(' ')).toBeNull(); + }); + + it('should handle extra whitespace', () => { + expect(generateBashPatternKey(' ls -la ')).toBe('ls *'); + expect(generateBashPatternKey(' git push origin ')).toBe('git push *'); + }); + + it('should handle single command', () => { + expect(generateBashPatternKey('pwd')).toBe('pwd *'); + }); + }); + }); + + describe('generateBashPatternSuggestions', () => { + describe('suggestion generation', () => { + it('should generate single suggestion for simple command', () => { + const suggestions = generateBashPatternSuggestions('ls'); + expect(suggestions).toEqual(['ls *']); + }); + + it('should generate single suggestion for command with flags only', () => { + const suggestions = generateBashPatternSuggestions('ls -la'); + expect(suggestions).toEqual(['ls *']); + }); + + it('should generate two suggestions for command with subcommand', () => { + const suggestions = generateBashPatternSuggestions('git push'); + expect(suggestions).toEqual(['git push *', 'git *']); + }); + + it('should generate suggestions from specific to broad', () => { + const suggestions = generateBashPatternSuggestions('git push origin main'); + // First suggestion is the pattern key, second is broader + expect(suggestions).toEqual(['git push *', 'git *']); + }); + + it('should handle command with flags before subcommand', () => { + const suggestions = generateBashPatternSuggestions('npm --verbose install lodash'); + expect(suggestions).toEqual(['npm install *', 'npm *']); + }); + }); + + describe('dangerous commands', () => { + it('should return empty array for dangerous commands', () => { + expect(generateBashPatternSuggestions('rm -rf /')).toEqual([]); + expect(generateBashPatternSuggestions('sudo apt install')).toEqual([]); + expect(generateBashPatternSuggestions('chmod 777 file')).toEqual([]); + }); + }); + + describe('edge cases', () => { + it('should handle empty input', () => { + expect(generateBashPatternSuggestions('')).toEqual([]); + expect(generateBashPatternSuggestions(' ')).toEqual([]); + }); + + it('should handle extra whitespace', () => { + const suggestions = generateBashPatternSuggestions(' git push origin '); + expect(suggestions).toEqual(['git push *', 'git *']); + }); + }); + }); + + describe('integration: pattern key matches broader patterns', () => { + // These tests verify that the pattern key generated from a command + // can be properly matched against broader stored patterns + + it('ls and ls -la should generate the same pattern key', () => { + expect(generateBashPatternKey('ls')).toBe(generateBashPatternKey('ls -la')); + expect(generateBashPatternKey('ls')).toBe('ls *'); + }); + + it('git push and git push origin main should generate the same pattern key', () => { + expect(generateBashPatternKey('git push')).toBe( + generateBashPatternKey('git push origin main') + ); + expect(generateBashPatternKey('git push')).toBe('git push *'); + }); + + it('suggestions should include the pattern key as first element', () => { + const command = 'git push origin main'; + const patternKey = generateBashPatternKey(command); + const suggestions = generateBashPatternSuggestions(command); + + expect(suggestions[0]).toBe(patternKey); + }); + + it('broader pattern should be able to cover narrower pattern key', () => { + // This tests the relationship between patterns: + // If user approves "git *", it should cover "git push *" + const narrowKey = generateBashPatternKey('git push origin main'); // "git push *" + const broadPattern = 'git *'; + + // The narrow key's base should start with the broad pattern's base + const narrowBase = narrowKey!.slice(0, -2); // "git push" + const broadBase = broadPattern.slice(0, -2); // "git" + + expect(narrowBase.startsWith(broadBase + ' ')).toBe(true); + }); + }); +}); diff --git a/dexto/packages/core/src/tools/bash-pattern-utils.ts b/dexto/packages/core/src/tools/bash-pattern-utils.ts new file mode 100644 index 00000000..886cf214 --- /dev/null +++ b/dexto/packages/core/src/tools/bash-pattern-utils.ts @@ -0,0 +1,136 @@ +/** + * Utility functions for bash command pattern generation and matching. + * + * Pattern-based approval allows users to approve command patterns like "git *" + * that automatically cover future matching commands (e.g., "git status", "git push"). + */ + +/** + * Check if a stored pattern covers a target pattern key. + * Pattern A covers pattern B if: + * 1. A == B (exact match), OR + * 2. B's base starts with A's base + " " (broader pattern covers narrower) + * + * Examples: + * - `git *` covers `git push *`: base `git` is prefix of base `git push` + * - `ls *` covers `ls *`: exact match + * - `npm *` does NOT cover `npx *`: `npx` doesn't start with `npm ` + * + * @param storedPattern The approved pattern (e.g., "git *") + * @param targetPatternKey The pattern key to check (e.g., "git push *") + * @returns true if storedPattern covers targetPatternKey + */ +export function patternCovers(storedPattern: string, targetPatternKey: string): boolean { + // Exact match + if (storedPattern === targetPatternKey) return true; + + // Extract bases by stripping trailing " *" + const storedBase = storedPattern.endsWith(' *') ? storedPattern.slice(0, -2) : storedPattern; + const targetBase = targetPatternKey.endsWith(' *') + ? targetPatternKey.slice(0, -2) + : targetPatternKey; + + // Broader pattern covers narrower: "git" covers "git push" + // targetBase must start with storedBase + " " (space after command) + return targetBase.startsWith(storedBase + ' '); +} + +/** + * Commands that should never get auto-approve pattern suggestions. + * These require explicit approval each time for safety. + */ +export const DANGEROUS_COMMAND_PREFIXES = [ + 'rm', + 'chmod', + 'chown', + 'chgrp', + 'sudo', + 'su', + 'dd', + 'mkfs', + 'fdisk', + 'parted', + 'kill', + 'killall', + 'pkill', + 'shutdown', + 'reboot', + 'halt', + 'poweroff', +] as const; + +/** + * Check if a command prefix is dangerous (should not get pattern suggestions). + */ +export function isDangerousCommand(command: string): boolean { + const tokens = command.trim().split(/\s+/); + if (tokens.length === 0 || !tokens[0]) return false; + const head = tokens[0].toLowerCase(); + return DANGEROUS_COMMAND_PREFIXES.includes(head as (typeof DANGEROUS_COMMAND_PREFIXES)[number]); +} + +/** + * Generate the pattern key for a bash command. + * This is what gets stored when user approves, and what gets checked against approved patterns. + * + * Examples: + * - "ls -la" → "ls *" (flags don't count as subcommand) + * - "git push origin" → "git push *" (first non-flag arg is subcommand) + * - "git status" → "git status *" + * - "rm -rf /" → null (dangerous command) + * + * @param command The bash command to generate a pattern key for + * @returns The pattern key, or null if the command is dangerous + */ +export function generateBashPatternKey(command: string): string | null { + const tokens = command.trim().split(/\s+/); + if (tokens.length === 0 || !tokens[0]) return null; + + const head = tokens[0]; + + if (isDangerousCommand(command)) { + return null; + } + + // Find first non-flag argument as subcommand + const subcommand = tokens.slice(1).find((arg) => !arg.startsWith('-')); + + // Generate pattern: "git push *" or "ls *" + return subcommand ? `${head} ${subcommand} *` : `${head} *`; +} + +/** + * Generate suggested patterns for UI selection. + * Returns progressively broader patterns from specific to general. + * + * Example: "git push origin main" generates: + * - "git push *" (the pattern key) + * - "git *" (broader) + * + * @param command The bash command to generate suggestions for + * @returns Array of pattern suggestions (empty for dangerous commands) + */ +export function generateBashPatternSuggestions(command: string): string[] { + const tokens = command.trim().split(/\s+/); + if (tokens.length === 0 || !tokens[0]) return []; + + const head = tokens[0]; + + if (isDangerousCommand(command)) { + return []; + } + + const patterns: string[] = []; + + // Find non-flag arguments + const nonFlagArgs = tokens.slice(1).filter((arg) => !arg.startsWith('-')); + + // Add progressively broader patterns + // "git push origin" → ["git push *", "git *"] + if (nonFlagArgs.length > 0) { + patterns.push(`${head} ${nonFlagArgs[0]} *`); + } + patterns.push(`${head} *`); + + return patterns; +} diff --git a/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/factory.ts b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/factory.ts new file mode 100644 index 00000000..7d0c61d4 --- /dev/null +++ b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/factory.ts @@ -0,0 +1,45 @@ +import { InMemoryAllowedToolsProvider } from './in-memory.js'; +import { StorageAllowedToolsProvider } from './storage.js'; +import type { IAllowedToolsProvider } from './types.js'; +import type { StorageManager } from '@core/storage/index.js'; +import { ToolError } from '../../errors.js'; +import type { IDextoLogger } from '@core/logger/v2/types.js'; + +// TODO: Re-evaluate storage + toolConfirmation config together to avoid duplication +// Currently we have: +// - InMemoryAllowedToolsProvider with its own Map +// - StorageAllowedToolsProvider using config.storage.database +// - But config.storage.database might ALSO be in-memory (separate storage!) +// This creates potential duplication when storage backend is in-memory. +// Consider: Always use StorageAllowedToolsProvider and let storage backend handle memory vs persistence. + +export type AllowedToolsConfig = + | { + type: 'memory'; + } + | { + type: 'storage'; + storageManager: StorageManager; + }; + +/** + * Create an AllowedToolsProvider based on configuration. + */ +export function createAllowedToolsProvider( + config: AllowedToolsConfig, + logger: IDextoLogger +): IAllowedToolsProvider { + switch (config.type) { + case 'memory': + return new InMemoryAllowedToolsProvider(); + case 'storage': + return new StorageAllowedToolsProvider(config.storageManager, logger); + default: { + // Exhaustive check; at runtime this guards malformed config + const _exhaustive: never = config; + throw ToolError.configInvalid( + `Unsupported AllowedToolsConfig type: ${(config as any)?.type}` + ); + } + } +} diff --git a/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/in-memory.ts b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/in-memory.ts new file mode 100644 index 00000000..c2da0fa2 --- /dev/null +++ b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/in-memory.ts @@ -0,0 +1,43 @@ +import type { IAllowedToolsProvider } from './types.js'; + +export class InMemoryAllowedToolsProvider implements IAllowedToolsProvider { + /** + * Map key is sessionId (undefined => global approvals). Value is a set of + * approved tool names. + */ + private store: Map> = new Map(); + + constructor(initialGlobal?: Set) { + if (initialGlobal) { + this.store.set(undefined, new Set(initialGlobal)); + } + } + + private getSet(sessionId?: string): Set { + const key = sessionId ?? undefined; + let set = this.store.get(key); + if (!set) { + set = new Set(); + this.store.set(key, set); + } + return set; + } + + async allowTool(toolName: string, sessionId?: string): Promise { + this.getSet(sessionId).add(toolName); + } + + async disallowTool(toolName: string, sessionId?: string): Promise { + this.getSet(sessionId).delete(toolName); + } + + async isToolAllowed(toolName: string, sessionId?: string): Promise { + const scopedSet = this.store.get(sessionId ?? undefined); + const globalSet = this.store.get(undefined); + return Boolean(scopedSet?.has(toolName) || globalSet?.has(toolName)); + } + + async getAllowedTools(sessionId?: string): Promise> { + return new Set(this.getSet(sessionId)); + } +} diff --git a/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/storage.test.ts b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/storage.test.ts new file mode 100644 index 00000000..0b411489 --- /dev/null +++ b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/storage.test.ts @@ -0,0 +1,214 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { StorageAllowedToolsProvider } from './storage.js'; +import type { StorageManager } from '@core/storage/index.js'; +import { createMockLogger } from '@core/logger/v2/test-utils.js'; + +const mockLogger = createMockLogger(); + +describe('StorageAllowedToolsProvider', () => { + let provider: StorageAllowedToolsProvider; + let mockStorageManager: StorageManager; + let mockDatabase: any; + + beforeEach(() => { + mockDatabase = { + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + list: vi.fn(), + }; + + mockStorageManager = { + getDatabase: vi.fn().mockReturnValue(mockDatabase), + getCache: vi.fn(), + } as any; + + provider = new StorageAllowedToolsProvider(mockStorageManager, mockLogger); + }); + + describe('Session-scoped tool allowance', () => { + it('should store session-scoped tool approval', async () => { + mockDatabase.get.mockResolvedValue([]); + + await provider.allowTool('testTool', 'session123'); + + expect(mockDatabase.set).toHaveBeenCalledWith('allowedTools:session123', ['testTool']); + }); + + it('should store global tool approval when no sessionId provided', async () => { + mockDatabase.get.mockResolvedValue([]); + + await provider.allowTool('testTool'); + + expect(mockDatabase.set).toHaveBeenCalledWith('allowedTools:global', ['testTool']); + }); + + it('should append to existing session-scoped approvals', async () => { + mockDatabase.get.mockResolvedValue(['existingTool']); + + await provider.allowTool('newTool', 'session123'); + + expect(mockDatabase.set).toHaveBeenCalledWith('allowedTools:session123', [ + 'existingTool', + 'newTool', + ]); + }); + + it('should not duplicate tools in approval list', async () => { + mockDatabase.get.mockResolvedValue(['testTool']); + + await provider.allowTool('testTool', 'session123'); + + expect(mockDatabase.set).toHaveBeenCalledWith('allowedTools:session123', ['testTool']); + }); + }); + + describe('Session-scoped tool checking', () => { + it('should check session-scoped approvals first', async () => { + mockDatabase.get + .mockResolvedValueOnce(['sessionTool']) // session-scoped + .mockResolvedValueOnce(['globalTool']); // global + + const result = await provider.isToolAllowed('sessionTool', 'session123'); + + expect(result).toBe(true); + expect(mockDatabase.get).toHaveBeenCalledWith('allowedTools:session123'); + }); + + it('should fallback to global approvals when not found in session', async () => { + mockDatabase.get + .mockResolvedValueOnce([]) // session-scoped (empty) + .mockResolvedValueOnce(['globalTool']); // global + + const result = await provider.isToolAllowed('globalTool', 'session123'); + + expect(result).toBe(true); + expect(mockDatabase.get).toHaveBeenCalledWith('allowedTools:session123'); + expect(mockDatabase.get).toHaveBeenCalledWith('allowedTools:global'); + }); + + it('should return false when tool not found in session or global', async () => { + mockDatabase.get + .mockResolvedValueOnce([]) // session-scoped (empty) + .mockResolvedValueOnce([]); // global (empty) + + const result = await provider.isToolAllowed('unknownTool', 'session123'); + + expect(result).toBe(false); + }); + + it('should only check global when no sessionId provided', async () => { + mockDatabase.get.mockResolvedValueOnce(['globalTool']); + + const result = await provider.isToolAllowed('globalTool'); + + expect(result).toBe(true); + expect(mockDatabase.get).toHaveBeenCalledWith('allowedTools:global'); + }); + }); + + describe('Session-scoped tool removal', () => { + it('should remove tool from session-scoped approvals', async () => { + mockDatabase.get.mockResolvedValue(['tool1', 'tool2', 'tool3']); + + await provider.disallowTool('tool2', 'session123'); + + expect(mockDatabase.set).toHaveBeenCalledWith('allowedTools:session123', [ + 'tool1', + 'tool3', + ]); + }); + + it('should handle removal from global approvals', async () => { + mockDatabase.get.mockResolvedValue(['tool1', 'tool2']); + + await provider.disallowTool('tool1'); + + expect(mockDatabase.set).toHaveBeenCalledWith('allowedTools:global', ['tool2']); + }); + + it('should handle removal of non-existent tool gracefully', async () => { + mockDatabase.get.mockResolvedValue(['tool1']); + + await provider.disallowTool('nonExistent', 'session123'); + + expect(mockDatabase.set).toHaveBeenCalledWith('allowedTools:session123', ['tool1']); + }); + + it('should not call set when storage returns non-array', async () => { + mockDatabase.get.mockResolvedValue(null); + + await provider.disallowTool('tool1', 'session123'); + + expect(mockDatabase.set).not.toHaveBeenCalled(); + }); + }); + + describe('getAllowedTools', () => { + it('should return session-scoped tools as Set', async () => { + mockDatabase.get.mockResolvedValue(['tool1', 'tool2']); + + const result = await provider.getAllowedTools('session123'); + + expect(result).toEqual(new Set(['tool1', 'tool2'])); + expect(mockDatabase.get).toHaveBeenCalledWith('allowedTools:session123'); + }); + + it('should return global tools when no sessionId provided', async () => { + mockDatabase.get.mockResolvedValue(['globalTool']); + + const result = await provider.getAllowedTools(); + + expect(result).toEqual(new Set(['globalTool'])); + expect(mockDatabase.get).toHaveBeenCalledWith('allowedTools:global'); + }); + + it('should return empty Set when no tools stored', async () => { + mockDatabase.get.mockResolvedValue(null); + + const result = await provider.getAllowedTools('session123'); + + expect(result).toEqual(new Set()); + }); + }); + + describe('Error handling', () => { + it('should handle storage errors gracefully', async () => { + mockDatabase.get.mockRejectedValue(new Error('Storage error')); + + await expect(provider.isToolAllowed('testTool', 'session123')).rejects.toThrow( + 'Storage error' + ); + }); + + it('should handle malformed data in storage', async () => { + mockDatabase.get.mockResolvedValue('invalid-data'); + + const result = await provider.isToolAllowed('testTool', 'session123'); + + expect(result).toBe(false); + }); + }); + + describe('Key naming convention', () => { + it('should use correct key format for session-scoped storage', async () => { + mockDatabase.get.mockResolvedValue([]); + + await provider.allowTool('testTool', 'user-session-456'); + + expect(mockDatabase.get).toHaveBeenCalledWith('allowedTools:user-session-456'); + expect(mockDatabase.set).toHaveBeenCalledWith('allowedTools:user-session-456', [ + 'testTool', + ]); + }); + + it('should use correct key format for global storage', async () => { + mockDatabase.get.mockResolvedValue([]); + + await provider.allowTool('testTool'); + + expect(mockDatabase.get).toHaveBeenCalledWith('allowedTools:global'); + expect(mockDatabase.set).toHaveBeenCalledWith('allowedTools:global', ['testTool']); + }); + }); +}); diff --git a/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/storage.ts b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/storage.ts new file mode 100644 index 00000000..dc5b0f44 --- /dev/null +++ b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/storage.ts @@ -0,0 +1,74 @@ +import type { StorageManager } from '@core/storage/index.js'; +import type { IAllowedToolsProvider } from './types.js'; +import type { IDextoLogger } from '@core/logger/v2/types.js'; + +/** + * Storage-backed implementation that persists allowed tools in the Dexto + * storage manager. The key scheme is: + * allowedTools: – approvals scoped to a session + * allowedTools:global – global approvals (sessionId undefined) + * + * Using the database backend for persistence. + */ +export class StorageAllowedToolsProvider implements IAllowedToolsProvider { + private logger: IDextoLogger; + + constructor( + private storageManager: StorageManager, + logger: IDextoLogger + ) { + this.logger = logger; + } + + private buildKey(sessionId?: string) { + return sessionId ? `allowedTools:${sessionId}` : 'allowedTools:global'; + } + + async allowTool(toolName: string, sessionId?: string): Promise { + const key = this.buildKey(sessionId); + this.logger.debug(`Adding allowed tool '${toolName}' for key '${key}'`); + + // Persist as a plain string array to avoid JSON <-> Set issues across backends + const existingRaw = await this.storageManager.getDatabase().get(key); + const newSet = new Set(Array.isArray(existingRaw) ? existingRaw : []); + newSet.add(toolName); + + // Store a fresh array copy – never the live Set instance + await this.storageManager.getDatabase().set(key, Array.from(newSet)); + this.logger.debug(`Added allowed tool '${toolName}' for key '${key}'`); + } + + async disallowTool(toolName: string, sessionId?: string): Promise { + const key = this.buildKey(sessionId); + this.logger.debug(`Removing allowed tool '${toolName}' for key '${key}'`); + + const existingRaw = await this.storageManager.getDatabase().get(key); + if (!Array.isArray(existingRaw)) return; + + const newSet = new Set(existingRaw); + newSet.delete(toolName); + await this.storageManager.getDatabase().set(key, Array.from(newSet)); + } + + async isToolAllowed(toolName: string, sessionId?: string): Promise { + const sessionArr = await this.storageManager + .getDatabase() + .get(this.buildKey(sessionId)); + if (Array.isArray(sessionArr) && sessionArr.includes(toolName)) return true; + + // Fallback to global approvals + const globalArr = await this.storageManager + .getDatabase() + .get(this.buildKey(undefined)); + const allowed = Array.isArray(globalArr) ? globalArr.includes(toolName) : false; + this.logger.debug( + `Checked allowed tool '${toolName}' in session '${sessionId ?? 'global'}' – allowed=${allowed}` + ); + return allowed; + } + + async getAllowedTools(sessionId?: string): Promise> { + const arr = await this.storageManager.getDatabase().get(this.buildKey(sessionId)); + return new Set(Array.isArray(arr) ? arr : []); + } +} diff --git a/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/types.ts b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/types.ts new file mode 100644 index 00000000..3b78373b --- /dev/null +++ b/dexto/packages/core/src/tools/confirmation/allowed-tools-provider/types.ts @@ -0,0 +1,36 @@ +/** + * Interface for allowed tools storage (in-memory, DB, etc.) + * + * Multi-user support: For multi-tenancy, we can consider either: + * - Instantiating a provider per user (current pattern, recommended for most cases) + * - Or, add userId as a parameter to each method for batch/admin/multi-user operations: + * allowTool(toolName: string, userId: string): Promise + * ...etc. + * - You can also add static/factory methods to create user-scoped providers, e.g., + * AllowedToolsProvider.forUser(userId) + * + * AllowedToolsProvider supports both single-user and multi-user scenarios. + * - If `userId` is omitted, the implementation will use a default user (e.g., from getUserId()). + * - For multi-user/admin scenarios, always pass `userId` explicitly. + * - We can enforce this by having a separate env variable/feature-flag for multi-user and having + * strict check for the user id if the feature flag is set. + */ +export interface IAllowedToolsProvider { + /** + * Persist an approval for a tool. If `sessionId` is provided the approval is + * scoped to that session. When omitted the approval is treated as global. + */ + allowTool(toolName: string, sessionId?: string): Promise; + + /** Remove an approval. */ + disallowTool(toolName: string, sessionId?: string): Promise; + + /** + * Check whether the given tool is currently allowed. If `sessionId` is + * provided the session-scoped list is checked first, then any global entry. + */ + isToolAllowed(toolName: string, sessionId?: string): Promise; + + /** Optional helper to introspect all approvals for debugging. */ + getAllowedTools?(sessionId?: string): Promise>; +} diff --git a/dexto/packages/core/src/tools/custom-tool-registry.test.ts b/dexto/packages/core/src/tools/custom-tool-registry.test.ts new file mode 100644 index 00000000..41e5748b --- /dev/null +++ b/dexto/packages/core/src/tools/custom-tool-registry.test.ts @@ -0,0 +1,591 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { CustomToolRegistry } from './custom-tool-registry.js'; +import type { CustomToolProvider, ToolCreationContext } from './custom-tool-registry.js'; +import type { InternalTool } from './types.js'; +import { z } from 'zod'; +import { ToolErrorCode } from './error-codes.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { customToolSchemaRegistry } from './custom-tool-schema-registry.js'; + +// Mock logger for testing +const mockLogger: IDextoLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), +} as any; + +// Mock agent for testing +const mockAgent = {} as any; + +// Mock context for testing +const mockContext = { logger: mockLogger, agent: mockAgent }; + +// Mock tool for testing +const createMockTool = (id: string): InternalTool => ({ + id, + description: `Mock tool ${id}`, + inputSchema: z.object({ param: z.string() }), + execute: async (input: unknown) => ({ result: 'success', input }), +}); + +// Mock provider configurations +const mockProviderAConfig = z.object({ + type: z.literal('mock-provider-a'), + settingA: z.string(), + optionalSetting: z.string().optional(), +}); + +const mockProviderBConfig = z.object({ + type: z.literal('mock-provider-b'), + settingB: z.number(), + requiredArray: z.array(z.string()), +}); + +type MockProviderAConfig = z.output; +type MockProviderBConfig = z.output; + +// Mock providers +const createMockProviderA = (): CustomToolProvider<'mock-provider-a', MockProviderAConfig> => ({ + type: 'mock-provider-a', + configSchema: mockProviderAConfig, + create: (config: MockProviderAConfig, _context: ToolCreationContext): InternalTool[] => { + return [createMockTool(`${config.type}-tool-1`), createMockTool(`${config.type}-tool-2`)]; + }, + metadata: { + displayName: 'Mock Provider A', + description: 'A mock provider for testing', + category: 'testing', + }, +}); + +const createMockProviderB = (): CustomToolProvider<'mock-provider-b', MockProviderBConfig> => ({ + type: 'mock-provider-b', + configSchema: mockProviderBConfig, + create: (config: MockProviderBConfig, _context: ToolCreationContext): InternalTool[] => { + return [createMockTool(`${config.type}-tool-1`)]; + }, + metadata: { + displayName: 'Mock Provider B', + description: 'Another mock provider for testing', + }, +}); + +describe('CustomToolRegistry', () => { + let registry: CustomToolRegistry; + + beforeEach(() => { + // Clear the global schema registry before each test + customToolSchemaRegistry.clear(); + registry = new CustomToolRegistry(); + }); + + describe('register()', () => { + it('successfully registers a provider', () => { + const provider = createMockProviderA(); + + expect(() => registry.register(provider)).not.toThrow(); + expect(registry.has('mock-provider-a')).toBe(true); + }); + + it('registers multiple different providers', () => { + const providerA = createMockProviderA(); + const providerB = createMockProviderB(); + + registry.register(providerA); + registry.register(providerB); + + expect(registry.has('mock-provider-a')).toBe(true); + expect(registry.has('mock-provider-b')).toBe(true); + expect(registry.getTypes()).toEqual(['mock-provider-a', 'mock-provider-b']); + }); + + it('throws error when registering duplicate provider type', () => { + const provider1 = createMockProviderA(); + const provider2 = createMockProviderA(); + + registry.register(provider1); + + expect(() => registry.register(provider2)).toThrow( + expect.objectContaining({ + code: ToolErrorCode.CUSTOM_TOOL_PROVIDER_ALREADY_REGISTERED, + scope: ErrorScope.TOOLS, + type: ErrorType.USER, + context: { type: 'mock-provider-a' }, + }) + ); + }); + + it('throws error with recovery suggestion for duplicate registration', () => { + const provider = createMockProviderA(); + registry.register(provider); + + try { + registry.register(provider); + expect.fail('Should have thrown error'); + } catch (error: any) { + expect(error.message).toContain( + "Custom tool provider 'mock-provider-a' is already registered" + ); + expect(error.recovery).toContain( + 'Use unregister() first if you want to replace it' + ); + } + }); + }); + + describe('unregister()', () => { + it('successfully unregisters an existing provider', () => { + const provider = createMockProviderA(); + registry.register(provider); + + const result = registry.unregister('mock-provider-a'); + + expect(result).toBe(true); + expect(registry.has('mock-provider-a')).toBe(false); + }); + + it('returns false when unregistering non-existent provider', () => { + const result = registry.unregister('non-existent-provider'); + + expect(result).toBe(false); + }); + + it('allows re-registration after unregistering', () => { + const provider = createMockProviderA(); + registry.register(provider); + registry.unregister('mock-provider-a'); + + // Also clear schema registry for this test since schema registry persists by design + customToolSchemaRegistry.clear(); + + expect(() => registry.register(provider)).not.toThrow(); + expect(registry.has('mock-provider-a')).toBe(true); + }); + + it('unregisters correct provider when multiple are registered', () => { + const providerA = createMockProviderA(); + const providerB = createMockProviderB(); + registry.register(providerA); + registry.register(providerB); + + registry.unregister('mock-provider-a'); + + expect(registry.has('mock-provider-a')).toBe(false); + expect(registry.has('mock-provider-b')).toBe(true); + }); + }); + + describe('get()', () => { + it('returns registered provider', () => { + const provider = createMockProviderA(); + registry.register(provider); + + const retrieved = registry.get('mock-provider-a'); + + expect(retrieved).toBeDefined(); + expect(retrieved?.type).toBe('mock-provider-a'); + expect(retrieved?.metadata?.displayName).toBe('Mock Provider A'); + }); + + it('returns undefined for non-existent provider', () => { + const retrieved = registry.get('non-existent-provider'); + + expect(retrieved).toBeUndefined(); + }); + + it('returns correct provider when multiple are registered', () => { + const providerA = createMockProviderA(); + const providerB = createMockProviderB(); + registry.register(providerA); + registry.register(providerB); + + const retrievedA = registry.get('mock-provider-a'); + const retrievedB = registry.get('mock-provider-b'); + + expect(retrievedA?.type).toBe('mock-provider-a'); + expect(retrievedB?.type).toBe('mock-provider-b'); + }); + + it('returned provider can create tools', () => { + const provider = createMockProviderA(); + registry.register(provider); + + const retrieved = registry.get('mock-provider-a'); + const tools = retrieved?.create( + { type: 'mock-provider-a', settingA: 'test' }, + mockContext + ); + + expect(tools).toHaveLength(2); + expect(tools![0]!.id).toBe('mock-provider-a-tool-1'); + expect(tools![1]!.id).toBe('mock-provider-a-tool-2'); + }); + }); + + describe('has()', () => { + it('returns true for registered provider', () => { + const provider = createMockProviderA(); + registry.register(provider); + + expect(registry.has('mock-provider-a')).toBe(true); + }); + + it('returns false for non-existent provider', () => { + expect(registry.has('non-existent-provider')).toBe(false); + }); + + it('returns false after provider is unregistered', () => { + const provider = createMockProviderA(); + registry.register(provider); + registry.unregister('mock-provider-a'); + + expect(registry.has('mock-provider-a')).toBe(false); + }); + }); + + describe('getTypes()', () => { + it('returns empty array when no providers registered', () => { + expect(registry.getTypes()).toEqual([]); + }); + + it('returns single type when one provider is registered', () => { + const provider = createMockProviderA(); + registry.register(provider); + + expect(registry.getTypes()).toEqual(['mock-provider-a']); + }); + + it('returns all registered types in order', () => { + const providerA = createMockProviderA(); + const providerB = createMockProviderB(); + registry.register(providerA); + registry.register(providerB); + + const types = registry.getTypes(); + expect(types).toHaveLength(2); + expect(types).toContain('mock-provider-a'); + expect(types).toContain('mock-provider-b'); + }); + + it('updates correctly when providers are unregistered', () => { + const providerA = createMockProviderA(); + const providerB = createMockProviderB(); + registry.register(providerA); + registry.register(providerB); + + registry.unregister('mock-provider-a'); + + expect(registry.getTypes()).toEqual(['mock-provider-b']); + }); + }); + + describe('validateConfig()', () => { + beforeEach(() => { + registry.register(createMockProviderA()); + registry.register(createMockProviderB()); + }); + + it('validates correct configuration for provider A', () => { + const config = { + type: 'mock-provider-a', + settingA: 'test-value', + }; + + const validated = registry.validateConfig(config); + + expect(validated).toEqual(config); + expect(validated.type).toBe('mock-provider-a'); + expect(validated.settingA).toBe('test-value'); + }); + + it('validates correct configuration for provider B', () => { + const config = { + type: 'mock-provider-b', + settingB: 42, + requiredArray: ['item1', 'item2'], + }; + + const validated = registry.validateConfig(config); + + expect(validated).toEqual(config); + expect(validated.type).toBe('mock-provider-b'); + expect(validated.settingB).toBe(42); + }); + + it('validates configuration with optional fields', () => { + const config = { + type: 'mock-provider-a', + settingA: 'test', + optionalSetting: 'optional-value', + }; + + const validated = registry.validateConfig(config); + + expect(validated.optionalSetting).toBe('optional-value'); + }); + + it('throws error for unknown provider type', () => { + const config = { + type: 'unknown-provider', + someSetting: 'value', + }; + + expect(() => registry.validateConfig(config)).toThrow( + expect.objectContaining({ + code: ToolErrorCode.CUSTOM_TOOL_PROVIDER_UNKNOWN, + scope: ErrorScope.TOOLS, + type: ErrorType.USER, + context: { + type: 'unknown-provider', + availableTypes: expect.arrayContaining([ + 'mock-provider-a', + 'mock-provider-b', + ]), + }, + }) + ); + }); + + it('includes available types in unknown provider error', () => { + const config = { type: 'unknown-provider' }; + + try { + registry.validateConfig(config); + expect.fail('Should have thrown error'); + } catch (error: any) { + expect(error.message).toContain("Unknown custom tool provider: 'unknown-provider'"); + expect(error.recovery).toContain('mock-provider-a'); + expect(error.recovery).toContain('mock-provider-b'); + } + }); + + it('throws ZodError for invalid configuration structure', () => { + const config = { + type: 'mock-provider-a', + // missing required settingA + }; + + expect(() => registry.validateConfig(config)).toThrow(); + }); + + it('throws ZodError for wrong type in configuration', () => { + const config = { + type: 'mock-provider-b', + settingB: 'should-be-number', // wrong type + requiredArray: ['item1'], + }; + + expect(() => registry.validateConfig(config)).toThrow(); + }); + + it('throws error for configuration missing type field', () => { + const config = { + settingA: 'test', + }; + + expect(() => registry.validateConfig(config)).toThrow(); + }); + + it('throws error for null config', () => { + expect(() => registry.validateConfig(null)).toThrow(); + }); + + it('throws error for undefined config', () => { + expect(() => registry.validateConfig(undefined)).toThrow(); + }); + + it('allows extra fields to pass through with passthrough', () => { + const config = { + type: 'mock-provider-a', + settingA: 'test', + extraField: 'should-pass-through', + }; + + // This should not throw because the type schema uses passthrough() + const validated = registry.validateConfig(config); + expect(validated.type).toBe('mock-provider-a'); + }); + }); + + describe('clear()', () => { + it('clears all registered providers', () => { + const providerA = createMockProviderA(); + const providerB = createMockProviderB(); + registry.register(providerA); + registry.register(providerB); + + registry.clear(); + + expect(registry.getTypes()).toEqual([]); + expect(registry.has('mock-provider-a')).toBe(false); + expect(registry.has('mock-provider-b')).toBe(false); + }); + + it('allows registration after clearing', () => { + const provider = createMockProviderA(); + registry.register(provider); + registry.clear(); + + // Also clear schema registry for this test since schema registry persists by design + customToolSchemaRegistry.clear(); + + expect(() => registry.register(provider)).not.toThrow(); + expect(registry.has('mock-provider-a')).toBe(true); + }); + + it('is safe to call on empty registry', () => { + expect(() => registry.clear()).not.toThrow(); + expect(registry.getTypes()).toEqual([]); + }); + + it('is safe to call multiple times', () => { + const provider = createMockProviderA(); + registry.register(provider); + + registry.clear(); + registry.clear(); + + expect(registry.getTypes()).toEqual([]); + }); + }); + + describe('Provider Integration', () => { + it('provider can access logger from creation context', () => { + const loggerSpy = vi.fn(); + const provider: CustomToolProvider<'test-logger', { type: 'test-logger' }> = { + type: 'test-logger', + configSchema: z.object({ type: z.literal('test-logger') }), + create: (_config, context) => { + loggerSpy(context.logger); + return []; + }, + }; + + registry.register(provider); + const retrieved = registry.get('test-logger'); + retrieved?.create({ type: 'test-logger' }, mockContext); + + expect(loggerSpy).toHaveBeenCalledWith(mockLogger); + }); + + it('provider can access services from creation context', () => { + const servicesSpy = vi.fn(); + const mockServices = { + searchService: { search: vi.fn() }, + customService: { custom: vi.fn() }, + }; + + const provider: CustomToolProvider<'test-services', { type: 'test-services' }> = { + type: 'test-services', + configSchema: z.object({ type: z.literal('test-services') }), + create: (_config, context) => { + servicesSpy(context.services); + return []; + }, + }; + + registry.register(provider); + const retrieved = registry.get('test-services'); + retrieved?.create( + { type: 'test-services' }, + { logger: mockLogger, agent: mockAgent, services: mockServices } + ); + + expect(servicesSpy).toHaveBeenCalledWith(mockServices); + }); + + it('provider can use validated config to create tools', () => { + const provider: CustomToolProvider< + 'config-based', + { type: 'config-based'; toolCount: number } + > = { + type: 'config-based', + configSchema: z.object({ + type: z.literal('config-based'), + toolCount: z.number(), + }), + create: (config, _context) => { + return Array.from({ length: config.toolCount }, (_, i) => + createMockTool(`tool-${i}`) + ); + }, + }; + + registry.register(provider); + const validated = registry.validateConfig({ type: 'config-based', toolCount: 3 }); + const retrieved = registry.get('config-based'); + const tools = retrieved?.create(validated, mockContext); + + expect(tools).toHaveLength(3); + expect(tools![0]!.id).toBe('tool-0'); + expect(tools![2]!.id).toBe('tool-2'); + }); + }); + + describe('Edge Cases', () => { + it('handles provider with empty metadata', () => { + const provider: CustomToolProvider<'no-metadata', { type: 'no-metadata' }> = { + type: 'no-metadata', + configSchema: z.object({ type: z.literal('no-metadata') }), + create: () => [], + }; + + registry.register(provider); + const retrieved = registry.get('no-metadata'); + + expect(retrieved?.metadata).toBeUndefined(); + }); + + it('handles provider that creates zero tools', () => { + const provider: CustomToolProvider<'zero-tools', { type: 'zero-tools' }> = { + type: 'zero-tools', + configSchema: z.object({ type: z.literal('zero-tools') }), + create: () => [], + }; + + registry.register(provider); + const retrieved = registry.get('zero-tools'); + const tools = retrieved?.create({ type: 'zero-tools' }, mockContext); + + expect(tools).toEqual([]); + }); + + it('handles provider with complex nested schema', () => { + const complexSchema = z.object({ + type: z.literal('complex'), + nested: z.object({ + level1: z.object({ + level2: z.array(z.string()), + }), + }), + }); + + const provider: CustomToolProvider<'complex', z.output> = { + type: 'complex', + configSchema: complexSchema, + create: () => [], + }; + + registry.register(provider); + + const config = { + type: 'complex', + nested: { + level1: { + level2: ['a', 'b', 'c'], + }, + }, + }; + + const validated = registry.validateConfig(config); + expect(validated).toEqual(config); + }); + }); +}); diff --git a/dexto/packages/core/src/tools/custom-tool-registry.ts b/dexto/packages/core/src/tools/custom-tool-registry.ts new file mode 100644 index 00000000..f2c05464 --- /dev/null +++ b/dexto/packages/core/src/tools/custom-tool-registry.ts @@ -0,0 +1,159 @@ +import type { InternalTool } from './types.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import type { DextoAgent } from '../agent/DextoAgent.js'; +import { z } from 'zod'; +import { ToolError } from './errors.js'; +import { BaseRegistry, type RegistryErrorFactory } from '../providers/base-registry.js'; +import { customToolSchemaRegistry } from './custom-tool-schema-registry.js'; + +/** + * Context passed to custom tool providers when creating tools. + * Provides access to the agent instance for bidirectional communication. + * + * **Bidirectional Services Pattern:** + * Some services need both: + * - Agent → Service: LLM calls tools that invoke service methods + * - Service → Agent: Service emits events that trigger agent invocation + * + * Implementation pattern: + * ```typescript + * create: (config, context) => { + * const service = new MyService(config, context.logger); + * + * // Wire up Service → Agent communication + * service.on('event', async (data) => { + * await context.agent.sendMessage({ + * role: 'user', + * content: data.prompt, + * }); + * }); + * + * // Return Agent → Service tools + * return [createMyTool(service)]; + * } + * ``` + * + * **Future Consideration:** + * For complex event routing or decoupled architectures, consider using an Event Bus pattern + * where services emit events to a central bus and the agent/app subscribes. This would + * provide better separation of concerns at the cost of more indirection and complexity. + */ +export interface ToolCreationContext { + logger: IDextoLogger; + agent: DextoAgent; + /** + * Optional services available to custom tool providers. + * + * Core services (provided by agent): + * - approvalManager: For tools that need approval flows + * - storageManager: For tools that need persistence + * - resourceManager: For tools that need resource access + * - searchService: For tools that need search capabilities + * + * External tool providers can add their own services using the index signature. + */ + services?: { + searchService?: any; + approvalManager?: any; + resourceManager?: any; + storageManager?: import('../storage/index.js').StorageManager; + [key: string]: any; // Extensible for external tool providers + }; +} + +/** + * Custom tool provider interface. + * Allows external code to register tool providers that create one or more tools. + * Mirrors the BlobStoreProvider pattern for consistency. + * + * @template TType - The provider type discriminator (must match config.type) + * @template TConfig - The provider configuration type (must include { type: TType }) + */ +export interface CustomToolProvider< + TType extends string = string, + TConfig extends { type: TType } = any, +> { + /** Unique type identifier matching the discriminator in config */ + type: TType; + + /** Zod schema for runtime validation of provider configuration */ + configSchema: z.ZodType; + + /** + * Factory function to create tools from validated configuration + * @param config - Validated configuration matching configSchema + * @param context - Tool creation context with logger and optional services + * @returns Array of tools to register + */ + create(config: TConfig, context: ToolCreationContext): InternalTool[]; + + /** Optional metadata for display and categorization */ + metadata?: { + displayName: string; + description: string; + category?: string; + }; +} + +/** + * Error factory for custom tool registry errors. + * Uses ToolError for consistent error handling. + */ +const customToolErrorFactory: RegistryErrorFactory = { + alreadyRegistered: (type: string) => ToolError.customToolProviderAlreadyRegistered(type), + notFound: (type: string, availableTypes: string[]) => + ToolError.unknownCustomToolProvider(type, availableTypes), +}; + +/** + * Registry for custom tool providers. + * Mirrors BlobStoreRegistry pattern for consistency across Dexto provider system. + * + * Custom tool providers can be registered from external code (CLI, apps, examples) + * and are validated at runtime using their Zod schemas. + * + * Extends BaseRegistry for common registry functionality. + * + * When a provider is registered, its config schema is also registered in the + * customToolSchemaRegistry for early validation at config load time. + */ +export class CustomToolRegistry extends BaseRegistry { + constructor() { + super(customToolErrorFactory); + } + + /** + * Register a custom tool provider. + * Also registers the provider's config schema for early validation. + * + * @param provider - The custom tool provider to register + * @throws Error if a provider with the same type is already registered + */ + override register(provider: CustomToolProvider): void { + // Register the provider with the base registry + super.register(provider); + + // Also register the provider's config schema for early validation + customToolSchemaRegistry.register(provider.type, provider.configSchema); + } + + /** + * Unregister a custom tool provider. + * Note: This does NOT unregister the schema from customToolSchemaRegistry + * to avoid breaking active configs that reference the schema. + * + * @param type - The provider type to unregister + * @returns true if the provider was unregistered, false if it wasn't registered + */ + override unregister(type: string): boolean { + // Only unregister from this registry, not from schema registry + // Schema registry should persist for the lifetime of the application + return super.unregister(type); + } +} + +/** + * Global singleton instance of the custom tool registry. + * Custom tool providers should be registered at application startup. + */ +export const customToolRegistry = new CustomToolRegistry(); diff --git a/dexto/packages/core/src/tools/custom-tool-schema-registry.ts b/dexto/packages/core/src/tools/custom-tool-schema-registry.ts new file mode 100644 index 00000000..8cdcac29 --- /dev/null +++ b/dexto/packages/core/src/tools/custom-tool-schema-registry.ts @@ -0,0 +1,204 @@ +/** + * Custom Tool Schema Registry + * + * Registry for custom tool provider configuration schemas. + * Allows core to validate provider-specific fields at config load time + * by building a discriminated union of all registered schemas. + * + * Architecture: + * 1. Providers register their config schemas when they register with customToolRegistry + * 2. Core uses this registry to build a discriminated union schema at runtime + * 3. Agent config validation uses the union to validate provider-specific fields early + * + * Benefits: + * - Early validation (at config load time, not runtime) + * - Type safety (full Zod validation for all provider fields) + * - IDE support (TypeScript knows all provider fields) + * - Single source of truth (provider schema defines everything) + */ + +import { z } from 'zod'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; + +/** + * Registry for custom tool provider configuration schemas. + * + * Providers register their config schemas here, allowing core to validate + * provider-specific configuration fields at config load time. + * + * Note: This is a lightweight registry that doesn't extend BaseRegistry + * to avoid complexity. It simply stores schemas in a Map. + */ +// Create a no-op logger for when logger is not available +const createNoOpLogger = (): IDextoLogger => { + const noOpLogger: IDextoLogger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + silly: () => {}, + trackException: () => {}, + setLevel: () => {}, + getLevel: () => 'info', + getLogFilePath: () => null, + destroy: () => Promise.resolve(), + createChild: () => noOpLogger, + }; + return noOpLogger; +}; + +class CustomToolSchemaRegistry { + private schemas = new Map>(); + private logger: IDextoLogger; + + constructor(logger: IDextoLogger) { + this.logger = logger; + } + + /** + * Register a provider's config schema. + * Called automatically when a custom tool provider is registered. + * + * @param type Provider type (must match the 'type' field in config) + * @param schema Zod schema for the provider's configuration + * + * @throws Error if schema is already registered for this type + */ + register>(type: string, schema: T): void { + if (this.schemas.has(type)) { + throw new Error(`Config schema already registered for provider type: ${type}`); + } + this.schemas.set(type, schema); + this.logger.debug(`Registered config schema for provider: ${type}`); + } + + /** + * Get a provider's config schema. + * + * @param type Provider type + * @returns The registered schema, or undefined if not found + */ + get(type: string): z.ZodType | undefined { + return this.schemas.get(type); + } + + /** + * Check if a provider type has a registered schema. + * + * @param type Provider type + * @returns true if schema is registered + */ + has(type: string): boolean { + return this.schemas.has(type); + } + + /** + * Get all registered provider types. + * + * @returns Array of provider type strings + */ + getRegisteredTypes(): string[] { + return Array.from(this.schemas.keys()); + } + + /** + * Create a discriminated union schema for all registered providers. + * This enables early validation of provider-specific fields at config load time. + * + * The union is discriminated by the 'type' field, which provides better error + * messages when validation fails. + * + * @returns Discriminated union schema if providers are registered, + * passthrough schema otherwise (for backward compatibility) + */ + createUnionSchema(): z.ZodType { + const types = this.getRegisteredTypes(); + + if (types.length === 0) { + // No providers registered - use base passthrough schema for backward compatibility + // This allows configs to be loaded before providers are registered + this.logger.debug( + 'No provider schemas registered - using passthrough schema for custom tools' + ); + return z + .object({ + type: z.string().describe('Custom tool provider type'), + }) + .passthrough() + .describe( + 'Custom tool provider configuration (no schemas registered - validation deferred to runtime)' + ); + } + + // Get all registered schemas - guaranteed to exist since we check types.length > 0 + const schemas: z.ZodType[] = []; + for (const type of types) { + const schema = this.get(type); + if (schema) { + schemas.push(schema); + } + } + + if (schemas.length === 0) { + // Shouldn't happen, but handle gracefully + this.logger.warn('No schemas found despite having registered types'); + return z + .object({ + type: z.string(), + }) + .passthrough(); + } + + if (schemas.length === 1) { + // Single provider - just return its schema + // Type assertion is safe because we just pushed it and checked length > 0 + this.logger.debug(`Using single provider schema: ${types[0]}`); + return schemas[0]!; + } + + // Multiple providers - create regular union (discriminated union requires specific schema types) + this.logger.debug( + `Creating union schema for ${schemas.length} providers: ${types.join(', ')}` + ); + + // Cast to tuple type required by z.union + return z.union(schemas as [z.ZodType, z.ZodType, ...z.ZodType[]]); + } + + /** + * Clear all registered schemas. + * Primarily used for testing. + */ + clear(): void { + this.schemas.clear(); + this.logger.debug('Cleared all registered provider schemas'); + } +} + +// Global singleton instance +// Uses DextoLogComponent.TOOLS for logging +let globalInstance: CustomToolSchemaRegistry | undefined; + +/** + * Get the global custom tool schema registry instance. + * Creates it on first access with the provided logger. + * + * @param logger Optional logger for the registry (only used on first access) + * @returns The global registry instance + */ +export function getCustomToolSchemaRegistry(logger?: IDextoLogger): CustomToolSchemaRegistry { + if (!globalInstance) { + const registryLogger = logger + ? logger.createChild(DextoLogComponent.TOOLS) + : createNoOpLogger(); + globalInstance = new CustomToolSchemaRegistry(registryLogger); + } + return globalInstance; +} + +/** + * Global custom tool schema registry instance. + * Use this for registering and retrieving provider config schemas. + */ +export const customToolSchemaRegistry = getCustomToolSchemaRegistry(); diff --git a/dexto/packages/core/src/tools/display-types.ts b/dexto/packages/core/src/tools/display-types.ts new file mode 100644 index 00000000..f4f1be43 --- /dev/null +++ b/dexto/packages/core/src/tools/display-types.ts @@ -0,0 +1,184 @@ +/** + * Tool Display Types + * + * Discriminated union types for structured tool result rendering. + * These types enable both CLI and WebUI to render tool results with + * appropriate formatting (diffs, shell output, search results, etc.) + * + * Tools return `_display` field in their result, which is preserved + * by the sanitizer in `SanitizedToolResult.meta.display`. + */ + +// ============================================================================= +// Discriminated Union Types +// ============================================================================= + +/** + * Discriminated union of all tool display data types. + * Switch on `type` field for exhaustive handling. + */ +export type ToolDisplayData = + | DiffDisplayData + | ShellDisplayData + | SearchDisplayData + | FileDisplayData + | GenericDisplayData; + +/** + * Display data for file edit operations (edit_file, write_file overwrites). + * Contains unified diff format for rendering changes. + */ +export interface DiffDisplayData { + type: 'diff'; + /** Unified diff string (output of `diff` package's createPatch) */ + unified: string; + /** Path to the file that was modified */ + filename: string; + /** Number of lines added */ + additions: number; + /** Number of lines removed */ + deletions: number; + /** Original file content (optional, for approval preview) */ + beforeContent?: string; + /** New file content (optional, for approval preview) */ + afterContent?: string; +} + +/** + * Display data for shell command execution (bash_exec). + * Contains command metadata and output for structured rendering. + */ +export interface ShellDisplayData { + type: 'shell'; + /** The command that was executed */ + command: string; + /** Exit code from the command (0 = success) */ + exitCode: number; + /** Execution duration in milliseconds */ + duration: number; + /** Whether command is running in background */ + isBackground?: boolean; + /** Standard output from the command */ + stdout?: string; + /** Standard error from the command */ + stderr?: string; +} + +/** + * Display data for search operations (grep_content, glob_files). + * Contains structured match results for formatted rendering. + */ +export interface SearchDisplayData { + type: 'search'; + /** The search pattern used */ + pattern: string; + /** Array of match results */ + matches: SearchMatch[]; + /** Total number of matches found (may exceed displayed matches) */ + totalMatches: number; + /** Whether results were truncated due to limits */ + truncated: boolean; +} + +/** + * Individual search match result. + */ +export interface SearchMatch { + /** File path where match was found */ + file: string; + /** Line number of the match (0 for glob results) */ + line: number; + /** Content of the matching line or filename */ + content: string; + /** Optional surrounding context lines */ + context?: string[]; +} + +/** + * Display data for file operations (read_file, write_file create). + * Contains file metadata for simple status rendering. + */ +export interface FileDisplayData { + type: 'file'; + /** Path to the file */ + path: string; + /** Type of operation performed */ + operation: 'read' | 'write' | 'create' | 'delete'; + /** File size in bytes (optional) */ + size?: number; + /** Number of lines read/written (optional) */ + lineCount?: number; + /** Path to backup file if created (optional) */ + backupPath?: string; + /** File content for create operations (used in approval preview) */ + content?: string; +} + +/** + * Fallback display data for unknown tools or MCP tools. + * Renderers should fall back to rendering content[] directly. + */ +export interface GenericDisplayData { + type: 'generic'; +} + +// ============================================================================= +// Type Guards +// ============================================================================= + +/** + * Type guard for DiffDisplayData. + */ +export function isDiffDisplay(d: ToolDisplayData): d is DiffDisplayData { + return d.type === 'diff'; +} + +/** + * Type guard for ShellDisplayData. + */ +export function isShellDisplay(d: ToolDisplayData): d is ShellDisplayData { + return d.type === 'shell'; +} + +/** + * Type guard for SearchDisplayData. + */ +export function isSearchDisplay(d: ToolDisplayData): d is SearchDisplayData { + return d.type === 'search'; +} + +/** + * Type guard for FileDisplayData. + */ +export function isFileDisplay(d: ToolDisplayData): d is FileDisplayData { + return d.type === 'file'; +} + +/** + * Type guard for GenericDisplayData. + */ +export function isGenericDisplay(d: ToolDisplayData): d is GenericDisplayData { + return d.type === 'generic'; +} + +// ============================================================================= +// Validation +// ============================================================================= + +/** Valid display type values */ +const VALID_DISPLAY_TYPES = ['diff', 'shell', 'search', 'file', 'generic'] as const; + +/** + * Validates that an unknown value is a valid ToolDisplayData. + * Used by sanitizer to safely extract _display from tool results. + */ +export function isValidDisplayData(d: unknown): d is ToolDisplayData { + if (d === null || typeof d !== 'object') { + return false; + } + const obj = d as Record; + return ( + typeof obj.type === 'string' && + (VALID_DISPLAY_TYPES as readonly string[]).includes(obj.type) + ); +} diff --git a/dexto/packages/core/src/tools/error-codes.ts b/dexto/packages/core/src/tools/error-codes.ts new file mode 100644 index 00000000..122d8698 --- /dev/null +++ b/dexto/packages/core/src/tools/error-codes.ts @@ -0,0 +1,33 @@ +/** + * Tools-specific error codes + * Includes tool execution, confirmation, and authorization errors + */ +export enum ToolErrorCode { + // Execution + EXECUTION_DENIED = 'tools_execution_denied', + EXECUTION_TIMEOUT = 'tools_execution_timeout', + EXECUTION_FAILED = 'tools_execution_failed', + DIRECTORY_ACCESS_DENIED = 'tools_directory_access_denied', + + // Validation (pre-execution) + VALIDATION_FAILED = 'tools_validation_failed', + FILE_MODIFIED_SINCE_PREVIEW = 'tools_file_modified_since_preview', + + // Confirmation + CONFIRMATION_HANDLER_MISSING = 'tools_confirmation_handler_missing', + CONFIRMATION_TIMEOUT = 'tools_confirmation_timeout', + CONFIRMATION_CANCELLED = 'tools_confirmation_cancelled', + + // Tool management + TOOL_NOT_FOUND = 'tools_tool_not_found', + TOOL_INVALID_ARGS = 'tools_invalid_args', + TOOL_UNAUTHORIZED = 'tools_unauthorized', + + // Configuration + CONFIG_INVALID = 'tools_config_invalid', + FEATURE_DISABLED = 'tools_feature_disabled', + + // Custom tool provider registry + CUSTOM_TOOL_PROVIDER_UNKNOWN = 'tools_custom_provider_unknown', + CUSTOM_TOOL_PROVIDER_ALREADY_REGISTERED = 'tools_custom_provider_already_registered', +} diff --git a/dexto/packages/core/src/tools/errors.ts b/dexto/packages/core/src/tools/errors.ts new file mode 100644 index 00000000..64d57d83 --- /dev/null +++ b/dexto/packages/core/src/tools/errors.ts @@ -0,0 +1,261 @@ +import { DextoRuntimeError } from '@core/errors/DextoRuntimeError.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; +import { ToolErrorCode } from './error-codes.js'; + +/** + * Tool error factory with typed methods for creating tool-specific errors + * Each method creates a properly typed DextoError with TOOLS scope + */ +export class ToolError { + /** + * Tool not found error + */ + static notFound(toolName: string) { + return new DextoRuntimeError( + ToolErrorCode.TOOL_NOT_FOUND, + ErrorScope.TOOLS, + ErrorType.NOT_FOUND, + `Tool '${toolName}' not found`, + { toolName } + ); + } + + /** + * Tool execution failed + */ + static executionFailed(toolName: string, reason: string, sessionId?: string) { + return new DextoRuntimeError( + ToolErrorCode.EXECUTION_FAILED, + ErrorScope.TOOLS, + ErrorType.SYSTEM, + `Tool '${toolName}' execution failed: ${reason}`, + { toolName, reason, sessionId } + ); + } + + /** + * Tool execution denied by user/policy + * @param toolName - Name of the tool that was denied + * @param sessionId - Optional session ID + * @param userMessage - Optional message from user (e.g., feedback for plan review) + */ + static executionDenied(toolName: string, sessionId?: string, userMessage?: string) { + const message = userMessage + ? `Tool '${toolName}' was denied. ${userMessage}` + : `Tool '${toolName}' execution was denied by the user`; + return new DextoRuntimeError( + ToolErrorCode.EXECUTION_DENIED, + ErrorScope.TOOLS, + ErrorType.FORBIDDEN, + message, + { toolName, sessionId, userMessage } + ); + } + + /** + * Directory access denied by user + * Used when a file tool tries to access a path outside allowed directories + * and the user denies the directory access approval + */ + static directoryAccessDenied(directory: string, sessionId?: string) { + return new DextoRuntimeError( + ToolErrorCode.DIRECTORY_ACCESS_DENIED, + ErrorScope.TOOLS, + ErrorType.FORBIDDEN, + `Access to directory '${directory}' was denied`, + { directory, sessionId }, + 'Request access to the directory or work within the allowed working directory' + ); + } + + /** + * Tool execution timeout + */ + static executionTimeout(toolName: string, timeoutMs: number, sessionId?: string) { + const message = + timeoutMs > 0 + ? `Tool '${toolName}' execution timed out after ${timeoutMs}ms` + : `Tool '${toolName}' execution timed out`; + return new DextoRuntimeError( + ToolErrorCode.EXECUTION_TIMEOUT, + ErrorScope.TOOLS, + ErrorType.TIMEOUT, + message, + { toolName, timeoutMs, sessionId } + ); + } + + /** + * Tool validation failed (pre-execution check) + * Used when tool inputs are semantically invalid (e.g., file not found, string not in file) + * This should fail before approval, not after. + */ + static validationFailed(toolName: string, reason: string, context?: Record) { + return new DextoRuntimeError( + ToolErrorCode.VALIDATION_FAILED, + ErrorScope.TOOLS, + ErrorType.USER, + `Tool '${toolName}' validation failed: ${reason}`, + { toolName, reason, ...context } + ); + } + + /** + * File was modified between preview and execute. + * This is a safety check to prevent corrupting user edits. + */ + static fileModifiedSincePreview(toolName: string, filePath: string) { + return new DextoRuntimeError( + ToolErrorCode.FILE_MODIFIED_SINCE_PREVIEW, + ErrorScope.TOOLS, + ErrorType.USER, + `File '${filePath}' was modified since the preview was generated. Please read the file again and retry the operation.`, + { + toolName, + filePath, + recovery: + 'Read the file with read_file tool to get current content, then retry the edit.', + } + ); + } + + /** + * Tool unauthorized access + */ + static unauthorized(toolName: string, sessionId?: string) { + return new DextoRuntimeError( + ToolErrorCode.TOOL_UNAUTHORIZED, + ErrorScope.TOOLS, + ErrorType.FORBIDDEN, + `Unauthorized access to tool '${toolName}'`, + { toolName, sessionId } + ); + } + + /** + * Confirmation handler missing + */ + static confirmationHandlerMissing(toolName: string) { + return new DextoRuntimeError( + ToolErrorCode.CONFIRMATION_HANDLER_MISSING, + ErrorScope.TOOLS, + ErrorType.SYSTEM, + `Confirmation handler missing for tool '${toolName}'`, + { toolName } + ); + } + + /** + * Confirmation timeout + */ + static confirmationTimeout(toolName: string, timeoutMs: number, sessionId?: string) { + return new DextoRuntimeError( + ToolErrorCode.CONFIRMATION_TIMEOUT, + ErrorScope.TOOLS, + ErrorType.TIMEOUT, + `Tool '${toolName}' confirmation timed out after ${timeoutMs}ms`, + { toolName, timeoutMs, sessionId } + ); + } + + /** + * Invalid tool name (semantic validation - empty name after prefix, malformed, etc.) + */ + static invalidName(toolName: string, reason: string) { + return new DextoRuntimeError( + ToolErrorCode.TOOL_INVALID_ARGS, + ErrorScope.TOOLS, + ErrorType.USER, + `Invalid tool name '${toolName}': ${reason}`, + { toolName, reason } + ); + } + + /** + * Internal tools provider not initialized + */ + static internalToolsNotInitialized(toolName: string) { + return new DextoRuntimeError( + ToolErrorCode.EXECUTION_FAILED, + ErrorScope.TOOLS, + ErrorType.SYSTEM, + `Internal tools not initialized, cannot execute: ${toolName}`, + { toolName } + ); + } + + /** + * Invalid tool configuration + */ + static configInvalid(message: string) { + return new DextoRuntimeError( + ToolErrorCode.CONFIG_INVALID, + ErrorScope.TOOLS, + ErrorType.USER, + message, + {} + ); + } + + /** + * Confirmation cancelled + */ + static confirmationCancelled(toolName: string, reason: string) { + return new DextoRuntimeError( + ToolErrorCode.CONFIRMATION_CANCELLED, + ErrorScope.TOOLS, + ErrorType.USER, + `Tool confirmation for '${toolName}' was cancelled: ${reason}`, + { toolName, reason } + ); + } + + /** + * Tool requires features which are currently disabled + */ + static featureDisabled( + toolName: string, + missingFeatures: string[], + message: string + ): DextoRuntimeError<{ toolName: string; missingFeatures: string[] }> { + return new DextoRuntimeError( + ToolErrorCode.FEATURE_DISABLED, + ErrorScope.TOOLS, + ErrorType.USER, + message, + { toolName, missingFeatures }, + [ + `Remove '${toolName}' from internalTools in your agent config`, + `Or enable required features: ${missingFeatures.map((f) => `${f}.enabled: true`).join(', ')}`, + ] + ); + } + + /** + * Unknown custom tool provider type + */ + static unknownCustomToolProvider(type: string, availableTypes: string[]): DextoRuntimeError { + return new DextoRuntimeError( + ToolErrorCode.CUSTOM_TOOL_PROVIDER_UNKNOWN, + ErrorScope.TOOLS, + ErrorType.USER, + `Unknown custom tool provider: '${type}'`, + { type, availableTypes }, + `Available types: ${availableTypes.length > 0 ? availableTypes.join(', ') : 'none'}` + ); + } + + /** + * Custom tool provider already registered + */ + static customToolProviderAlreadyRegistered(type: string): DextoRuntimeError { + return new DextoRuntimeError( + ToolErrorCode.CUSTOM_TOOL_PROVIDER_ALREADY_REGISTERED, + ErrorScope.TOOLS, + ErrorType.USER, + `Custom tool provider '${type}' is already registered`, + { type }, + `Use unregister() first if you want to replace it` + ); + } +} diff --git a/dexto/packages/core/src/tools/index.ts b/dexto/packages/core/src/tools/index.ts new file mode 100644 index 00000000..f331a1e3 --- /dev/null +++ b/dexto/packages/core/src/tools/index.ts @@ -0,0 +1,36 @@ +/** + * Tools System for Dexto + * + * This module provides the unified tool management system that handles + * MCP servers and internal tools. + */ + +// Core types and interfaces +export * from './types.js'; + +// Display types for tool result rendering +export * from './display-types.js'; + +// Internal tools provider and types +export * from './internal-tools/index.js'; + +// Custom tool registry and provider interface +export { + customToolRegistry, + type CustomToolProvider, + type CustomToolRegistry, + type ToolCreationContext, +} from './custom-tool-registry.js'; + +// Custom tool schema registry for early validation +export { customToolSchemaRegistry } from './custom-tool-schema-registry.js'; + +// Schemas/types +export * from './schemas.js'; + +// Tool errors and error codes +export { ToolError } from './errors.js'; +export { ToolErrorCode } from './error-codes.js'; + +// Unified tool manager (main interface for LLM) +export { ToolManager, type InternalToolsOptions } from './tool-manager.js'; diff --git a/dexto/packages/core/src/tools/internal-tools/constants.ts b/dexto/packages/core/src/tools/internal-tools/constants.ts new file mode 100644 index 00000000..4354202a --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/constants.ts @@ -0,0 +1,22 @@ +/** + * Internal tool constants + * + * Separated from registry to avoid circular dependencies and browser bundle pollution + */ + +// TODO: Update docs/docs/guides/configuring-dexto/internalTools.md to reflect these new tools. + +/** + * Available internal tool names + * Must be kept in sync with INTERNAL_TOOL_REGISTRY in registry.ts + */ +export const INTERNAL_TOOL_NAMES = [ + 'search_history', + 'ask_user', + 'delegate_to_url', + 'list_resources', + 'get_resource', + 'invoke_skill', +] as const; + +export type KnownInternalTool = (typeof INTERNAL_TOOL_NAMES)[number]; diff --git a/dexto/packages/core/src/tools/internal-tools/implementations/ask-user-tool.ts b/dexto/packages/core/src/tools/internal-tools/implementations/ask-user-tool.ts new file mode 100644 index 00000000..12474aeb --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/implementations/ask-user-tool.ts @@ -0,0 +1,68 @@ +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext } from '../../types.js'; +import { ApprovalManager } from '../../../approval/manager.js'; + +const AskUserInputSchema = z + .object({ + question: z.string().describe('The question or prompt to display to the user'), + schema: z + .object({ + type: z.literal('object'), + properties: z.record(z.unknown()), + required: z.array(z.string()).optional(), + }) + .passthrough() + .describe( + 'JSON Schema defining form fields. Use descriptive property names as labels (e.g., "favorite_team", "World Cup winner country") - NOT generic names like "q1". Use "enum" for dropdowns, "boolean" for yes/no, "number" for numeric inputs, "string" for text. Include "required" array for mandatory fields.' + ), + }) + .strict(); + +type AskUserInput = z.input; + +/** + * Internal tool for asking the user questions during agent execution + * Leverages the ApprovalManager to prompt the user cross-platform (CLI, WebUI) + * + * Usage distinction: + * - ask_user tool: Agent-initiated form-based input requests during task execution + * (e.g., agent decides it needs specific information to complete a task) + * - MCP elicitation: Server-initiated input requests from external MCP servers + * (e.g., MCP server requires configuration or authentication data) + * + * Both use ApprovalManager.requestElicitation() under the hood but serve different purposes: + * - ask_user: Part of agent's internal reasoning and task workflow + * - MCP elicitation: External server requirements for tool/resource access + */ +export function createAskUserTool(approvalManager: ApprovalManager): InternalTool { + return { + id: 'ask_user', + description: + 'Collect structured input from the user through a form interface. ONLY use this tool when you need: 1) Multiple fields at once (e.g., name + email + preferences), 2) Pre-defined options/choices (use enum for dropdowns like ["small","medium","large"]), 3) Specific data types with validation (boolean for yes/no, number for quantities). DO NOT use for simple conversational questions - just ask those naturally in your response. This tool is for form-like data collection, not chat. Examples: collecting user profile info, configuration settings, or selecting from preset options.', + inputSchema: AskUserInputSchema, + execute: async (input: unknown, context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { question, schema } = input as AskUserInput; + + // Build elicitation request + const elicitationRequest: { + schema: Record; + prompt: string; + serverName: string; + sessionId?: string; + } = { + schema, + prompt: question, + serverName: 'Dexto Agent', + }; + + // Add sessionId if available + if (context?.sessionId !== undefined) { + elicitationRequest.sessionId = context.sessionId; + } + + // Delegate to shared helper for typed errors and consistent logic + return approvalManager.getElicitationData(elicitationRequest); + }, + }; +} diff --git a/dexto/packages/core/src/tools/internal-tools/implementations/delegate-to-url-tool.test.ts b/dexto/packages/core/src/tools/internal-tools/implementations/delegate-to-url-tool.test.ts new file mode 100644 index 00000000..2ce04c0d --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/implementations/delegate-to-url-tool.test.ts @@ -0,0 +1,338 @@ +import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; +import { createDelegateToUrlTool } from './delegate-to-url-tool.js'; +import { DextoRuntimeError } from '../../../errors/DextoRuntimeError.js'; + +// Mock fetch globally +const mockFetch = vi.fn(); +global.fetch = mockFetch as any; + +describe('delegate_to_url tool', () => { + let tool: ReturnType; + + beforeEach(() => { + tool = createDelegateToUrlTool(); + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('Tool Definition', () => { + it('should have correct tool metadata', () => { + expect(tool.id).toBe('delegate_to_url'); + expect(tool.description).toBeDefined(); + expect(tool.description.length).toBeGreaterThan(0); + expect(tool.inputSchema).toBeDefined(); + }); + + it('should have required input schema fields', () => { + const schema = tool.inputSchema as any; // Zod schema shape checking + expect(schema.shape.url).toBeDefined(); + expect(schema.shape.message).toBeDefined(); + expect(schema.shape.sessionId).toBeDefined(); + expect(schema.shape.timeout).toBeDefined(); + }); + }); + + describe('Input Validation', () => { + it('should accept valid input', () => { + const validInput = { + url: 'http://localhost:3001', + message: 'Test message', + }; + + const result = tool.inputSchema.safeParse(validInput); + expect(result.success).toBe(true); + }); + + it('should reject invalid URL', () => { + const invalidInput = { + url: 'not-a-url', + message: 'Test message', + }; + + const result = tool.inputSchema.safeParse(invalidInput); + expect(result.success).toBe(false); + }); + + it('should reject empty message', () => { + const invalidInput = { + url: 'http://localhost:3001', + message: '', + }; + + const result = tool.inputSchema.safeParse(invalidInput); + expect(result.success).toBe(false); + }); + + it('should accept optional sessionId', () => { + const validInput = { + url: 'http://localhost:3001', + message: 'Test message', + sessionId: 'session-123', + }; + + const result = tool.inputSchema.safeParse(validInput); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.sessionId).toBe('session-123'); + } + }); + + it('should set default timeout', () => { + const input = { + url: 'http://localhost:3001', + message: 'Test message', + }; + + const result = tool.inputSchema.safeParse(input); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.timeout).toBe(30000); + } + }); + }); + + describe('Successful Delegation', () => { + it('should delegate message and return response', async () => { + // Mock successful A2A response + const mockResponse = { + jsonrpc: '2.0', + id: 'test-id', + result: { + id: 'task-123', + contextId: 'context-123', + status: { + state: 'completed', + }, + history: [ + { + role: 'user', + parts: [{ kind: 'text', text: 'Test message' }], + }, + { + role: 'agent', + parts: [{ kind: 'text', text: 'Response from delegated agent' }], + }, + ], + kind: 'task', + }, + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockResponse, + }); + + const result = (await tool.execute({ + url: 'http://localhost:3001', + message: 'Test message', + })) as any; + + expect(result.success).toBe(true); + expect(result.agentUrl).toBe('http://localhost:3001'); + expect(result.response).toBe('Response from delegated agent'); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('should include sessionId in response when provided', async () => { + const mockResponse = { + jsonrpc: '2.0', + id: 'test-id', + result: { + id: 'task-123', + contextId: 'context-123', + status: { state: 'completed' }, + history: [ + { + role: 'agent', + parts: [{ kind: 'text', text: 'Response' }], + }, + ], + kind: 'task', + }, + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockResponse, + }); + + const result = (await tool.execute({ + url: 'http://localhost:3001', + message: 'Test message', + sessionId: 'session-456', + })) as any; + + expect(result.sessionId).toBe('session-456'); + }); + + it('should try multiple endpoints', async () => { + // First endpoint fails + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 404, + statusText: 'Not Found', + }); + + // Second endpoint succeeds + const mockResponse = { + jsonrpc: '2.0', + id: 'test-id', + result: { + id: 'task-123', + history: [ + { + role: 'agent', + parts: [{ kind: 'text', text: 'Success' }], + }, + ], + }, + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockResponse, + }); + + const result = (await tool.execute({ + url: 'http://localhost:3001', + message: 'Test message', + })) as any; + + expect(result.success).toBe(true); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + }); + + describe('Error Handling', () => { + it('should throw DextoRuntimeError on timeout', async () => { + // Mock fetch to simulate AbortError (what fetch does when aborted) + mockFetch.mockRejectedValue( + Object.assign(new Error('The operation was aborted'), { name: 'AbortError' }) + ); + + await expect( + tool.execute({ + url: 'http://localhost:3001', + message: 'Test message', + timeout: 100, // Very short timeout + }) + ).rejects.toThrow(DextoRuntimeError); + }); + + it('should throw DextoRuntimeError when all endpoints fail', async () => { + // All endpoints return errors + mockFetch.mockResolvedValue({ + ok: false, + status: 500, + statusText: 'Internal Server Error', + }); + + await expect( + tool.execute({ + url: 'http://localhost:3001', + message: 'Test message', + }) + ).rejects.toThrow(DextoRuntimeError); + + // Should try both endpoints + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('should handle JSON-RPC error responses', async () => { + const errorResponse = { + jsonrpc: '2.0', + id: 'test-id', + error: { + code: -32603, + message: 'Internal error', + }, + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => errorResponse, + }); + + // Should fail on first endpoint with error, try second endpoint + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 500, + statusText: 'Internal Server Error', + }); + + // Expect DextoRuntimeError to be thrown (error gets caught and retried on other endpoints) + await expect( + tool.execute({ + url: 'http://localhost:3001', + message: 'Test message', + }) + ).rejects.toThrow(DextoRuntimeError); + }); + + it('should handle network errors', async () => { + mockFetch.mockRejectedValue(new Error('Network error')); + + await expect( + tool.execute({ + url: 'http://localhost:3001', + message: 'Test message', + }) + ).rejects.toThrow(DextoRuntimeError); + }); + }); + + describe('URL Handling', () => { + it('should handle URLs with trailing slash', async () => { + const mockResponse = { + jsonrpc: '2.0', + id: 'test-id', + result: { + history: [ + { + role: 'agent', + parts: [{ kind: 'text', text: 'Success' }], + }, + ], + }, + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockResponse, + }); + + await tool.execute({ + url: 'http://localhost:3001/', + message: 'Test message', + }); + + // Check that fetch was called with correct endpoint (no double slash) + const fetchCall = mockFetch.mock.calls[0]?.[0]; + expect(fetchCall).not.toContain('//v1'); + expect(fetchCall).toContain('/v1/jsonrpc'); + }); + + it('should construct correct endpoint URLs', async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + jsonrpc: '2.0', + result: { + history: [{ role: 'agent', parts: [{ kind: 'text', text: 'ok' }] }], + }, + }), + }); + + await tool.execute({ + url: 'http://localhost:3001', + message: 'Test', + }); + + const firstCall = mockFetch.mock.calls[0]?.[0]; + expect(firstCall).toBe('http://localhost:3001/v1/jsonrpc'); + }); + }); +}); diff --git a/dexto/packages/core/src/tools/internal-tools/implementations/delegate-to-url-tool.ts b/dexto/packages/core/src/tools/internal-tools/implementations/delegate-to-url-tool.ts new file mode 100644 index 00000000..85acc436 --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/implementations/delegate-to-url-tool.ts @@ -0,0 +1,284 @@ +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext } from '../../types.js'; +import { DextoRuntimeError, ErrorScope, ErrorType } from '../../../errors/index.js'; + +/** + * Input schema for delegate_to_url tool + */ +const DelegateToUrlInputSchema = z + .object({ + url: z + .string() + .url() + .describe( + 'The A2A-compliant agent URL (e.g., "http://localhost:3001" or "https://agent.example.com"). The tool will automatically append the correct JSON-RPC endpoint.' + ), + message: z + .string() + .min(1) + .describe( + 'The message or task to delegate to the agent. This will be sent as natural language input.' + ), + sessionId: z + .string() + .optional() + .describe( + 'Optional session ID for maintaining conversation state across multiple delegations to the same agent' + ), + timeout: z + .number() + .optional() + .default(30000) + .describe('Request timeout in milliseconds (default: 30000)'), + }) + .strict(); + +type DelegateToUrlInput = z.output; + +/** + * A2A JSON-RPC 2.0 message structure + */ +interface A2AMessage { + role: 'user' | 'agent'; + parts: Array<{ + kind: 'text'; + text: string; + metadata?: Record; + }>; + messageId: string; + taskId?: string; + contextId?: string; + kind: 'message'; +} + +/** + * Simple A2A client for basic delegation + * Implements A2A Protocol v0.3.0 JSON-RPC transport + */ +class SimpleA2AClient { + private url: string; + private timeout: number; + + constructor(url: string, timeout: number = 30000) { + // Ensure URL doesn't have trailing slash + this.url = url.replace(/\/$/, ''); + this.timeout = timeout; + } + + /** + * Send a message to the A2A agent using JSON-RPC 2.0 + */ + async sendMessage(message: string, sessionId?: string): Promise { + // Generate IDs + const messageId = this.generateId(); + const taskId = sessionId || this.generateId(); + + // Build A2A message structure + const a2aMessage: A2AMessage = { + role: 'user', + parts: [ + { + kind: 'text', + text: message, + }, + ], + messageId, + taskId, + contextId: taskId, + kind: 'message', + }; + + // Build JSON-RPC 2.0 request + const rpcRequest = { + jsonrpc: '2.0', + id: this.generateId(), + method: 'message/send', + params: { + message: a2aMessage, + configuration: { + blocking: true, // Wait for completion + }, + }, + }; + + // Determine endpoint (try JSON-RPC first, fallback to legacy) + const endpoints = [ + `${this.url}/v1/jsonrpc`, // A2A v0.3.0 JSON-RPC endpoint + `${this.url}/jsonrpc`, // Alternative path + ]; + + let lastError: Error | null = null; + + // Try each endpoint + for (const endpoint of endpoints) { + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), this.timeout); + + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'User-Agent': '@dexto/core', + }, + body: JSON.stringify(rpcRequest), + signal: controller.signal, + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + lastError = new Error( + `HTTP ${response.status}: ${response.statusText} (tried ${endpoint})` + ); + continue; // Try next endpoint + } + + const data = await response.json(); + + // Check for JSON-RPC error + if ('error' in data && data.error) { + throw new Error( + `Agent returned error: ${data.error.message || 'Unknown error'}` + ); + } + + // Extract result from JSON-RPC response + if ('result' in data) { + return this.extractTaskResponse(data.result); + } + + // Return raw data if not standard JSON-RPC + return data; + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + throw new DextoRuntimeError( + `Delegation timeout after ${this.timeout}ms`, + ErrorScope.TOOLS, + ErrorType.TIMEOUT, + 'DELEGATION_TIMEOUT' + ); + } + lastError = error instanceof Error ? error : new Error(String(error)); + // Continue to next endpoint + } + } + + // All endpoints failed + throw new DextoRuntimeError( + `Failed to connect to agent at ${this.url}. Tried endpoints: ${endpoints.join(', ')}. Last error: ${lastError?.message || 'Unknown error'}`, + ErrorScope.TOOLS, + ErrorType.THIRD_PARTY, + 'DELEGATION_FAILED' + ); + } + + /** + * Extract response text from A2A Task structure + */ + private extractTaskResponse(task: any): string { + // Extract from history (get last agent message) + if (task.history && Array.isArray(task.history)) { + const agentMessages = task.history.filter((m: any) => m.role === 'agent'); + if (agentMessages.length > 0) { + const lastMessage = agentMessages[agentMessages.length - 1]; + if (lastMessage.parts && Array.isArray(lastMessage.parts)) { + const textParts = lastMessage.parts + .filter((p: any) => p.kind === 'text') + .map((p: any) => p.text); + if (textParts.length > 0) { + return textParts.join('\n'); + } + } + } + } + + // Fallback: return stringified task + return JSON.stringify(task, null, 2); + } + + /** + * Generate unique ID + */ + private generateId(): string { + return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`; + } +} + +/** + * Internal tool for agent-to-agent delegation + * + * Delegates tasks to other A2A-compliant agents at known URLs. + * Supports stateful multi-turn conversations via sessionId parameter. + * + * Features: + * - A2A Protocol v0.3.0 compliant (JSON-RPC 2.0 transport) + * - Stateful conversations with session management + * - Automatic endpoint discovery (/v1/jsonrpc, /jsonrpc) + * - Configurable timeouts and error handling + * + * @example + * ```typescript + * // Single delegation + * delegate_to_url({ + * url: "http://pdf-analyzer:3001", + * message: "Extract all tables from the Q4 sales report" + * }) + * → Returns: {sessionId: "delegation-xyz", response: "..."} + * + * // Multi-turn conversation (use same sessionId) + * delegate_to_url({ + * url: "http://pdf-analyzer:3001", + * message: "Which table had the highest values?", + * sessionId: "delegation-xyz" + * }) + * → Agent remembers the previous extraction and can answer specifically + * ``` + */ +export function createDelegateToUrlTool(): InternalTool { + return { + id: 'delegate_to_url', + description: + 'Delegate a task to another A2A-compliant agent at a specific URL. Supports STATEFUL multi-turn conversations via sessionId parameter. USAGE: (1) First delegation: provide url + message. Tool returns a response AND a sessionId. (2) Follow-up: use the SAME sessionId to continue the conversation with that agent. The agent remembers previous context. EXAMPLE: First call {url: "http://agent:3001", message: "Analyze data X"} returns {sessionId: "xyz", response: "..."}. Second call {url: "http://agent:3001", message: "What was the top insight?", sessionId: "xyz"}. The agent will remember the first analysis and can answer specifically.', + inputSchema: DelegateToUrlInputSchema, + execute: async (input: unknown, _context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { url, message, sessionId, timeout } = input as DelegateToUrlInput; + + try { + // Create client and send message + const client = new SimpleA2AClient(url, timeout); + + // Use provided sessionId or generate new one for conversation tracking + const effectiveSessionId = + sessionId || + `delegation-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; + const response = await client.sendMessage(message, effectiveSessionId); + + // Return formatted response with sessionId for conversation resumption + return { + success: true, + agentUrl: url, + sessionId: effectiveSessionId, + response, + _hint: sessionId + ? 'Continued existing conversation' + : 'Started new conversation - use this sessionId for follow-ups', + }; + } catch (error) { + // Re-throw DextoRuntimeError as-is + if (error instanceof DextoRuntimeError) { + throw error; + } + + // Wrap other errors + throw new DextoRuntimeError( + `Delegation failed: ${error instanceof Error ? error.message : String(error)}`, + ErrorScope.TOOLS, + ErrorType.SYSTEM, + 'DELEGATION_ERROR' + ); + } + }, + }; +} diff --git a/dexto/packages/core/src/tools/internal-tools/implementations/get-resource-tool.ts b/dexto/packages/core/src/tools/internal-tools/implementations/get-resource-tool.ts new file mode 100644 index 00000000..bb12b754 --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/implementations/get-resource-tool.ts @@ -0,0 +1,161 @@ +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext } from '../../types.js'; +import type { ResourceManager } from '../../../resources/manager.js'; + +/** + * Input schema for get_resource tool + */ +const GetResourceInputSchema = z + .object({ + reference: z + .string() + .describe( + 'The resource reference to access. Formats: "blob:abc123" (from list_resources), ' + + '"resource_ref:blob:abc123" (from tool annotations)' + ), + format: z + .enum(['url', 'metadata']) + .default('url') + .describe( + 'Output format: "url" for a shareable URL (requires remote storage like Supabase), ' + + '"metadata" for resource information without loading the data' + ), + }) + .strict(); + +type GetResourceInput = z.output; + +/** + * Internal tool for accessing resources. + * + * This tool provides access to stored resources (images, files, etc.) in formats + * suitable for sharing or inspection, without loading binary data into context. + * + * Formats: + * - `url`: Returns a shareable URL. Requires remote storage (e.g., Supabase). + * Local/memory storage does not support URL generation. + * - `metadata`: Returns resource information (size, mimeType, filename, etc.) + * without loading the actual data. + * + * Design principle: Resources exist to keep binary data OUT of the context window. + * This tool never returns base64 or raw data - use URLs for sharing instead. + * + * @example + * ```typescript + * // Get a shareable URL for an image + * get_resource({ reference: 'blob:abc123', format: 'url' }) + * → { success: true, url: 'https://...', mimeType: 'image/png', ... } + * + * // Get metadata about a resource + * get_resource({ reference: 'blob:abc123', format: 'metadata' }) + * → { success: true, mimeType: 'image/png', size: 12345, filename: '...', ... } + * ``` + */ +export function createGetResourceTool(resourceManager: ResourceManager): InternalTool { + return { + id: 'get_resource', + description: + 'Access a stored resource. Use format "url" to get a shareable URL for other agents ' + + 'or external systems (requires remote storage like Supabase). Use format "metadata" ' + + 'to get resource information without loading data. ' + + 'References can be obtained from tool result annotations or list_resources.', + inputSchema: GetResourceInputSchema, + execute: async (input: unknown, _context?: ToolExecutionContext) => { + const { reference, format } = input as GetResourceInput; + + try { + const blobStore = resourceManager.getBlobStore(); + const storeType = blobStore.getStoreType(); + + // Normalize the reference - handle various formats: + // - "resource_ref:blob:abc123" (from tool annotations) + // - "blob:abc123" (from list_resources) + // - "abc123" (just the ID) + let blobUri = reference; + + // Strip resource_ref: prefix if present + if (blobUri.startsWith('resource_ref:')) { + blobUri = blobUri.substring('resource_ref:'.length); + } + + // Strip @ prefix if present (legacy) + if (blobUri.startsWith('@')) { + blobUri = blobUri.substring(1); + } + + // Ensure it starts with blob: + if (!blobUri.startsWith('blob:')) { + blobUri = `blob:${blobUri}`; + } + + // Check if blob exists + const exists = await blobStore.exists(blobUri); + if (!exists) { + return { + success: false, + error: `Resource not found: ${reference}`, + _hint: 'Use list_resources to see available resources', + }; + } + + // Handle format: metadata + if (format === 'metadata') { + // Get metadata without loading blob data by using listBlobs() + const allBlobs = await blobStore.listBlobs(); + const blobRef = allBlobs.find((b) => b.uri === blobUri); + + if (!blobRef) { + return { + success: false, + error: `Resource metadata not found: ${reference}`, + _hint: 'Use list_resources to see available resources', + }; + } + + return { + success: true, + format: 'metadata', + reference: blobUri, + mimeType: blobRef.metadata.mimeType, + size: blobRef.metadata.size, + filename: blobRef.metadata.originalName, + source: blobRef.metadata.source, + createdAt: blobRef.metadata.createdAt.toISOString(), + }; + } + + // Handle format: url + // URL generation only supported for remote stores + if (storeType === 'memory' || storeType === 'local') { + return { + success: false, + error: 'URL generation not available with local/memory storage', + _hint: + 'Configure remote storage (e.g., Supabase) in your agent config to enable ' + + 'URL sharing. Local storage cannot generate shareable URLs.', + storeType, + }; + } + + // For Supabase and other remote stores, generate URL + const blob = await blobStore.retrieve(blobUri, 'url'); + + return { + success: true, + format: 'url', + url: blob.data, + reference: blobUri, + mimeType: blob.metadata.mimeType, + size: blob.metadata.size, + filename: blob.metadata.originalName, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + success: false, + error: `Failed to access resource: ${message}`, + }; + } + }, + }; +} diff --git a/dexto/packages/core/src/tools/internal-tools/implementations/invoke-skill-tool.test.ts b/dexto/packages/core/src/tools/internal-tools/implementations/invoke-skill-tool.test.ts new file mode 100644 index 00000000..24385866 --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/implementations/invoke-skill-tool.test.ts @@ -0,0 +1,388 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { createInvokeSkillTool } from './invoke-skill-tool.js'; +import type { PromptManager } from '../../../prompts/prompt-manager.js'; +import type { InternalToolsServices, TaskForker } from '../registry.js'; + +describe('invoke_skill tool', () => { + let tool: ReturnType; + let mockPromptManager: Partial; + let services: InternalToolsServices; + + const mockAutoInvocablePrompts = { + 'config:plugin:skill-one': { + name: 'config:plugin:skill-one', + displayName: 'plugin:skill-one', + description: 'First skill', + }, + 'config:simple-skill': { + name: 'config:simple-skill', + displayName: 'simple-skill', + description: 'Simple skill without namespace', + }, + }; + + beforeEach(() => { + mockPromptManager = { + listAutoInvocablePrompts: vi.fn().mockResolvedValue(mockAutoInvocablePrompts), + // GetPromptResult structure: { messages: [{ content: { type: 'text', text: '...' } }] } + getPrompt: vi.fn().mockResolvedValue({ + messages: [{ content: { type: 'text', text: 'Skill instructions here' } }], + }), + // Return undefined for context (inline execution is default) + getPromptDefinition: vi.fn().mockResolvedValue(undefined), + }; + + services = { promptManager: mockPromptManager as PromptManager }; + tool = createInvokeSkillTool(services); + }); + + describe('Tool Definition', () => { + it('should have correct tool metadata', () => { + expect(tool.id).toBe('invoke_skill'); + expect(tool.description).toBeDefined(); + expect(tool.description.length).toBeGreaterThan(0); + expect(tool.inputSchema).toBeDefined(); + }); + + it('should have required input schema fields', () => { + const schema = tool.inputSchema as any; + expect(schema.shape.skill).toBeDefined(); + expect(schema.shape.args).toBeDefined(); + }); + + it('should have helpful description', () => { + expect(tool.description).toContain('skill'); + expect(tool.description).toContain('instructions'); + }); + }); + + describe('Input Validation', () => { + it('should accept valid input with just skill name', () => { + const validInput = { + skill: 'skill-one', + }; + + const result = tool.inputSchema.safeParse(validInput); + expect(result.success).toBe(true); + }); + + it('should accept valid input with skill name and args', () => { + const validInput = { + skill: 'skill-one', + args: { key: 'value' }, + }; + + const result = tool.inputSchema.safeParse(validInput); + expect(result.success).toBe(true); + }); + + it('should accept valid input with taskContext', () => { + const validInput = { + skill: 'skill-one', + taskContext: 'User wants to accomplish X', + }; + + const result = tool.inputSchema.safeParse(validInput); + expect(result.success).toBe(true); + }); + + it('should reject empty skill name', () => { + const invalidInput = { + skill: '', + }; + + const result = tool.inputSchema.safeParse(invalidInput); + expect(result.success).toBe(false); + }); + + it('should reject missing skill name', () => { + const invalidInput = {}; + + const result = tool.inputSchema.safeParse(invalidInput); + expect(result.success).toBe(false); + }); + }); + + describe('Skill Lookup', () => { + it('should find skill by full key', async () => { + const result = (await tool.execute({ + skill: 'config:plugin:skill-one', + })) as any; + + expect(result.skill).toBe('config:plugin:skill-one'); + expect(result.content).toContain('Skill instructions here'); + expect(mockPromptManager.getPrompt).toHaveBeenCalledWith( + 'config:plugin:skill-one', + undefined + ); + }); + + it('should find skill by displayName', async () => { + const result = (await tool.execute({ + skill: 'plugin:skill-one', + })) as any; + + expect(result.skill).toBe('config:plugin:skill-one'); + expect(result.content).toContain('Skill instructions here'); + }); + + it('should find skill by name', async () => { + const result = (await tool.execute({ + skill: 'simple-skill', + })) as any; + + // Should match config:simple-skill by checking `info.name === 'config:${skill}'` + expect(result.skill).toBe('config:simple-skill'); + }); + + it('should pass args to getPrompt', async () => { + const args = { format: 'json', verbose: 'true' }; + + await tool.execute({ + skill: 'plugin:skill-one', + args, + }); + + expect(mockPromptManager.getPrompt).toHaveBeenCalledWith( + 'config:plugin:skill-one', + args + ); + }); + }); + + describe('Error Handling', () => { + it('should return error for unknown skill', async () => { + const result = (await tool.execute({ + skill: 'nonexistent-skill', + })) as any; + + expect(result.error).toBeDefined(); + expect(result.error).toContain('not found'); + expect(result.availableSkills).toEqual([ + 'config:plugin:skill-one', + 'config:simple-skill', + ]); + }); + + it('should include available skills in error response', async () => { + const result = (await tool.execute({ + skill: 'unknown', + })) as any; + + expect(result.availableSkills).toBeDefined(); + expect(Array.isArray(result.availableSkills)).toBe(true); + expect(result.availableSkills.length).toBe(2); + }); + }); + + describe('Successful Invocation', () => { + it('should return skill content and instructions', async () => { + const result = (await tool.execute({ + skill: 'simple-skill', + })) as any; + + expect(result.skill).toBeDefined(); + expect(result.content).toBeDefined(); + expect(result.instructions).toBeDefined(); + expect(result.instructions).toContain('Follow the instructions'); + }); + + it('should handle array of prompt content', async () => { + mockPromptManager.getPrompt = vi.fn().mockResolvedValue({ + messages: [ + { content: { type: 'text', text: 'Part 1' } }, + { content: { type: 'text', text: 'Part 2' } }, + ], + }); + + const result = (await tool.execute({ + skill: 'simple-skill', + })) as any; + + expect(result.content).toContain('Part 1'); + expect(result.content).toContain('Part 2'); + }); + }); + + describe('Empty Skills List', () => { + it('should handle when no skills are available', async () => { + mockPromptManager.listAutoInvocablePrompts = vi.fn().mockResolvedValue({}); + + const result = (await tool.execute({ + skill: 'any-skill', + })) as any; + + expect(result.error).toContain('not found'); + expect(result.availableSkills).toEqual([]); + }); + }); + + describe('Context: Fork Execution', () => { + // Fork skills call taskForker.fork() directly - no additional tool calls needed + let mockTaskForker: TaskForker; + + beforeEach(() => { + mockTaskForker = { + fork: vi.fn().mockResolvedValue({ + success: true, + response: 'Forked task completed successfully', + }), + }; + }); + + it('should execute fork via taskForker when context is fork', async () => { + mockPromptManager.getPromptDefinition = vi.fn().mockResolvedValue({ + context: 'fork', + }); + services.taskForker = mockTaskForker; + + const result = await tool.execute({ + skill: 'simple-skill', + }); + + // Fork skills return just the result text (not JSON) + expect(result).toBe('Forked task completed successfully'); + expect(mockTaskForker.fork).toHaveBeenCalledWith( + expect.objectContaining({ + task: 'Skill: simple-skill', + instructions: 'Skill instructions here', + }) + ); + }); + + it('should include taskContext in forked instructions', async () => { + mockPromptManager.getPromptDefinition = vi.fn().mockResolvedValue({ + context: 'fork', + }); + services.taskForker = mockTaskForker; + + await tool.execute({ + skill: 'simple-skill', + taskContext: 'User wants to analyze code quality', + }); + + expect(mockTaskForker.fork).toHaveBeenCalledWith( + expect.objectContaining({ + instructions: expect.stringContaining('## Task Context'), + }) + ); + expect(mockTaskForker.fork).toHaveBeenCalledWith( + expect.objectContaining({ + instructions: expect.stringContaining('User wants to analyze code quality'), + }) + ); + }); + + it('should pass agentId from skill definition to fork', async () => { + mockPromptManager.getPromptDefinition = vi.fn().mockResolvedValue({ + context: 'fork', + agent: 'explore-agent', + }); + services.taskForker = mockTaskForker; + + await tool.execute({ + skill: 'simple-skill', + }); + + expect(mockTaskForker.fork).toHaveBeenCalledWith( + expect.objectContaining({ + agentId: 'explore-agent', + }) + ); + }); + + it('should return error when fork required but taskForker not available', async () => { + mockPromptManager.getPromptDefinition = vi.fn().mockResolvedValue({ + context: 'fork', + }); + // Don't set taskForker on services + + const result = (await tool.execute({ + skill: 'simple-skill', + })) as any; + + expect(result.error).toContain('requires fork execution'); + expect(result.error).toContain('agent spawning is not available'); + }); + + it('should handle fork execution failure', async () => { + mockPromptManager.getPromptDefinition = vi.fn().mockResolvedValue({ + context: 'fork', + }); + services.taskForker = { + fork: vi.fn().mockResolvedValue({ + success: false, + error: 'Subagent timed out', + }), + }; + + const result = await tool.execute({ + skill: 'simple-skill', + }); + + // Fork errors return error message as text + expect(result).toBe('Error: Subagent timed out'); + }); + + it('should use inline execution when context is inline', async () => { + mockPromptManager.getPromptDefinition = vi.fn().mockResolvedValue({ + context: 'inline', + }); + services.taskForker = mockTaskForker; + + const result = (await tool.execute({ + skill: 'simple-skill', + })) as any; + + // Should NOT call fork + expect(mockTaskForker.fork).not.toHaveBeenCalled(); + // Should return inline content + expect(result.content).toContain('Skill instructions here'); + expect(result.forked).toBeUndefined(); + }); + + it('should use inline execution when context is undefined', async () => { + mockPromptManager.getPromptDefinition = vi.fn().mockResolvedValue({}); + services.taskForker = mockTaskForker; + + const result = (await tool.execute({ + skill: 'simple-skill', + })) as any; + + expect(mockTaskForker.fork).not.toHaveBeenCalled(); + expect(result.content).toContain('Skill instructions here'); + }); + + it('should pass toolCallId and sessionId to fork when provided', async () => { + mockPromptManager.getPromptDefinition = vi.fn().mockResolvedValue({ + context: 'fork', + }); + services.taskForker = mockTaskForker; + + await tool.execute( + { skill: 'simple-skill' }, + { toolCallId: 'call-123', sessionId: 'session-456' } + ); + + expect(mockTaskForker.fork).toHaveBeenCalledWith( + expect.objectContaining({ + toolCallId: 'call-123', + sessionId: 'session-456', + }) + ); + }); + }); + + describe('PromptManager not available', () => { + it('should return error when promptManager is not set', async () => { + const emptyServices: InternalToolsServices = {}; + const toolWithoutManager = createInvokeSkillTool(emptyServices); + + const result = (await toolWithoutManager.execute({ + skill: 'any-skill', + })) as any; + + expect(result.error).toContain('PromptManager not available'); + }); + }); +}); diff --git a/dexto/packages/core/src/tools/internal-tools/implementations/invoke-skill-tool.ts b/dexto/packages/core/src/tools/internal-tools/implementations/invoke-skill-tool.ts new file mode 100644 index 00000000..384629d8 --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/implementations/invoke-skill-tool.ts @@ -0,0 +1,188 @@ +import { z } from 'zod'; +import type { InternalTool, ToolExecutionContext } from '../../types.js'; +import type { InternalToolsServices } from '../registry.js'; +import { flattenPromptResult } from '../../../prompts/utils.js'; + +const InvokeSkillInputSchema = z + .object({ + skill: z + .string() + .min(1, 'Skill name is required') + .describe( + 'The name of the skill to invoke (e.g., "plugin-name:skill-name" or "skill-name")' + ), + args: z.record(z.string()).optional().describe('Optional arguments to pass to the skill'), + taskContext: z + .string() + .optional() + .describe( + 'Context about what task this skill should accomplish. Recommended for forked skills to provide context since they run in isolation without conversation history.' + ), + }) + .strict(); + +type InvokeSkillInput = z.input; + +/** + * Internal tool for invoking skills/prompts during agent execution. + * + * This tool allows the LLM to load and execute skills that are registered + * with the PromptManager. Skills are prompts that can be auto-invoked by + * the model (not disabled via `disableModelInvocation`). + * + * Execution modes: + * - **inline** (default): Skill content is returned for the LLM to follow in the current session + * - **fork**: Skill is executed in an isolated subagent with no conversation history access + * + * Usage: + * - The LLM sees available skills in its system prompt + * - When a skill is relevant, the LLM calls this tool with the skill name + * - For inline skills: content is returned for the LLM to follow + * - For forked skills: execution happens in isolation and result is returned + * + * Note: Takes services object (not individual deps) to support late-binding of taskForker. + * The taskForker may be set after tool creation when agent-spawner custom tool is registered. + */ +export function createInvokeSkillTool(services: InternalToolsServices): InternalTool { + return { + id: 'invoke_skill', + description: buildToolDescription(), + inputSchema: InvokeSkillInputSchema, + execute: async (input: unknown, context?: ToolExecutionContext) => { + const { skill, args, taskContext } = input as InvokeSkillInput; + + // Get promptManager from services (set via setPromptManager before initialize) + const promptManager = services.promptManager; + if (!promptManager) { + return { + error: 'PromptManager not available. This is a configuration error.', + }; + } + + // Check if the prompt exists and is auto-invocable + const autoInvocable = await promptManager.listAutoInvocablePrompts(); + + // Find the skill by checking various name formats + let skillKey: string | undefined; + for (const key of Object.keys(autoInvocable)) { + const info = autoInvocable[key]; + if (!info) continue; + // Match by full key, displayName, commandName, or name + if ( + key === skill || + info.displayName === skill || + info.commandName === skill || + info.name === skill + ) { + skillKey = key; + break; + } + } + + if (!skillKey) { + return { + error: `Skill '${skill}' not found or not available for model invocation. Use a skill from the available list.`, + availableSkills: Object.keys(autoInvocable), + }; + } + + // Get the prompt definition to check execution context + const promptDef = await promptManager.getPromptDefinition(skillKey); + + // Get the prompt content with arguments applied + const promptResult = await promptManager.getPrompt(skillKey, args); + const flattened = flattenPromptResult(promptResult); + const content = flattened.text; + + // Check if this skill should be forked (executed in isolated subagent) + if (promptDef?.context === 'fork') { + // Fork execution - run in isolated subagent via taskForker + // taskForker is looked up lazily to support late-binding (set after tool creation) + const taskForker = services.taskForker; + if (!taskForker) { + return { + error: `Skill '${skill}' requires fork execution (context: fork), but agent spawning is not available. Configure agent-spawner custom tool to enable forked skills.`, + skill: skillKey, + }; + } + + // Build instructions for the forked agent + let instructions: string; + if (taskContext) { + instructions = `## Task Context\n${taskContext}\n\n## Skill Instructions\n${content}`; + } else { + instructions = content; + } + + // Execute in isolated context via taskForker + const forkOptions: { + task: string; + instructions: string; + agentId?: string; + autoApprove?: boolean; + toolCallId?: string; + sessionId?: string; + } = { + task: `Skill: ${skill}`, + instructions, + // Fork skills auto-approve by default since they run in isolation + autoApprove: true, + }; + // Pass agent from skill definition if specified + if (promptDef.agent) { + forkOptions.agentId = promptDef.agent; + } + if (context?.toolCallId) { + forkOptions.toolCallId = context.toolCallId; + } + if (context?.sessionId) { + forkOptions.sessionId = context.sessionId; + } + + const result = await taskForker.fork(forkOptions); + + if (result.success) { + // Return just the result text - no JSON wrapping + // This gives cleaner display in CLI and WebUI + return result.response ?? 'Task completed successfully.'; + } else { + // For errors, return a simple error message + return `Error: ${result.error ?? 'Unknown error during forked execution'}`; + } + } + + // Inline execution (default) - return content for LLM to follow + return { + skill: skillKey, + content, + instructions: + 'Follow the instructions in the skill content above to complete the task.', + }; + }, + }; +} + +/** + * Builds the tool description. + */ +function buildToolDescription(): string { + return `Invoke a skill to load and execute specialized instructions for a task. Skills are predefined prompts that guide how to handle specific scenarios. + +When to use: +- When you recognize a task that matches an available skill +- When you need specialized guidance for a complex operation +- When the user references a skill by name + +Parameters: +- skill: The name of the skill to invoke +- args: Optional arguments to pass to the skill (e.g., for $ARGUMENTS substitution) +- taskContext: Context about what you're trying to accomplish (important for forked skills that run in isolation) + +Execution modes: +- **Inline skills**: Return instructions for you to follow in the current conversation +- **Fork skills**: Automatically execute in an isolated subagent and return the result (no additional tool calls needed) + +Fork skills run in complete isolation without access to conversation history. They're useful for tasks that should run independently. + +Available skills are listed in your system prompt. Use the skill name exactly as shown.`; +} diff --git a/dexto/packages/core/src/tools/internal-tools/implementations/list-resources-tool.ts b/dexto/packages/core/src/tools/internal-tools/implementations/list-resources-tool.ts new file mode 100644 index 00000000..81d4af53 --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/implementations/list-resources-tool.ts @@ -0,0 +1,154 @@ +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext } from '../../types.js'; +import type { ResourceManager } from '../../../resources/manager.js'; + +/** + * Input schema for list_resources tool + */ +const ListResourcesInputSchema = z + .object({ + source: z + .enum(['all', 'tool', 'user']) + .optional() + .default('all') + .describe( + 'Filter by source: "tool" for tool-generated resources, "user" for user-uploaded, "all" for both' + ), + kind: z + .enum(['all', 'image', 'audio', 'video', 'binary']) + .optional() + .default('all') + .describe('Filter by type: "image", "audio", "video", "binary", or "all"'), + limit: z + .number() + .optional() + .default(50) + .describe('Maximum number of resources to return (default: 50)'), + }) + .strict(); + +type ListResourcesInput = z.output; + +/** + * Resource information returned to the agent + * + * The reference uses "blob:" prefix (not "@blob:") to avoid + * triggering base64 expansion by expandBlobsInText() when this JSON + * is serialized as a tool result. + */ +interface ResourceInfo { + /** The blob reference (e.g., "blob:abc123") - use this with get_resource */ + reference: string; + /** Resource type (image, audio, video, binary) */ + kind: string; + /** MIME type */ + mimeType: string; + /** Original filename if available */ + filename?: string; + /** Source of the resource */ + source: 'tool' | 'user' | 'system'; + /** Size in bytes */ + size: number; + /** When the resource was created */ + createdAt: string; +} + +/** + * Internal tool for listing available resources. + * + * This tool allows agents to discover what resources (images, files, etc.) are + * available. Use this to find resources that can be accessed with get_resource. + * + * Resources are stored blobs from: + * - Tool results (images generated by tools, screenshots, etc.) + * - User uploads (files attached to messages) + * + * @example + * ```typescript + * // List all tool-generated images + * list_resources({ source: 'tool', kind: 'image' }) + * → { resources: [{ reference: 'blob:abc123', kind: 'image', ... }] } + * + * // Get a URL for sharing + * get_resource({ reference: 'blob:abc123', format: 'url' }) + * ``` + */ +export function createListResourcesTool(resourceManager: ResourceManager): InternalTool { + return { + id: 'list_resources', + description: + 'List available resources (images, files, etc.). Returns resource references ' + + 'that can be used with get_resource to obtain shareable URLs or metadata. ' + + 'Filter by source (tool/user) or kind (image/audio/video/binary).', + inputSchema: ListResourcesInputSchema, + execute: async (input: unknown, _context?: ToolExecutionContext) => { + const { source, kind, limit } = input as ListResourcesInput; + + try { + const blobStore = resourceManager.getBlobStore(); + const allBlobs = await blobStore.listBlobs(); + + // Filter and transform blobs + const resources: ResourceInfo[] = []; + + for (const blob of allBlobs) { + // Skip system resources (internal prompts, etc.) + if (blob.metadata.source === 'system') { + continue; + } + + // Filter by source + if (source !== 'all' && blob.metadata.source !== source) { + continue; + } + + // Determine resource kind from MIME type + const mimeType = blob.metadata.mimeType; + let resourceKind: 'image' | 'audio' | 'video' | 'binary' = 'binary'; + if (mimeType.startsWith('image/')) resourceKind = 'image'; + else if (mimeType.startsWith('audio/')) resourceKind = 'audio'; + else if (mimeType.startsWith('video/')) resourceKind = 'video'; + + // Filter by kind + if (kind !== 'all' && resourceKind !== kind) { + continue; + } + + // Use blob.uri without @ prefix to avoid expansion by expandBlobsInText() + resources.push({ + reference: blob.uri, + kind: resourceKind, + mimeType: blob.metadata.mimeType, + ...(blob.metadata.originalName && { filename: blob.metadata.originalName }), + source: blob.metadata.source || 'tool', + size: blob.metadata.size, + createdAt: blob.metadata.createdAt.toISOString(), + }); + } + + // Sort by creation time (newest first), then apply limit + // This ensures we return the N newest resources, not arbitrary N resources + resources.sort( + (a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime() + ); + const limitedResources = resources.slice(0, limit); + + return { + success: true, + count: limitedResources.length, + resources: limitedResources, + _hint: + limitedResources.length > 0 + ? 'Use get_resource with a reference to get a shareable URL or metadata' + : 'No resources found matching the criteria', + }; + } catch (error) { + return { + success: false, + error: `Failed to list resources: ${error instanceof Error ? error.message : String(error)}`, + resources: [], + }; + } + }, + }; +} diff --git a/dexto/packages/core/src/tools/internal-tools/implementations/search-history-tool.ts b/dexto/packages/core/src/tools/internal-tools/implementations/search-history-tool.ts new file mode 100644 index 00000000..9699aef8 --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/implementations/search-history-tool.ts @@ -0,0 +1,65 @@ +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext } from '../../types.js'; +import { SearchService } from '../../../search/index.js'; +import type { SearchOptions } from '../../../search/index.js'; + +const SearchHistoryInputSchema = z.object({ + query: z.string().describe('The search query to find in conversation history'), + mode: z + .enum(['messages', 'sessions']) + .describe( + 'Search mode: "messages" searches for individual messages, "sessions" finds sessions containing the query' + ), + sessionId: z + .string() + .optional() + .describe('Optional: limit search to a specific session (only for mode="messages")'), + role: z + .enum(['user', 'assistant', 'system', 'tool']) + .optional() + .describe('Optional: filter by message role (only for mode="messages")'), + limit: z + .number() + .optional() + .default(20) + .describe( + 'Optional: maximum number of results to return (default: 20, only for mode="messages")' + ), + offset: z + .number() + .optional() + .default(0) + .describe('Optional: offset for pagination (default: 0, only for mode="messages")'), +}); + +type SearchHistoryInput = z.input; + +/** + * Internal tool for searching conversation history + */ +export function createSearchHistoryTool(searchService: SearchService): InternalTool { + return { + id: 'search_history', + description: + 'Search through conversation history across sessions. Use mode="messages" to search for specific messages, or mode="sessions" to find sessions containing the query. For message search, you can filter by sessionId (specific session), role (user/assistant/system/tool), limit results, and set pagination offset.', + inputSchema: SearchHistoryInputSchema, + // TODO: Enhance to get SearchService via ToolExecutionContext for better separation of concerns + execute: async (input: unknown, _context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { query, mode, sessionId, role, limit, offset } = input as SearchHistoryInput; + + if (mode === 'messages') { + const searchOptions: SearchOptions = {}; + if (sessionId !== undefined) searchOptions.sessionId = sessionId; + if (role !== undefined) searchOptions.role = role; + if (limit !== undefined) searchOptions.limit = limit; + if (offset !== undefined) searchOptions.offset = offset; + + return await searchService.searchMessages(query, searchOptions); + } else { + // mode is 'sessions' - TypeScript knows this due to the enum + return await searchService.searchSessions(query); + } + }, + }; +} diff --git a/dexto/packages/core/src/tools/internal-tools/index.ts b/dexto/packages/core/src/tools/internal-tools/index.ts new file mode 100644 index 00000000..20f76923 --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/index.ts @@ -0,0 +1,3 @@ +export * from './provider.js'; +export * from './registry.js'; +export * from './constants.js'; diff --git a/dexto/packages/core/src/tools/internal-tools/provider.test.ts b/dexto/packages/core/src/tools/internal-tools/provider.test.ts new file mode 100644 index 00000000..e6eef37b --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/provider.test.ts @@ -0,0 +1,672 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { z } from 'zod'; +import { InternalToolsProvider } from './provider.js'; +import type { InternalToolsServices } from './registry.js'; +import type { InternalToolsConfig } from '../schemas.js'; +import type { InternalTool } from '../types.js'; +import { DextoRuntimeError } from '../../errors/DextoRuntimeError.js'; +import { ToolErrorCode } from '../error-codes.js'; +import { ErrorScope, ErrorType } from '../../errors/types.js'; +import { ApprovalManager } from '../../approval/manager.js'; + +// No need to mock logger - we'll pass mockLogger directly to constructors + +// Mock zodToJsonSchema +vi.mock('zod-to-json-schema', () => ({ + zodToJsonSchema: vi.fn().mockReturnValue({ + type: 'object', + properties: { + query: { type: 'string' }, + mode: { type: 'string', enum: ['messages', 'sessions'] }, + }, + required: ['query', 'mode'], + }), +})); + +describe('InternalToolsProvider', () => { + let mockServices: InternalToolsServices; + let config: InternalToolsConfig; + let mockLogger: any; + + beforeEach(() => { + mockLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(function (this: any) { + return this; + }), + destroy: vi.fn(), + } as any; + + // Create ApprovalManager in auto-approve mode for tests + const approvalManager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + // Mock services including approvalManager (now part of InternalToolsServices) + mockServices = { + searchService: { + searchMessages: vi + .fn() + .mockResolvedValue([{ id: '1', content: 'test message', role: 'user' }]), + searchSessions: vi + .fn() + .mockResolvedValue([{ id: 'session1', title: 'Test Session' }]), + } as any, + approvalManager, + }; + + config = ['search_history']; + + vi.clearAllMocks(); + }); + + describe('Initialization', () => { + it('should initialize with empty config', async () => { + const provider = new InternalToolsProvider(mockServices, [], [], mockLogger); + await provider.initialize(); + + expect(provider.getToolCount()).toBe(0); + expect(provider.getToolNames()).toEqual([]); + }); + + it('should register tools when services are available', async () => { + const provider = new InternalToolsProvider(mockServices, config, [], mockLogger); + await provider.initialize(); + + expect(provider.getToolCount()).toBe(1); + expect(provider.getToolNames()).toContain('search_history'); + }); + + it('should skip tools when required services are missing', async () => { + const servicesWithoutSearch: InternalToolsServices = { + // Missing searchService but has approvalManager + approvalManager: mockServices.approvalManager!, + }; + + const provider = new InternalToolsProvider( + servicesWithoutSearch, + config, + [], + mockLogger + ); + await provider.initialize(); + + expect(provider.getToolCount()).toBe(0); + expect(provider.getToolNames()).toEqual([]); + }); + + it('should handle tool registration errors gracefully', async () => { + // Create a provider with services that will cause the tool factory to fail + const failingServices: InternalToolsServices = { + searchService: null as any, // This should cause issues during tool creation + approvalManager: mockServices.approvalManager!, + }; + + const provider = new InternalToolsProvider(failingServices, config, [], mockLogger); + await provider.initialize(); + + // Tool should be skipped due to missing service, so count should be 0 + expect(provider.getToolCount()).toBe(0); + }); + + it('should log initialization progress', async () => { + const provider = new InternalToolsProvider(mockServices, config, [], mockLogger); + await provider.initialize(); + + expect(mockLogger.info).toHaveBeenCalledWith('Initializing InternalToolsProvider...'); + expect(mockLogger.info).toHaveBeenCalledWith( + 'InternalToolsProvider initialized with 1 tools (1 internal, 0 custom)' + ); + }); + }); + + describe('Tool Management', () => { + let provider: InternalToolsProvider; + + beforeEach(async () => { + provider = new InternalToolsProvider(mockServices, config, [], mockLogger); + await provider.initialize(); + }); + + it('should check if tool exists', () => { + expect(provider.hasTool('search_history')).toBe(true); + expect(provider.hasTool('nonexistent_tool')).toBe(false); + }); + + it('should return correct tool count', () => { + expect(provider.getToolCount()).toBe(1); + }); + + it('should return tool names', () => { + const names = provider.getToolNames(); + expect(names).toEqual(['search_history']); + }); + + it('should convert tools to ToolSet format', () => { + const toolSet = provider.getInternalTools(); + + expect(toolSet).toHaveProperty('search_history'); + expect(toolSet.search_history).toEqual({ + name: 'search_history', + description: expect.stringContaining('Search through conversation history'), + parameters: { + type: 'object', + properties: { + query: { type: 'string' }, + mode: { type: 'string', enum: ['messages', 'sessions'] }, + }, + required: ['query', 'mode'], + }, + }); + }); + + it('should handle Zod schema conversion errors gracefully', async () => { + const { zodToJsonSchema } = await import('zod-to-json-schema'); + + // Mock zodToJsonSchema to throw an error + (zodToJsonSchema as any).mockImplementationOnce(() => { + throw new Error('Schema conversion failed'); + }); + + const toolSet = provider.getInternalTools(); + + // Should return fallback schema + expect(toolSet.search_history?.parameters).toEqual({ + type: 'object', + properties: {}, + }); + }); + }); + + describe('Tool Execution', () => { + let provider: InternalToolsProvider; + + beforeEach(async () => { + provider = new InternalToolsProvider(mockServices, config, [], mockLogger); + await provider.initialize(); + }); + + it('should execute tool with correct arguments and context', async () => { + const args = { query: 'test query', mode: 'messages' as const }; + const sessionId = 'test-session-123'; + + const result = await provider.executeTool('search_history', args, sessionId); + + expect(mockServices.searchService?.searchMessages).toHaveBeenCalledWith( + 'test query', + expect.objectContaining({ + limit: 20, // Default from Zod schema + offset: 0, // Default from Zod schema + // sessionId and role are undefined, so not included in the object + }) + ); + expect(result).toEqual([{ id: '1', content: 'test message', role: 'user' }]); + }); + + it('should execute tool without sessionId', async () => { + const args = { query: 'test query', mode: 'sessions' as const }; + + const result = await provider.executeTool('search_history', args); + + expect(mockServices.searchService?.searchSessions).toHaveBeenCalledWith('test query'); + expect(result).toEqual([{ id: 'session1', title: 'Test Session' }]); + }); + + it('should throw error for nonexistent tool', async () => { + const error = (await provider + .executeTool('nonexistent_tool', {}) + .catch((e) => e)) as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(ToolErrorCode.TOOL_NOT_FOUND); + expect(error.scope).toBe(ErrorScope.TOOLS); + expect(error.type).toBe(ErrorType.NOT_FOUND); + }); + + it('should propagate tool execution errors', async () => { + // Mock search service to throw error + mockServices.searchService!.searchMessages = vi + .fn() + .mockRejectedValue(new Error('Search service failed')); + + await expect( + provider.executeTool('search_history', { + query: 'test', + mode: 'messages' as const, + }) + ).rejects.toThrow('Search service failed'); + }); + + it('should log execution errors', async () => { + // Mock search service to throw error + mockServices.searchService!.searchMessages = vi + .fn() + .mockRejectedValue(new Error('Search service failed')); + + try { + await provider.executeTool('search_history', { + query: 'test', + mode: 'messages' as const, + }); + } catch { + // Expected to throw + } + + expect(mockLogger.error).toHaveBeenCalledWith( + '❌ Internal tool execution failed: search_history', + expect.objectContaining({ error: expect.any(String) }) + ); + }); + + it('should validate input against tool schema', async () => { + const mockTool: InternalTool = { + id: 'test_tool', + description: 'Test tool', + inputSchema: z.object({ + required_param: z.string(), + optional_param: z.number().optional(), + }), + execute: vi.fn().mockResolvedValue('test result'), + }; + + // Manually add the mock tool to the provider + (provider as any).internalTools.set('test_tool', mockTool); + + // Test with invalid input - missing required field + const error = (await provider + .executeTool('test_tool', { optional_param: 42 }) + .catch((e) => e)) as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(ToolErrorCode.TOOL_INVALID_ARGS); + expect(error.scope).toBe(ErrorScope.TOOLS); + expect(error.type).toBe(ErrorType.USER); + + // Tool should not have been called + expect(mockTool.execute).not.toHaveBeenCalled(); + }); + + it('should provide correct tool execution context', async () => { + // Create a mock tool to verify context is passed correctly + const mockTool: InternalTool = { + id: 'test_tool', + description: 'Test tool', + inputSchema: z.object({ + param: z.string(), + }), + execute: vi.fn().mockResolvedValue('test result'), + }; + + // Manually add the mock tool to the provider + (provider as any).internalTools.set('test_tool', mockTool); + + const sessionId = 'test-session-456'; + await provider.executeTool('test_tool', { param: 'value' }, sessionId); + + expect(mockTool.execute).toHaveBeenCalledWith( + { param: 'value' }, + { sessionId: 'test-session-456' } + ); + }); + }); + + describe('Service Dependencies', () => { + it('should only register tools when all required services are available', async () => { + const partialServices: InternalToolsServices = { + // Has searchService and approvalManager + searchService: mockServices.searchService!, + approvalManager: mockServices.approvalManager!, + }; + + const provider = new InternalToolsProvider( + partialServices, + ['search_history'], // This tool requires searchService + [], + mockLogger + ); + await provider.initialize(); + + expect(provider.hasTool('search_history')).toBe(true); + }); + + it('should skip tools when any required service is missing', async () => { + const emptyServices: InternalToolsServices = { + approvalManager: mockServices.approvalManager!, + }; + + const provider = new InternalToolsProvider( + emptyServices, + ['search_history'], // This tool requires searchService + [], + mockLogger + ); + await provider.initialize(); + + expect(provider.hasTool('search_history')).toBe(false); + }); + + it('should log when skipping tools due to missing services', async () => { + const emptyServices: InternalToolsServices = { + approvalManager: mockServices.approvalManager!, + }; + + const provider = new InternalToolsProvider( + emptyServices, + ['search_history'], + [], + mockLogger + ); + await provider.initialize(); + + expect(mockLogger.debug).toHaveBeenCalledWith( + 'Skipping search_history internal tool - missing services: searchService' + ); + }); + }); + + describe('Configuration Handling', () => { + it('should handle multiple tools in config', async () => { + // Note: Only search_history is available in current registry + const provider = new InternalToolsProvider( + mockServices, + ['search_history'], // Add more tools here as they're implemented + [], + mockLogger + ); + await provider.initialize(); + + expect(provider.getToolCount()).toBe(1); + }); + + it('should handle unknown tools in config gracefully', async () => { + // The provider should handle unknown tools by catching errors during getInternalToolInfo + const provider = new InternalToolsProvider( + mockServices, + ['search_history'], // Only use known tools for now + [], + mockLogger + ); + + // This should not throw during initialization + await provider.initialize(); + + // Should register the known tool + expect(provider.hasTool('search_history')).toBe(true); + }); + }); + + describe('Error Handling', () => { + it('should handle initialization failures gracefully', async () => { + // Test with services that only have approvalManager to ensure error handling works + const servicesWithOnlyApproval: InternalToolsServices = { + approvalManager: mockServices.approvalManager!, + }; + const provider = new InternalToolsProvider( + servicesWithOnlyApproval, + config, + [], + mockLogger + ); + + // Should not throw, but should skip tools due to missing services + await provider.initialize(); + + // Should have 0 tools registered due to missing services + expect(provider.getToolCount()).toBe(0); + }); + + it('should handle tool execution context properly', async () => { + const provider = new InternalToolsProvider(mockServices, config, [], mockLogger); + await provider.initialize(); + + // Execute without sessionId + await provider.executeTool('search_history', { + query: 'test', + mode: 'messages' as const, + }); + + // Should create context with undefined sessionId + expect(mockServices.searchService?.searchMessages).toHaveBeenCalledWith( + 'test', + expect.any(Object) + ); + }); + }); + + describe('Custom Tool Provider Validation', () => { + // Mock agent for custom tool tests + const mockAgent = { + agentEventBus: { emit: vi.fn() }, + } as any; + + it('should throw error when custom tool provider is not found', async () => { + const customToolsConfig = [ + { + type: 'nonexistent-provider', + config: {}, + }, + ]; + + const provider = new InternalToolsProvider( + mockServices, + [], + customToolsConfig, + mockLogger + ); + provider.setAgent(mockAgent); + + // Should throw during initialization because provider is missing + await expect(provider.initialize()).rejects.toThrow(DextoRuntimeError); + await expect(provider.initialize()).rejects.toThrow( + "Unknown custom tool provider: 'nonexistent-provider'" + ); + }); + + it('should include available types in error message', async () => { + const customToolsConfig = [ + { + type: 'missing-provider', + config: {}, + }, + ]; + + const provider = new InternalToolsProvider( + mockServices, + [], + customToolsConfig, + mockLogger + ); + provider.setAgent(mockAgent); + + const error = (await provider.initialize().catch((e) => e)) as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(ToolErrorCode.CUSTOM_TOOL_PROVIDER_UNKNOWN); + expect(error.scope).toBe(ErrorScope.TOOLS); + expect(error.type).toBe(ErrorType.USER); + // Error context should include available types (even if empty) + expect(error.context).toHaveProperty('availableTypes'); + }); + + it('should fail fast on first missing provider', async () => { + const customToolsConfig = [ + { + type: 'first-missing-provider', + config: {}, + }, + { + type: 'second-missing-provider', + config: {}, + }, + ]; + + const provider = new InternalToolsProvider( + mockServices, + [], + customToolsConfig, + mockLogger + ); + provider.setAgent(mockAgent); + + // Should throw on the first missing provider + await expect(provider.initialize()).rejects.toThrow( + "Unknown custom tool provider: 'first-missing-provider'" + ); + }); + + it('should initialize successfully when no custom tools are configured', async () => { + const provider = new InternalToolsProvider(mockServices, [], [], mockLogger); + + await provider.initialize(); + + expect(provider.getToolCount()).toBe(0); + }); + + it('should throw error if agent not set before initialization with custom tools', async () => { + const customToolsConfig = [ + { + type: 'some-provider', + config: {}, + }, + ]; + + const provider = new InternalToolsProvider( + mockServices, + [], + customToolsConfig, + mockLogger + ); + // Don't call setAgent() + + await expect(provider.initialize()).rejects.toThrow( + 'Agent reference not set. Call setAgent() before initialize() when using custom tools.' + ); + }); + }); + + describe('Required Features', () => { + it('should throw when tool requires a feature that is disabled', async () => { + // Create ApprovalManager with elicitation DISABLED + const approvalManagerWithDisabledElicitation = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: false, // Elicitation is disabled + timeout: 120000, + }, + }, + mockLogger + ); + + const servicesWithDisabledElicitation: InternalToolsServices = { + approvalManager: approvalManagerWithDisabledElicitation, + }; + + // ask_user requires elicitation feature + const provider = new InternalToolsProvider( + servicesWithDisabledElicitation, + ['ask_user'], + [], + mockLogger + ); + + // Should throw during initialization + await expect(provider.initialize()).rejects.toThrow(DextoRuntimeError); + }); + + it('should include feature name and recovery hint in error message', async () => { + const approvalManagerWithDisabledElicitation = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: false, + timeout: 120000, + }, + }, + mockLogger + ); + + const servicesWithDisabledElicitation: InternalToolsServices = { + approvalManager: approvalManagerWithDisabledElicitation, + }; + + const provider = new InternalToolsProvider( + servicesWithDisabledElicitation, + ['ask_user'], + [], + mockLogger + ); + + const error = (await provider.initialize().catch((e) => e)) as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(ToolErrorCode.FEATURE_DISABLED); + expect(error.scope).toBe(ErrorScope.TOOLS); + expect(error.type).toBe(ErrorType.USER); + expect(error.message).toContain('elicitation'); + expect(error.message).toContain('ask_user'); + }); + + it('should register tool when required feature is enabled', async () => { + // mockServices already has elicitation enabled (from beforeEach) + const provider = new InternalToolsProvider(mockServices, ['ask_user'], [], mockLogger); + + await provider.initialize(); + + expect(provider.hasTool('ask_user')).toBe(true); + expect(provider.getToolCount()).toBe(1); + }); + + it('should register tool that has no required features', async () => { + // search_history has no requiredFeatures + const provider = new InternalToolsProvider( + mockServices, + ['search_history'], + [], + mockLogger + ); + await provider.initialize(); + + expect(provider.hasTool('search_history')).toBe(true); + }); + + it('should skip tool when required services are missing (before feature check)', async () => { + // ask_user requires approvalManager service + // When service is missing, tool is skipped BEFORE feature checking + const servicesWithoutApprovalManager: InternalToolsServices = { + searchService: mockServices.searchService!, + // Missing approvalManager - ask_user needs this as a service + }; + + const provider = new InternalToolsProvider( + servicesWithoutApprovalManager, + ['ask_user'], + [], + mockLogger + ); + + // Should NOT throw - tool is skipped due to missing service + await provider.initialize(); + + // Tool should not be registered + expect(provider.hasTool('ask_user')).toBe(false); + expect(provider.getToolCount()).toBe(0); + }); + }); +}); diff --git a/dexto/packages/core/src/tools/internal-tools/provider.ts b/dexto/packages/core/src/tools/internal-tools/provider.ts new file mode 100644 index 00000000..9523df1b --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/provider.ts @@ -0,0 +1,386 @@ +import { ToolExecutionContext, ToolSet, InternalTool } from '../types.js'; +import type { IDextoLogger } from '../../logger/v2/types.js'; +import type { DextoAgent } from '../../agent/DextoAgent.js'; +import { ToolError } from '../errors.js'; +import { convertZodSchemaToJsonSchema } from '../../utils/schema.js'; +import { InternalToolsServices, getInternalToolInfo, type AgentFeature } from './registry.js'; +import type { PromptManager } from '../../prompts/prompt-manager.js'; +import type { InternalToolsConfig, CustomToolsConfig } from '../schemas.js'; +import { customToolRegistry, type ToolCreationContext } from '../custom-tool-registry.js'; +import { DextoRuntimeError } from '../../errors/DextoRuntimeError.js'; +import { ToolErrorCode } from '../error-codes.js'; + +/** + * Provider for built-in internal tools and custom tool providers + * + * This provider manages: + * 1. Built-in internal tools that are shipped with the core system + * 2. Custom tools registered via the customToolRegistry + * + * Benefits: + * - Clean separation: ToolManager doesn't need to know about specific services + * - Easy to extend: Just add new tools and services as needed + * - Lightweight: Direct tool management without complex infrastructure + * - No unnecessary ProcessedInternalTool wrapper - uses InternalTool directly + * - Custom tools follow the same provider pattern as blob storage + */ +export class InternalToolsProvider { + private services: InternalToolsServices; + private internalTools: Map = new Map(); // Built-in internal tools + private customTools: Map = new Map(); // Custom tool provider tools + private config: InternalToolsConfig; + private customToolConfigs: CustomToolsConfig; + private logger: IDextoLogger; + private agent?: DextoAgent; // Set after construction to avoid circular dependency + + constructor( + services: InternalToolsServices, + config: InternalToolsConfig = [], + customToolConfigs: CustomToolsConfig = [], + logger: IDextoLogger + ) { + this.services = services; + this.config = config; + this.customToolConfigs = customToolConfigs; + this.logger = logger; + this.logger.debug('InternalToolsProvider initialized with config:', { + config, + customToolConfigs, + }); + } + + /** + * Set agent reference after construction (avoids circular dependency) + * Must be called before initialize() if custom tools need agent access + */ + setAgent(agent: DextoAgent): void { + this.agent = agent; + } + + /** + * Set prompt manager after construction (avoids circular dependency) + * Must be called before initialize() if invoke_skill tool is enabled + */ + setPromptManager(promptManager: PromptManager): void { + this.services.promptManager = promptManager; + } + + /** + * Set task forker for context:fork skill execution (late-binding) + * Called by agent-spawner custom tool provider after RuntimeService is created. + * This enables invoke_skill to fork execution to an isolated subagent. + */ + setTaskForker(taskForker: import('./registry.js').TaskForker): void { + this.services.taskForker = taskForker; + } + + /** + * Initialize the internal tools provider by registering all available internal tools + * and custom tools from the registry + */ + async initialize(): Promise { + this.logger.info('Initializing InternalToolsProvider...'); + + try { + // Register built-in internal tools + if (this.config.length > 0) { + this.registerInternalTools(); + } else { + this.logger.info('No internal tools enabled by configuration'); + } + + // Register custom tools from registry + if (this.customToolConfigs.length > 0) { + this.registerCustomTools(); + } else { + this.logger.debug('No custom tool providers configured'); + } + + const internalCount = this.internalTools.size; + const customCount = this.customTools.size; + this.logger.info( + `InternalToolsProvider initialized with ${internalCount + customCount} tools (${internalCount} internal, ${customCount} custom)` + ); + } catch (error) { + this.logger.error( + `Failed to initialize InternalToolsProvider: ${error instanceof Error ? error.message : String(error)}` + ); + throw error; + } + } + + /** + * Register all available internal tools based on available services and configuration + */ + private registerInternalTools(): void { + // Build feature flags from services + const featureFlags: Record = { + elicitation: this.services.approvalManager?.getConfig().elicitation.enabled ?? false, + }; + + for (const toolName of this.config) { + const toolInfo = getInternalToolInfo(toolName); + + // Check if all required services are available + const missingServices = toolInfo.requiredServices.filter( + (serviceKey) => !this.services[serviceKey] + ); + + if (missingServices.length > 0) { + this.logger.debug( + `Skipping ${toolName} internal tool - missing services: ${missingServices.join(', ')}` + ); + continue; + } + + // Check if all required features are enabled - fail hard if not + const missingFeatures = (toolInfo.requiredFeatures ?? []).filter( + (feature) => !featureFlags[feature] + ); + + if (missingFeatures.length > 0) { + throw ToolError.featureDisabled( + toolName, + missingFeatures, + `Tool '${toolName}' requires features which are currently disabled: ${missingFeatures.join(', ')}. ` + + `Either remove '${toolName}' from internalTools, or enable: ${missingFeatures.map((f) => `${f}.enabled: true`).join(', ')}` + ); + } + + try { + // Create the tool using its factory and store directly + const tool = toolInfo.factory(this.services); + this.internalTools.set(toolName, tool); // Store in internal tools map + this.logger.debug(`Registered ${toolName} internal tool`); + } catch (error) { + this.logger.error( + `Failed to register ${toolName} internal tool: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + } + + /** + * Register custom tools from the custom tool registry. + * Tools are stored by their original ID - prefixing is handled by ToolManager. + */ + private registerCustomTools(): void { + if (!this.agent) { + throw ToolError.configInvalid( + 'Agent reference not set. Call setAgent() before initialize() when using custom tools.' + ); + } + + const context: ToolCreationContext = { + logger: this.logger, + agent: this.agent, + services: { + ...this.services, + // Include storageManager from agent services for custom tools that need persistence + storageManager: this.agent.services?.storageManager, + }, + }; + + for (const toolConfig of this.customToolConfigs) { + try { + // Validate config against provider schema + const validatedConfig = customToolRegistry.validateConfig(toolConfig); + const provider = customToolRegistry.get(validatedConfig.type); + + if (!provider) { + const availableTypes = customToolRegistry.getTypes(); + throw ToolError.unknownCustomToolProvider(validatedConfig.type, availableTypes); + } + + // Create tools from provider + const tools = provider.create(validatedConfig, context); + + // Register each tool by its ID (no prefix - ToolManager handles prefixing) + for (const tool of tools) { + // Check for conflicts with other custom tools + if (this.customTools.has(tool.id)) { + this.logger.warn( + `Custom tool '${tool.id}' conflicts with existing custom tool. Skipping.` + ); + continue; + } + + this.customTools.set(tool.id, tool); + this.logger.debug( + `Registered custom tool: ${tool.id} from provider '${provider.metadata?.displayName || validatedConfig.type}'` + ); + } + } catch (error) { + // Re-throw validation errors (unknown provider, invalid config) + // These are user errors that should fail fast + if ( + error instanceof DextoRuntimeError && + error.code === ToolErrorCode.CUSTOM_TOOL_PROVIDER_UNKNOWN + ) { + throw error; + } + + // Log and continue for other errors (e.g., provider initialization failures) + this.logger.error( + `Failed to register custom tool provider: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + } + + /** + * Check if a tool exists (checks both internal and custom tools) + */ + hasTool(toolName: string): boolean { + return this.internalTools.has(toolName) || this.customTools.has(toolName); + } + + /** + * Check if an internal tool exists + */ + hasInternalTool(toolName: string): boolean { + return this.internalTools.has(toolName); + } + + /** + * Check if a custom tool exists + */ + hasCustomTool(toolName: string): boolean { + return this.customTools.has(toolName); + } + + /** + * Get an internal tool by name + * Returns undefined if tool doesn't exist + */ + getTool(toolName: string): InternalTool | undefined { + return this.internalTools.get(toolName) || this.customTools.get(toolName); + } + + /** + * Execute an internal tool - confirmation is handled by ToolManager + */ + async executeTool( + toolName: string, + args: Record, + sessionId?: string, + abortSignal?: AbortSignal, + toolCallId?: string + ): Promise { + // Check internal tools first, then custom tools + const tool = this.internalTools.get(toolName) || this.customTools.get(toolName); + if (!tool) { + this.logger.error(`❌ No tool found: ${toolName}`); + this.logger.debug( + `Available internal tools: ${Array.from(this.internalTools.keys()).join(', ')}` + ); + this.logger.debug( + `Available custom tools: ${Array.from(this.customTools.keys()).join(', ')}` + ); + throw ToolError.notFound(toolName); + } + + // Validate input against tool's Zod schema + const validationResult = tool.inputSchema.safeParse(args); + if (!validationResult.success) { + this.logger.error( + `❌ Invalid arguments for tool ${toolName}: ${validationResult.error.message}` + ); + throw ToolError.invalidName( + toolName, + `Invalid arguments: ${validationResult.error.message}` + ); + } + + try { + const context: ToolExecutionContext = { + sessionId, + abortSignal, + toolCallId, + }; + const result = await tool.execute(validationResult.data, context); + return result; + } catch (error) { + this.logger.error(`❌ Internal tool execution failed: ${toolName}`, { + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } + } + + /** + * Get internal tools in ToolSet format (excludes custom tools) + */ + getInternalTools(): ToolSet { + const toolSet: ToolSet = {}; + + for (const [name, tool] of this.internalTools) { + toolSet[name] = { + name: tool.id, + description: tool.description, + parameters: convertZodSchemaToJsonSchema(tool.inputSchema, this.logger), + }; + } + + return toolSet; + } + + /** + * Get custom tools in ToolSet format (excludes internal tools) + */ + getCustomTools(): ToolSet { + const toolSet: ToolSet = {}; + + for (const [name, tool] of this.customTools) { + toolSet[name] = { + name: tool.id, + description: tool.description, + parameters: convertZodSchemaToJsonSchema(tool.inputSchema, this.logger), + }; + } + + return toolSet; + } + + /** + * Get internal tool names + */ + getInternalToolNames(): string[] { + return Array.from(this.internalTools.keys()); + } + + /** + * Get custom tool names + */ + getCustomToolNames(): string[] { + return Array.from(this.customTools.keys()); + } + + /** + * Get all tool names (internal + custom) + */ + getToolNames(): string[] { + return [...this.internalTools.keys(), ...this.customTools.keys()]; + } + + /** + * Get tool count + */ + getToolCount(): number { + return this.internalTools.size + this.customTools.size; + } + + /** + * Get internal tool count + */ + getInternalToolCount(): number { + return this.internalTools.size; + } + + /** + * Get custom tool count + */ + getCustomToolCount(): number { + return this.customTools.size; + } +} diff --git a/dexto/packages/core/src/tools/internal-tools/registry.ts b/dexto/packages/core/src/tools/internal-tools/registry.ts new file mode 100644 index 00000000..d0b2c22a --- /dev/null +++ b/dexto/packages/core/src/tools/internal-tools/registry.ts @@ -0,0 +1,139 @@ +import { InternalTool } from '../types.js'; +import { SearchService } from '../../search/index.js'; +import { ApprovalManager } from '../../approval/manager.js'; +import { ResourceManager } from '../../resources/manager.js'; +import type { PromptManager } from '../../prompts/prompt-manager.js'; +import { createSearchHistoryTool } from './implementations/search-history-tool.js'; +import { createAskUserTool } from './implementations/ask-user-tool.js'; +import { createDelegateToUrlTool } from './implementations/delegate-to-url-tool.js'; +import { createListResourcesTool } from './implementations/list-resources-tool.js'; +import { createGetResourceTool } from './implementations/get-resource-tool.js'; +import { createInvokeSkillTool } from './implementations/invoke-skill-tool.js'; +import type { KnownInternalTool } from './constants.js'; + +/** + * Agent features that tools can depend on. + * Tools can declare required features via `requiredFeatures` in the registry. + * If a required feature is disabled, the tool will not be registered and agent startup will fail. + * + * To add new features: + * 1. Add the feature name to this union type (e.g., 'elicitation' | 'file_access' | 'network_access') + * 2. Add the feature flag derivation in provider.ts `registerInternalTools()`: + * ``` + * const featureFlags: Record = { + * elicitation: this.services.approvalManager?.getConfig().elicitation.enabled ?? false, + * file_access: config.fileAccess?.enabled ?? false, // example + * }; + * ``` + * 3. Add `requiredFeatures: ['feature_name'] as const` to tool entries that need it + * + * Tools can require multiple features - all must be enabled or startup fails with a clear error. + */ +export type AgentFeature = 'elicitation'; + +/** + * Interface for forking skill execution to an isolated subagent. + * Implemented by RuntimeService in @dexto/agent-management. + */ +export interface TaskForker { + /** + * Execute a task in an isolated subagent context. + * The subagent has no access to the parent's conversation history. + * + * @param options.task - Short description for UI/logs + * @param options.instructions - Full instructions for the subagent + * @param options.agentId - Optional agent ID from registry to use for execution + * @param options.autoApprove - Auto-approve tool calls (default: true for fork skills) + * @param options.toolCallId - Optional tool call ID for progress events + * @param options.sessionId - Optional session ID for progress events + * @returns Result with success status and response/error + */ + fork(options: { + task: string; + instructions: string; + agentId?: string; + autoApprove?: boolean; + toolCallId?: string; + sessionId?: string; + }): Promise<{ + success: boolean; + response?: string; + error?: string; + }>; +} + +/** + * Services available to internal tools + * Add new services here as needed for internal tools + */ +export interface InternalToolsServices { + searchService?: SearchService; + approvalManager?: ApprovalManager; + resourceManager?: ResourceManager; + promptManager?: PromptManager; + /** Optional forker for executing skills in isolated context (context: fork) */ + taskForker?: TaskForker; +} + +/** + * Internal tool factory function type + */ +type InternalToolFactory = (services: InternalToolsServices) => InternalTool; + +/** + * Internal tool registry entry type + */ +export interface InternalToolRegistryEntry { + factory: InternalToolFactory; + requiredServices: readonly (keyof InternalToolsServices)[]; + requiredFeatures?: readonly AgentFeature[]; + /** Short description for discovery/UI purposes */ + description: string; +} + +/** + * Internal tool registry - Must match names array exactly (TypeScript enforces this) + */ +export const INTERNAL_TOOL_REGISTRY: Record = { + search_history: { + factory: (services: InternalToolsServices) => + createSearchHistoryTool(services.searchService!), + requiredServices: ['searchService'] as const, + description: 'Search through conversation history across sessions', + }, + ask_user: { + factory: (services: InternalToolsServices) => createAskUserTool(services.approvalManager!), + requiredServices: ['approvalManager'] as const, + requiredFeatures: ['elicitation'] as const, + description: 'Collect structured input from the user through a form interface', + }, + delegate_to_url: { + factory: (_services: InternalToolsServices) => createDelegateToUrlTool(), + requiredServices: [] as const, + description: 'Delegate tasks to another A2A-compliant agent via URL', + }, + list_resources: { + factory: (services: InternalToolsServices) => + createListResourcesTool(services.resourceManager!), + requiredServices: ['resourceManager'] as const, + description: 'List available resources (images, files, etc.)', + }, + get_resource: { + factory: (services: InternalToolsServices) => + createGetResourceTool(services.resourceManager!), + requiredServices: ['resourceManager'] as const, + description: 'Access a stored resource to get URLs or metadata', + }, + invoke_skill: { + factory: (services: InternalToolsServices) => createInvokeSkillTool(services), + requiredServices: ['promptManager'] as const, + description: 'Invoke a skill to load specialized instructions for a task', + }, +}; + +/** + * Type-safe registry access + */ +export function getInternalToolInfo(toolName: KnownInternalTool): InternalToolRegistryEntry { + return INTERNAL_TOOL_REGISTRY[toolName]; +} diff --git a/dexto/packages/core/src/tools/schemas.test.ts b/dexto/packages/core/src/tools/schemas.test.ts new file mode 100644 index 00000000..de0abc64 --- /dev/null +++ b/dexto/packages/core/src/tools/schemas.test.ts @@ -0,0 +1,520 @@ +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { + InternalToolsSchema, + ToolConfirmationConfigSchema, + ToolPoliciesSchema, + type InternalToolsConfig, + type ToolConfirmationConfig, + type ValidatedToolConfirmationConfig, + type ToolPolicies, +} from './schemas.js'; + +// safeParse for invalid test cases to check exact error codes +// parse for valid test cases for less boilerplate +describe('InternalToolsSchema', () => { + describe('Array Validation', () => { + it('should accept empty array as default', () => { + const result = InternalToolsSchema.parse([]); + expect(result).toEqual([]); + }); + + it('should accept valid internal tool names', () => { + const result = InternalToolsSchema.parse(['search_history']); + expect(result).toEqual(['search_history']); + }); + + it('should reject invalid tool names', () => { + const result = InternalToolsSchema.safeParse(['invalid-tool']); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_enum_value); + expect(result.error?.issues[0]?.path).toEqual([0]); + }); + // TODO: update when more valid tools are added + it('should accept multiple valid tools', () => { + const result = InternalToolsSchema.parse(['search_history']); + expect(result).toHaveLength(1); + }); + }); + + describe('Default Values', () => { + it('should apply default empty array when undefined', () => { + const result = InternalToolsSchema.parse(undefined); + expect(result).toEqual([]); + }); + }); + + describe('Type Safety', () => { + it('should have correct type inference', () => { + const result: InternalToolsConfig = InternalToolsSchema.parse([]); + expect(Array.isArray(result)).toBe(true); + }); + }); +}); + +describe('ToolConfirmationConfigSchema', () => { + describe('Field Validation', () => { + it('should validate mode enum values', () => { + const validModes = ['manual', 'auto-approve', 'auto-deny']; + + validModes.forEach((mode) => { + const result = ToolConfirmationConfigSchema.parse({ mode }); + expect(result.mode).toBe(mode); + }); + + const invalidResult = ToolConfirmationConfigSchema.safeParse({ mode: 'invalid' }); + expect(invalidResult.success).toBe(false); + expect(invalidResult.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_enum_value); + expect(invalidResult.error?.issues[0]?.path).toEqual(['mode']); + }); + + it('should validate timeout as positive integer', () => { + // Negative should fail + let result = ToolConfirmationConfigSchema.safeParse({ timeout: -1 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['timeout']); + + // Zero should fail + result = ToolConfirmationConfigSchema.safeParse({ timeout: 0 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.too_small); + expect(result.error?.issues[0]?.path).toEqual(['timeout']); + + // Float should fail + result = ToolConfirmationConfigSchema.safeParse({ timeout: 1.5 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['timeout']); + + // Valid values should pass + const valid1 = ToolConfirmationConfigSchema.parse({ timeout: 1000 }); + expect(valid1.timeout).toBe(1000); + + const valid2 = ToolConfirmationConfigSchema.parse({ timeout: 120000 }); + expect(valid2.timeout).toBe(120000); + }); + + it('should validate allowedToolsStorage enum values', () => { + const validStorage = ['memory', 'storage']; + + validStorage.forEach((allowedToolsStorage) => { + const result = ToolConfirmationConfigSchema.parse({ allowedToolsStorage }); + expect(result.allowedToolsStorage).toBe(allowedToolsStorage); + }); + + const invalidResult = ToolConfirmationConfigSchema.safeParse({ + allowedToolsStorage: 'invalid', + }); + expect(invalidResult.success).toBe(false); + expect(invalidResult.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_enum_value); + expect(invalidResult.error?.issues[0]?.path).toEqual(['allowedToolsStorage']); + }); + }); + + describe('Default Values', () => { + it('should apply all field defaults for empty object', () => { + const result = ToolConfirmationConfigSchema.parse({}); + + // Note: timeout is now optional with no default (undefined = infinite wait) + expect(result).toEqual({ + mode: 'auto-approve', + allowedToolsStorage: 'storage', + toolPolicies: { + alwaysAllow: [], + alwaysDeny: [], + }, + }); + expect(result.timeout).toBeUndefined(); + }); + + it('should apply field defaults for partial config', () => { + const result1 = ToolConfirmationConfigSchema.parse({ mode: 'auto-approve' }); + // timeout is optional - undefined when not specified + expect(result1).toEqual({ + mode: 'auto-approve', + allowedToolsStorage: 'storage', + toolPolicies: { + alwaysAllow: [], + alwaysDeny: [], + }, + }); + + const result2 = ToolConfirmationConfigSchema.parse({ timeout: 15000 }); + expect(result2).toEqual({ + mode: 'auto-approve', + timeout: 15000, + allowedToolsStorage: 'storage', + toolPolicies: { + alwaysAllow: [], + alwaysDeny: [], + }, + }); + + const result3 = ToolConfirmationConfigSchema.parse({ allowedToolsStorage: 'memory' }); + // timeout is optional - undefined when not specified + expect(result3).toEqual({ + mode: 'auto-approve', + allowedToolsStorage: 'memory', + toolPolicies: { + alwaysAllow: [], + alwaysDeny: [], + }, + }); + }); + + it('should override defaults when values provided', () => { + const config = { + mode: 'auto-deny' as const, + timeout: 60000, + allowedToolsStorage: 'memory' as const, + toolPolicies: { + alwaysAllow: [], + alwaysDeny: [], + }, + }; + + const result = ToolConfirmationConfigSchema.parse(config); + expect(result).toEqual(config); + }); + }); + + describe('Edge Cases', () => { + it('should handle boundary timeout values', () => { + // Very small valid value + const small = ToolConfirmationConfigSchema.parse({ timeout: 1 }); + expect(small.timeout).toBe(1); + + // Large timeout value + const large = ToolConfirmationConfigSchema.parse({ timeout: 300000 }); // 5 minutes + expect(large.timeout).toBe(300000); + }); + + it('should reject non-string mode values', () => { + // Number should fail + let result = ToolConfirmationConfigSchema.safeParse({ mode: 123 }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['mode']); + + // Null should fail + result = ToolConfirmationConfigSchema.safeParse({ mode: null }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['mode']); + }); + + it('should reject non-numeric timeout values', () => { + // String should fail + let result = ToolConfirmationConfigSchema.safeParse({ timeout: 'abc' }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['timeout']); + + // Null should fail + result = ToolConfirmationConfigSchema.safeParse({ timeout: null }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['timeout']); + }); + + it('should reject extra fields with strict validation', () => { + const configWithExtra = { + mode: 'manual', + timeout: 30000, + allowedToolsStorage: 'storage', + unknownField: 'should fail', + }; + + const result = ToolConfirmationConfigSchema.safeParse(configWithExtra); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + + describe('Type Safety', () => { + it('should have correct input and output types', () => { + // Input type allows optional fields (due to defaults) + const input: ToolConfirmationConfig = {}; + const inputPartial: ToolConfirmationConfig = { mode: 'auto-approve' }; + const inputFull: ToolConfirmationConfig = { + mode: 'manual', + timeout: 30000, + allowedToolsStorage: 'storage', + }; + + expect(() => ToolConfirmationConfigSchema.parse(input)).not.toThrow(); + expect(() => ToolConfirmationConfigSchema.parse(inputPartial)).not.toThrow(); + expect(() => ToolConfirmationConfigSchema.parse(inputFull)).not.toThrow(); + }); + + it('should produce validated output type', () => { + const result: ValidatedToolConfirmationConfig = ToolConfirmationConfigSchema.parse({}); + + // Output type guarantees required fields are present + expect(typeof result.mode).toBe('string'); + expect(typeof result.allowedToolsStorage).toBe('string'); + // timeout is optional - undefined when not specified (means infinite wait) + expect(result.timeout).toBeUndefined(); + + // When timeout is provided, it should be a positive number + const resultWithTimeout: ValidatedToolConfirmationConfig = + ToolConfirmationConfigSchema.parse({ timeout: 60000 }); + expect(typeof resultWithTimeout.timeout).toBe('number'); + expect(resultWithTimeout.timeout).toBeGreaterThan(0); + }); + }); + + describe('Real-world Scenarios', () => { + it('should handle interactive mode configuration', () => { + const interactiveConfig = { + mode: 'manual' as const, + timeout: 30000, + allowedToolsStorage: 'storage' as const, + toolPolicies: { + alwaysAllow: [], + alwaysDeny: [], + }, + }; + + const result = ToolConfirmationConfigSchema.parse(interactiveConfig); + expect(result).toEqual(interactiveConfig); + }); + + it('should handle auto-approve configuration', () => { + const autoApproveConfig = { + mode: 'auto-approve' as const, + timeout: 1000, // Lower timeout since no user interaction + allowedToolsStorage: 'memory' as const, // Memory for development + toolPolicies: { + alwaysAllow: [], + alwaysDeny: [], + }, + }; + + const result = ToolConfirmationConfigSchema.parse(autoApproveConfig); + expect(result).toEqual(autoApproveConfig); + }); + + it('should handle strict security configuration', () => { + const strictConfig = { + mode: 'auto-deny' as const, + timeout: 5000, // Short timeout + allowedToolsStorage: 'memory' as const, // No persistent approvals + toolPolicies: { + alwaysAllow: [], + alwaysDeny: [], + }, + }; + + const result = ToolConfirmationConfigSchema.parse(strictConfig); + expect(result).toEqual(strictConfig); + }); + + it('should handle configuration with tool policies', () => { + const configWithPolicies = { + mode: 'manual' as const, + timeout: 30000, + allowedToolsStorage: 'storage' as const, + toolPolicies: { + alwaysAllow: ['internal--ask_user', 'mcp--filesystem--read_file'], + alwaysDeny: ['mcp--filesystem--delete_file'], + }, + }; + + const result = ToolConfirmationConfigSchema.parse(configWithPolicies); + expect(result).toEqual(configWithPolicies); + expect(result.toolPolicies?.alwaysAllow).toHaveLength(2); + expect(result.toolPolicies?.alwaysDeny).toHaveLength(1); + }); + }); +}); + +describe('ToolPoliciesSchema', () => { + describe('Field Validation', () => { + it('should accept empty arrays for both fields', () => { + const result = ToolPoliciesSchema.parse({ + alwaysAllow: [], + alwaysDeny: [], + }); + expect(result).toEqual({ + alwaysAllow: [], + alwaysDeny: [], + }); + }); + + it('should accept valid tool names in alwaysAllow', () => { + const result = ToolPoliciesSchema.parse({ + alwaysAllow: ['internal--ask_user', 'mcp--filesystem--read_file'], + alwaysDeny: [], + }); + expect(result.alwaysAllow).toEqual([ + 'internal--ask_user', + 'mcp--filesystem--read_file', + ]); + }); + + it('should accept valid tool names in alwaysDeny', () => { + const result = ToolPoliciesSchema.parse({ + alwaysAllow: [], + alwaysDeny: ['mcp--filesystem--delete_file', 'mcp--playwright--execute_script'], + }); + expect(result.alwaysDeny).toEqual([ + 'mcp--filesystem--delete_file', + 'mcp--playwright--execute_script', + ]); + }); + + it('should accept both lists populated', () => { + const result = ToolPoliciesSchema.parse({ + alwaysAllow: ['internal--ask_user'], + alwaysDeny: ['mcp--filesystem--delete_file'], + }); + expect(result.alwaysAllow).toHaveLength(1); + expect(result.alwaysDeny).toHaveLength(1); + }); + + it('should reject non-array values for alwaysAllow', () => { + const result = ToolPoliciesSchema.safeParse({ + alwaysAllow: 'not-an-array', + alwaysDeny: [], + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['alwaysAllow']); + }); + + it('should reject non-array values for alwaysDeny', () => { + const result = ToolPoliciesSchema.safeParse({ + alwaysAllow: [], + alwaysDeny: 'not-an-array', + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + expect(result.error?.issues[0]?.path).toEqual(['alwaysDeny']); + }); + + it('should reject non-string elements in arrays', () => { + const result = ToolPoliciesSchema.safeParse({ + alwaysAllow: [123, 456], + alwaysDeny: [], + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.invalid_type); + }); + }); + + describe('Default Values', () => { + it('should apply default empty arrays when undefined', () => { + const result = ToolPoliciesSchema.parse(undefined); + expect(result).toEqual({ + alwaysAllow: [], + alwaysDeny: [], + }); + }); + + it('should apply defaults for missing fields', () => { + const result = ToolPoliciesSchema.parse({}); + expect(result).toEqual({ + alwaysAllow: [], + alwaysDeny: [], + }); + }); + }); + + describe('Edge Cases', () => { + it('should allow duplicate tool names in the same list', () => { + // Schema doesn't enforce uniqueness - that's application logic + const result = ToolPoliciesSchema.parse({ + alwaysAllow: ['tool1', 'tool1'], + alwaysDeny: [], + }); + expect(result.alwaysAllow).toEqual(['tool1', 'tool1']); + }); + + it('should allow same tool name in both lists', () => { + // Schema validation allows this - precedence is handled by application logic + const result = ToolPoliciesSchema.parse({ + alwaysAllow: ['tool1'], + alwaysDeny: ['tool1'], + }); + expect(result.alwaysAllow).toContain('tool1'); + expect(result.alwaysDeny).toContain('tool1'); + }); + + it('should reject extra fields with strict validation', () => { + const policiesWithExtra = { + alwaysAllow: [], + alwaysDeny: [], + extraField: 'should fail', + }; + + const result = ToolPoliciesSchema.safeParse(policiesWithExtra); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.code).toBe(z.ZodIssueCode.unrecognized_keys); + }); + }); + + describe('Type Safety', () => { + it('should have correct type inference', () => { + const result: ToolPolicies = ToolPoliciesSchema.parse({ + alwaysAllow: ['tool1'], + alwaysDeny: ['tool2'], + }); + expect(Array.isArray(result.alwaysAllow)).toBe(true); + expect(Array.isArray(result.alwaysDeny)).toBe(true); + }); + }); + + describe('Real-world Scenarios', () => { + it('should handle safe development configuration', () => { + const devPolicies = { + alwaysAllow: [ + 'internal--ask_user', + 'mcp--filesystem--read_file', + 'mcp--filesystem--list_directory', + ], + alwaysDeny: ['mcp--filesystem--write_file', 'mcp--filesystem--delete_file'], + }; + + const result = ToolPoliciesSchema.parse(devPolicies); + expect(result).toEqual(devPolicies); + }); + + it('should handle production security configuration', () => { + const prodPolicies = { + alwaysAllow: ['internal--ask_user'], + alwaysDeny: [ + 'mcp--filesystem--delete_file', + 'mcp--playwright--execute_script', + 'mcp--shell--execute', + ], + }; + + const result = ToolPoliciesSchema.parse(prodPolicies); + expect(result).toEqual(prodPolicies); + }); + + it('should handle minimal allow-only policy', () => { + const allowOnlyPolicy = { + alwaysAllow: ['internal--ask_user', 'mcp--filesystem--read_file'], + alwaysDeny: [], + }; + + const result = ToolPoliciesSchema.parse(allowOnlyPolicy); + expect(result.alwaysAllow).toHaveLength(2); + expect(result.alwaysDeny).toHaveLength(0); + }); + + it('should handle strict deny-only policy', () => { + const denyOnlyPolicy = { + alwaysAllow: [], + alwaysDeny: ['mcp--filesystem--delete_file', 'mcp--shell--execute'], + }; + + const result = ToolPoliciesSchema.parse(denyOnlyPolicy); + expect(result.alwaysAllow).toHaveLength(0); + expect(result.alwaysDeny).toHaveLength(2); + }); + }); +}); diff --git a/dexto/packages/core/src/tools/schemas.ts b/dexto/packages/core/src/tools/schemas.ts new file mode 100644 index 00000000..ba214c12 --- /dev/null +++ b/dexto/packages/core/src/tools/schemas.ts @@ -0,0 +1,186 @@ +import { z } from 'zod'; +import { INTERNAL_TOOL_NAMES } from './internal-tools/constants.js'; +import { customToolSchemaRegistry } from './custom-tool-schema-registry.js'; + +export const TOOL_CONFIRMATION_MODES = ['manual', 'auto-approve', 'auto-deny'] as const; +export type ToolConfirmationMode = (typeof TOOL_CONFIRMATION_MODES)[number]; + +export const ALLOWED_TOOLS_STORAGE_TYPES = ['memory', 'storage'] as const; +export type AllowedToolsStorageType = (typeof ALLOWED_TOOLS_STORAGE_TYPES)[number]; + +export const DEFAULT_TOOL_CONFIRMATION_MODE: ToolConfirmationMode = 'auto-approve'; +export const DEFAULT_ALLOWED_TOOLS_STORAGE: AllowedToolsStorageType = 'storage'; + +// Internal tools schema - separate for type derivation + +export const InternalToolsSchema = z + .array(z.enum(INTERNAL_TOOL_NAMES).describe('Available internal tool names')) + .default([]) + .describe( + `Array of internal tool names to enable. Empty array = disabled. Available tools: ${INTERNAL_TOOL_NAMES.join(', ')}` + ); +// Derive type from schema +export type InternalToolsConfig = z.output; + +/** + * Get the custom tool config schema based on registered providers. + * + * This function creates a discriminated union of all registered provider schemas, + * enabling early validation of provider-specific fields at config load time. + * + * IMPORTANT: Providers must be registered (via image imports or customToolRegistry) + * before config validation for early validation to work. If no providers are + * registered, falls back to passthrough schema for backward compatibility. + * + * @returns Discriminated union schema or passthrough schema + */ +function getCustomToolConfigSchema(): z.ZodType { + return customToolSchemaRegistry.createUnionSchema(); +} + +/** + * Custom tool configuration schema. + * + * This schema is built dynamically from registered providers: + * - If providers are registered → discriminated union with full validation + * - If no providers registered → passthrough schema (backward compatible) + * + * Provider-specific fields are validated based on their registered schemas. + */ +export const CustomToolConfigSchema = z.lazy(() => getCustomToolConfigSchema()); + +export type CustomToolConfig = z.output; + +/** + * OpenAPI-safe version of CustomToolConfigSchema. + * Uses a generic object schema instead of lazy loading for OpenAPI compatibility. + */ +export const CustomToolConfigSchemaForOpenAPI = z + .object({ + type: z.string().describe('Tool provider type identifier'), + }) + .passthrough() + .describe('Custom tool provider configuration (generic representation for OpenAPI docs)'); + +/** + * Array of custom tool provider configurations. + * + * Custom tools must be registered via customToolRegistry before loading agent config + * for early validation to work. If providers are not registered, validation will + * fall back to runtime validation by the provider. + */ +export const CustomToolsSchema = z + .array(CustomToolConfigSchema) + .default([]) + .describe( + 'Array of custom tool provider configurations. Providers are validated against registered schemas.' + ); + +export type CustomToolsConfig = z.output; + +/** + * OpenAPI-safe version of CustomToolsSchema. + * Uses generic object schema for OpenAPI compatibility. + */ +export const CustomToolsSchemaForOpenAPI = z + .array(CustomToolConfigSchemaForOpenAPI) + .default([]) + .describe('Array of custom tool provider configurations'); + +// Tool policies schema - static allow/deny lists for fine-grained control +export const ToolPoliciesSchema = z + .object({ + alwaysAllow: z + .array(z.string()) + .default([]) + .describe( + 'Tools that never require approval (low-risk). Use full qualified names (e.g., "internal--ask_user", "mcp--filesystem--read_file")' + ), + alwaysDeny: z + .array(z.string()) + .default([]) + .describe( + 'Tools that are always denied (high-risk). Takes precedence over alwaysAllow. Use full qualified names (e.g., "mcp--filesystem--delete_file")' + ), + }) + .strict() + .default({ alwaysAllow: [], alwaysDeny: [] }) + .describe('Static tool policies for allow/deny lists'); + +export type ToolPolicies = z.output; + +export const ToolConfirmationConfigSchema = z + .object({ + mode: z + .enum(TOOL_CONFIRMATION_MODES) + .default(DEFAULT_TOOL_CONFIRMATION_MODE) + .describe( + 'Tool confirmation mode: manual (interactive), auto-approve (all tools), auto-deny (no tools)' + ), + timeout: z + .number() + .int() + .positive() + .optional() + .describe( + 'Timeout for tool confirmation requests in milliseconds. If not set, waits indefinitely.' + ), + allowedToolsStorage: z + .enum(ALLOWED_TOOLS_STORAGE_TYPES) + .default(DEFAULT_ALLOWED_TOOLS_STORAGE) + .describe( + 'Storage type for remembered tool approvals: memory (session-only) or storage (persistent)' + ), + toolPolicies: ToolPoliciesSchema.describe( + 'Static tool policies for fine-grained allow/deny control. Deny list takes precedence over allow list.' + ), + }) + .strict() + .describe('Tool confirmation and approval configuration'); + +export type ToolConfirmationConfig = z.input; +export type ValidatedToolConfirmationConfig = z.output; + +// Elicitation configuration schema - independent from tool confirmation +export const ElicitationConfigSchema = z + .object({ + enabled: z + .boolean() + .default(false) + .describe( + 'Enable elicitation support (ask_user tool and MCP server elicitations). When disabled, elicitation requests will be rejected.' + ), + timeout: z + .number() + .int() + .positive() + .optional() + .describe( + 'Timeout for elicitation requests in milliseconds. If not set, waits indefinitely.' + ), + }) + .strict() + .describe( + 'Elicitation configuration for user input requests. Independent from tool confirmation mode, allowing auto-approve for tools while still supporting elicitation.' + ); + +export type ElicitationConfig = z.input; +export type ValidatedElicitationConfig = z.output; + +// Tool limits configuration +export const ToolLimitsSchema = z + .object({ + maxOutputChars: z + .number() + .optional() + .describe('Maximum number of characters for tool output'), + maxLines: z.number().optional().describe('Maximum number of lines for tool output'), + maxLineLength: z.number().optional().describe('Maximum length of a single line'), + }) + .strict(); + +export const ToolsConfigSchema = z + .record(ToolLimitsSchema) + .describe('Per-tool configuration limits'); + +export type ToolsConfig = z.output; diff --git a/dexto/packages/core/src/tools/tool-manager.integration.test.ts b/dexto/packages/core/src/tools/tool-manager.integration.test.ts new file mode 100644 index 00000000..dd9d283c --- /dev/null +++ b/dexto/packages/core/src/tools/tool-manager.integration.test.ts @@ -0,0 +1,640 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { ToolManager } from './tool-manager.js'; +import { MCPManager } from '../mcp/manager.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { ToolErrorCode } from './error-codes.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import type { InternalToolsServices } from './internal-tools/registry.js'; +import type { InternalToolsConfig } from './schemas.js'; +import type { IMCPClient } from '../mcp/types.js'; +import { AgentEventBus } from '../events/index.js'; +import { ApprovalManager } from '../approval/manager.js'; +import type { IAllowedToolsProvider } from './confirmation/allowed-tools-provider/types.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; + +// Mock logger +vi.mock('../logger/index.js', () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + getLevel: vi.fn().mockReturnValue('info'), + silly: vi.fn(), + }, +})); + +describe('ToolManager Integration Tests', () => { + let mcpManager: MCPManager; + let approvalManager: ApprovalManager; + let allowedToolsProvider: IAllowedToolsProvider; + let internalToolsServices: InternalToolsServices; + let internalToolsConfig: InternalToolsConfig; + let mockAgentEventBus: AgentEventBus; + const mockLogger = createMockLogger(); + + beforeEach(() => { + // Create real MCPManager + mcpManager = new MCPManager(mockLogger); + + // Create mock AgentEventBus + mockAgentEventBus = { + on: vi.fn(), + emit: vi.fn(), + off: vi.fn(), + once: vi.fn(), + removeAllListeners: vi.fn(), + } as any; + + // Create ApprovalManager in auto-approve mode for integration tests + approvalManager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + + // Create mock AllowedToolsProvider + allowedToolsProvider = { + isToolAllowed: vi.fn().mockResolvedValue(false), + allowTool: vi.fn().mockResolvedValue(undefined), + disallowTool: vi.fn().mockResolvedValue(undefined), + } as any; + + // Mock SearchService for internal tools + const mockSearchService = { + searchMessages: vi + .fn() + .mockResolvedValue([{ id: '1', content: 'test message', role: 'user' }]), + searchSessions: vi.fn().mockResolvedValue([{ id: 'session1', title: 'Test Session' }]), + } as any; + + internalToolsServices = { + searchService: mockSearchService, + }; + + internalToolsConfig = ['search_history']; + }); + + describe('End-to-End Tool Execution', () => { + it('should execute MCP tools through the complete pipeline', async () => { + // Create mock MCP client + const mockClient: IMCPClient = { + getTools: vi.fn().mockResolvedValue({ + test_tool: { + name: 'test_tool', + description: 'Test MCP tool', + parameters: { type: 'object', properties: {} }, + }, + }), + callTool: vi.fn().mockResolvedValue('mcp tool result'), + listPrompts: vi.fn().mockResolvedValue([]), + listResources: vi.fn().mockResolvedValue([]), + } as any; + + // Register mock client and update cache + mcpManager.registerClient('test-server', mockClient); + // Need to manually call updateClientCache since registerClient doesn't do it + await (mcpManager as any).updateClientCache('test-server', mockClient); + + // Create ToolManager with real components + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices: {}, + internalToolsConfig: [], + }, + mockLogger + ); + await toolManager.initialize(); + + // Execute tool through complete pipeline + const result = await toolManager.executeTool( + 'mcp--test_tool', + { param: 'value' }, + 'test-call-id' + ); + + expect(mockClient.callTool).toHaveBeenCalledWith('test_tool', { param: 'value' }); + expect(result).toEqual({ result: 'mcp tool result' }); + }); + + it('should execute internal tools through the complete pipeline', async () => { + // Create ToolManager with internal tools + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices, + internalToolsConfig, + }, + mockLogger + ); + + await toolManager.initialize(); + + // Execute internal tool + const result = await toolManager.executeTool( + 'internal--search_history', + { query: 'test query', mode: 'messages' }, + 'test-call-id' + ); + + expect(internalToolsServices.searchService?.searchMessages).toHaveBeenCalledWith( + 'test query', + expect.objectContaining({ + limit: 20, // Default from Zod schema + offset: 0, // Default from Zod schema + }) + ); + expect(result).toEqual({ + result: [{ id: '1', content: 'test message', role: 'user' }], + }); + }); + + it('should work with both MCP and internal tools together', async () => { + // Set up MCP tool + const mockClient: IMCPClient = { + getTools: vi.fn().mockResolvedValue({ + file_read: { + name: 'file_read', + description: 'Read file', + parameters: { type: 'object', properties: {} }, + }, + }), + callTool: vi.fn().mockResolvedValue('file content'), + listPrompts: vi.fn().mockResolvedValue([]), + listResources: vi.fn().mockResolvedValue([]), + } as any; + + mcpManager.registerClient('file-server', mockClient); + await (mcpManager as any).updateClientCache('file-server', mockClient); + + // Create ToolManager with both MCP and internal tools + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices, + internalToolsConfig, + }, + mockLogger + ); + + await toolManager.initialize(); + + // Get all tools - should include both types with proper prefixing + const allTools = await toolManager.getAllTools(); + + expect(allTools['mcp--file_read']).toBeDefined(); + expect(allTools['internal--search_history']).toBeDefined(); + expect(allTools['mcp--file_read']?.description).toContain('(via MCP servers)'); + expect(allTools['internal--search_history']?.description).toContain('(internal tool)'); + + // Execute both types + const mcpResult = await toolManager.executeTool( + 'mcp--file_read', + { path: '/test' }, + 'test-call-id-1' + ); + const internalResult = await toolManager.executeTool( + 'internal--search_history', + { query: 'search test', mode: 'sessions' }, + 'test-call-id-2' + ); + + expect(mcpResult).toEqual({ result: 'file content' }); + expect(internalResult).toEqual({ + result: [{ id: 'session1', title: 'Test Session' }], + }); + }); + }); + + describe('Confirmation Flow Integration', () => { + it('should work with auto-approve mode', async () => { + const autoApproveManager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + const mockClient: IMCPClient = { + getTools: vi.fn().mockResolvedValue({ + test_tool: { + name: 'test_tool', + description: 'Test tool', + parameters: { type: 'object', properties: {} }, + }, + }), + callTool: vi.fn().mockResolvedValue('approved result'), + listPrompts: vi.fn().mockResolvedValue([]), + listResources: vi.fn().mockResolvedValue([]), + } as any; + + const mcpMgr = new MCPManager(mockLogger); + mcpMgr.registerClient('test-server', mockClient); + await (mcpMgr as any).updateClientCache('test-server', mockClient); + + const toolManager = new ToolManager( + mcpMgr, + autoApproveManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices: {}, + internalToolsConfig: [], + }, + mockLogger + ); + const result = await toolManager.executeTool('mcp--test_tool', {}, 'test-call-id'); + + expect(result).toEqual({ result: 'approved result' }); + }); + + it('should work with auto-deny mode', async () => { + const autoDenyManager = new ApprovalManager( + { + toolConfirmation: { + mode: 'auto-deny', + timeout: 120000, + }, + elicitation: { + enabled: true, + timeout: 120000, + }, + }, + mockLogger + ); + const mockClient: IMCPClient = { + getTools: vi.fn().mockResolvedValue({ + test_tool: { + name: 'test_tool', + description: 'Test tool', + parameters: { type: 'object', properties: {} }, + }, + }), + callTool: vi.fn().mockResolvedValue('should not execute'), + listPrompts: vi.fn().mockResolvedValue([]), + listResources: vi.fn().mockResolvedValue([]), + } as any; + + const mcpMgr = new MCPManager(mockLogger); + mcpMgr.registerClient('test-server', mockClient); + await (mcpMgr as any).updateClientCache('test-server', mockClient); + + const toolManager = new ToolManager( + mcpMgr, + autoDenyManager, + allowedToolsProvider, + 'auto-deny', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices: {}, + internalToolsConfig: [], + }, + mockLogger + ); + + const error = (await toolManager + .executeTool('mcp--test_tool', {}, 'test-call-id') + .catch((e) => e)) as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(ToolErrorCode.EXECUTION_DENIED); + expect(error.scope).toBe(ErrorScope.TOOLS); + expect(error.type).toBe(ErrorType.FORBIDDEN); + + expect(mockClient.callTool).not.toHaveBeenCalled(); + }); + }); + + describe('Error Scenarios and Recovery', () => { + it('should handle MCP client failures gracefully', async () => { + const failingClient: IMCPClient = { + getTools: vi.fn().mockRejectedValue(new Error('MCP connection failed')), + callTool: vi.fn(), + listPrompts: vi.fn().mockResolvedValue([]), + listResources: vi.fn().mockResolvedValue([]), + } as any; + + mcpManager.registerClient('failing-server', failingClient); + + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices, + internalToolsConfig, + }, + mockLogger + ); + + await toolManager.initialize(); + + // Should still return internal tools even if MCP fails + const allTools = await toolManager.getAllTools(); + expect(allTools['internal--search_history']).toBeDefined(); + expect(Object.keys(allTools).filter((name) => name.startsWith('mcp--'))).toHaveLength( + 0 + ); + }); + + it('should handle internal tools initialization failures gracefully', async () => { + // Mock services that will cause tool initialization to fail + const failingServices: InternalToolsServices = { + // Missing searchService - should cause search_history tool to be skipped + }; + + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices: failingServices, + internalToolsConfig, + }, + mockLogger + ); + + await toolManager.initialize(); + + const allTools = await toolManager.getAllTools(); + // Should not have any internal tools since searchService is missing + expect( + Object.keys(allTools).filter((name) => name.startsWith('internal--')) + ).toHaveLength(0); + }); + + it('should handle tool execution failures properly', async () => { + const failingClient: IMCPClient = { + getTools: vi.fn().mockResolvedValue({ + failing_tool: { + name: 'failing_tool', + description: 'Tool that fails', + parameters: { type: 'object', properties: {} }, + }, + }), + callTool: vi.fn().mockRejectedValue(new Error('Tool execution failed')), + listPrompts: vi.fn().mockResolvedValue([]), + listResources: vi.fn().mockResolvedValue([]), + } as any; + + mcpManager.registerClient('failing-server', failingClient); + await (mcpManager as any).updateClientCache('failing-server', failingClient); + + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices: {}, + internalToolsConfig: [], + }, + mockLogger + ); + + await expect( + toolManager.executeTool('mcp--failing_tool', {}, 'test-call-id') + ).rejects.toThrow(Error); + }); + + it('should handle internal tool execution failures properly', async () => { + // Mock SearchService to throw error + const failingSearchService = { + searchMessages: vi.fn().mockRejectedValue(new Error('Search service failed')), + searchSessions: vi.fn().mockRejectedValue(new Error('Search service failed')), + } as any; + + const failingServices: InternalToolsServices = { + searchService: failingSearchService, + }; + + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices: failingServices, + internalToolsConfig, + }, + mockLogger + ); + + await toolManager.initialize(); + + await expect( + toolManager.executeTool( + 'internal--search_history', + { query: 'test', mode: 'messages' }, + 'test-call-id' + ) + ).rejects.toThrow(Error); + }); + }); + + describe('Performance and Caching', () => { + it('should cache tool discovery results efficiently', async () => { + const mockClient: IMCPClient = { + getTools: vi.fn().mockResolvedValue({ + test_tool: { + name: 'test_tool', + description: 'Test tool', + parameters: { type: 'object', properties: {} }, + }, + }), + callTool: vi.fn(), + listPrompts: vi.fn().mockResolvedValue([]), + listResources: vi.fn().mockResolvedValue([]), + } as any; + + mcpManager.registerClient('test-server', mockClient); + await (mcpManager as any).updateClientCache('test-server', mockClient); + + // MCP client's getTools gets called during updateClientCache (1) + expect(mockClient.getTools).toHaveBeenCalledTimes(1); + vi.mocked(mockClient.getTools).mockClear(); + + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices, + internalToolsConfig, + }, + mockLogger + ); + + await toolManager.initialize(); + + // Multiple calls to getAllTools should use cache + await toolManager.getAllTools(); + await toolManager.getAllTools(); + await toolManager.getAllTools(); + + // With new architecture: MCPManager caches tools during updateClientCache + // So mockClient.getTools is NOT called again by toolManager.getAllTools() + expect(mockClient.getTools).toHaveBeenCalledTimes(0); + }); + + it('should refresh cache when requested', async () => { + const mockClient: IMCPClient = { + getTools: vi.fn().mockResolvedValue({ + test_tool: { + name: 'test_tool', + description: 'Test tool', + parameters: { type: 'object', properties: {} }, + }, + }), + callTool: vi.fn(), + listPrompts: vi.fn().mockResolvedValue([]), + listResources: vi.fn().mockResolvedValue([]), + } as any; + + mcpManager.registerClient('test-server', mockClient); + await (mcpManager as any).updateClientCache('test-server', mockClient); + expect(mockClient.getTools).toHaveBeenCalledTimes(1); + vi.mocked(mockClient.getTools).mockClear(); + + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices: {}, + internalToolsConfig: [], + }, + mockLogger + ); + + // First call uses MCPManager's cache (no client call) + await toolManager.getAllTools(); + expect(mockClient.getTools).toHaveBeenCalledTimes(0); + + // ToolManager.refresh() now cascades to MCPManager.refresh() + // This refreshes server capabilities by calling client.getTools() again + await toolManager.refresh(); + expect(mockClient.getTools).toHaveBeenCalledTimes(1); + vi.mocked(mockClient.getTools).mockClear(); + + // Multiple calls after refresh still use cache + await toolManager.getAllTools(); + await toolManager.getAllTools(); + expect(mockClient.getTools).toHaveBeenCalledTimes(0); + }); + }); + + describe('Session ID Handling', () => { + it('should pass sessionId through the complete execution pipeline', async () => { + const mockClient: IMCPClient = { + getTools: vi.fn().mockResolvedValue({ + test_tool: { + name: 'test_tool', + description: 'Test tool', + parameters: { type: 'object', properties: {} }, + }, + }), + callTool: vi.fn().mockResolvedValue('result'), + listPrompts: vi.fn().mockResolvedValue([]), + listResources: vi.fn().mockResolvedValue([]), + } as any; + + mcpManager.registerClient('test-server', mockClient); + await (mcpManager as any).updateClientCache('test-server', mockClient); + + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { + internalToolsServices, + internalToolsConfig, + }, + mockLogger + ); + + await toolManager.initialize(); + + const sessionId = 'test-session-123'; + + // Execute MCP tool with sessionId + await toolManager.executeTool( + 'mcp--test_tool', + { param: 'value' }, + 'test-call-id-1', + sessionId + ); + + // Execute internal tool with sessionId + await toolManager.executeTool( + 'internal--search_history', + { query: 'test', mode: 'messages' }, + 'test-call-id-2', + sessionId + ); + + // Verify MCP tool received sessionId (note: MCPManager doesn't use sessionId in callTool currently) + expect(mockClient.callTool).toHaveBeenCalledWith('test_tool', { param: 'value' }); + + // Verify internal tool was called with proper defaults + expect(internalToolsServices.searchService?.searchMessages).toHaveBeenCalledWith( + 'test', + expect.objectContaining({ + limit: 20, // Default from Zod schema + offset: 0, // Default from Zod schema + }) + ); + }); + }); +}); diff --git a/dexto/packages/core/src/tools/tool-manager.test.ts b/dexto/packages/core/src/tools/tool-manager.test.ts new file mode 100644 index 00000000..7e83e428 --- /dev/null +++ b/dexto/packages/core/src/tools/tool-manager.test.ts @@ -0,0 +1,1607 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { ToolManager } from './tool-manager.js'; +import { MCPManager } from '../mcp/manager.js'; +import { DextoRuntimeError } from '../errors/DextoRuntimeError.js'; +import { ToolErrorCode } from './error-codes.js'; +import { ErrorScope, ErrorType } from '../errors/types.js'; +import { AgentEventBus } from '../events/index.js'; +import type { ApprovalManager } from '../approval/manager.js'; +import type { IAllowedToolsProvider } from './confirmation/allowed-tools-provider/types.js'; +import { ApprovalStatus } from '../approval/types.js'; +import { createMockLogger } from '../logger/v2/test-utils.js'; + +// Mock logger +vi.mock('../logger/index.js', () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +describe('ToolManager - Unit Tests (Pure Logic)', () => { + let mockMcpManager: MCPManager; + let mockApprovalManager: ApprovalManager; + let mockAllowedToolsProvider: IAllowedToolsProvider; + let mockAgentEventBus: AgentEventBus; + const mockLogger = createMockLogger(); + + beforeEach(() => { + mockMcpManager = { + getAllTools: vi.fn(), + executeTool: vi.fn(), + getToolClient: vi.fn(), + refresh: vi.fn().mockResolvedValue(undefined), + } as any; + + mockApprovalManager = { + requestToolConfirmation: vi.fn().mockResolvedValue({ + approvalId: 'test-approval-id', + status: ApprovalStatus.APPROVED, + data: { rememberChoice: false }, + }), + getPendingApprovals: vi.fn().mockReturnValue([]), + cancelApproval: vi.fn(), + cancelAllApprovals: vi.fn(), + } as any; + + mockAllowedToolsProvider = { + isToolAllowed: vi.fn().mockResolvedValue(false), + allowTool: vi.fn().mockResolvedValue(undefined), + disallowTool: vi.fn().mockResolvedValue(undefined), + } as any; + + mockAgentEventBus = { + on: vi.fn(), + emit: vi.fn(), + off: vi.fn(), + once: vi.fn(), + removeAllListeners: vi.fn(), + } as any; + + vi.clearAllMocks(); + }); + + describe('Tool Source Detection Logic', () => { + it('should correctly identify MCP tools', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + expect(toolManager.getToolSource('mcp--file_read')).toBe('mcp'); + expect(toolManager.getToolSource('mcp--web_search')).toBe('mcp'); + }); + + it('should correctly identify internal tools', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + expect(toolManager.getToolSource('internal--search_history')).toBe('internal'); + expect(toolManager.getToolSource('internal--config_manager')).toBe('internal'); + }); + + it('should identify unknown tools', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + expect(toolManager.getToolSource('invalid_tool')).toBe('unknown'); + expect(toolManager.getToolSource('file_read')).toBe('unknown'); // No prefix + expect(toolManager.getToolSource('')).toBe('unknown'); // Empty + }); + + it('should handle edge cases with empty tool names', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + expect(toolManager.getToolSource('mcp--')).toBe('unknown'); // Prefix but no name + expect(toolManager.getToolSource('internal--')).toBe('unknown'); // Prefix but no name + }); + }); + + describe('Tool Name Parsing Logic', () => { + it('should extract actual tool name from MCP prefix', () => { + const prefixedName = 'mcp--file_read'; + const actualName = prefixedName.substring('mcp--'.length); + expect(actualName).toBe('file_read'); + }); + + it('should extract actual tool name from internal prefix', () => { + const prefixedName = 'internal--search_history'; + const actualName = prefixedName.substring('internal--'.length); + expect(actualName).toBe('search_history'); + }); + + it('should handle complex tool names', () => { + const complexName = 'mcp--complex_tool_name_with_underscores'; + const actualName = complexName.substring('mcp--'.length); + expect(actualName).toBe('complex_tool_name_with_underscores'); + }); + }); + + describe('Tool Validation Logic', () => { + it('should reject tools without proper prefix', async () => { + mockMcpManager.getAllTools = vi.fn().mockResolvedValue({}); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const error = (await toolManager + .executeTool('invalid_tool', {}, 'test-call-id') + .catch((e) => e)) as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(ToolErrorCode.TOOL_NOT_FOUND); + expect(error.scope).toBe(ErrorScope.TOOLS); + expect(error.type).toBe(ErrorType.NOT_FOUND); + }); + + it('should reject tools with prefix but no name', async () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const mcpError = (await toolManager + .executeTool('mcp--', {}, 'test-call-id') + .catch((e) => e)) as DextoRuntimeError; + expect(mcpError).toBeInstanceOf(DextoRuntimeError); + expect(mcpError.code).toBe(ToolErrorCode.TOOL_INVALID_ARGS); + expect(mcpError.scope).toBe(ErrorScope.TOOLS); + expect(mcpError.type).toBe(ErrorType.USER); + + const internalError = (await toolManager + .executeTool('internal--', {}, 'test-call-id') + .catch((e) => e)) as DextoRuntimeError; + expect(internalError).toBeInstanceOf(DextoRuntimeError); + expect(internalError.code).toBe(ToolErrorCode.TOOL_INVALID_ARGS); + expect(internalError.scope).toBe(ErrorScope.TOOLS); + expect(internalError.type).toBe(ErrorType.USER); + + // Should NOT call the underlying managers + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + + it('should reject internal tools when provider not initialized', async () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await expect( + toolManager.executeTool('internal--search_history', {}, 'test-call-id') + ).rejects.toThrow( + 'Internal tools not initialized, cannot execute: internal--search_history' + ); + }); + }); + + describe('Confirmation Flow Logic', () => { + it('should request approval via ApprovalManager with correct parameters', async () => { + mockMcpManager.executeTool = vi.fn().mockResolvedValue('result'); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await toolManager.executeTool( + 'mcp--file_read', + { path: '/test' }, + 'call-123', + 'session123' + ); + + expect(mockApprovalManager.requestToolConfirmation).toHaveBeenCalledWith({ + toolName: 'mcp--file_read', + toolCallId: 'call-123', + args: { path: '/test' }, + sessionId: 'session123', + }); + }); + + it('should request approval without sessionId when not provided', async () => { + mockMcpManager.executeTool = vi.fn().mockResolvedValue('result'); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await toolManager.executeTool('mcp--file_read', { path: '/test' }, 'call-456'); + + expect(mockApprovalManager.requestToolConfirmation).toHaveBeenCalledWith({ + toolName: 'mcp--file_read', + toolCallId: 'call-456', + args: { path: '/test' }, + }); + }); + + it('should throw execution denied error when approval denied', async () => { + mockApprovalManager.requestToolConfirmation = vi.fn().mockResolvedValue({ + approvalId: 'test-approval-id', + status: ApprovalStatus.DENIED, + }); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const error = (await toolManager + .executeTool('mcp--file_read', { path: '/test' }, 'test-call-id', 'session123') + .catch((e) => e)) as DextoRuntimeError; + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(ToolErrorCode.EXECUTION_DENIED); + expect(error.scope).toBe(ErrorScope.TOOLS); + expect(error.type).toBe(ErrorType.FORBIDDEN); + + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + + it('should proceed with execution when approval granted', async () => { + mockMcpManager.executeTool = vi.fn().mockResolvedValue('success'); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--file_read', + { path: '/test' }, + 'test-call-id' + ); + + expect(mockMcpManager.executeTool).toHaveBeenCalledWith( + 'file_read', + { path: '/test' }, + undefined + ); + expect(result).toEqual({ + result: 'success', + requireApproval: true, + approvalStatus: 'approved', + }); + }); + + it('should skip confirmation for tools in allowed list', async () => { + mockAllowedToolsProvider.isToolAllowed = vi.fn().mockResolvedValue(true); + mockMcpManager.executeTool = vi.fn().mockResolvedValue('success'); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--file_read', + { path: '/test' }, + 'test-call-id' + ); + + expect(mockAllowedToolsProvider.isToolAllowed).toHaveBeenCalledWith( + 'mcp--file_read', + undefined + ); + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + expect(result).toEqual({ result: 'success' }); + }); + + it('should auto-approve when mode is auto-approve', async () => { + mockMcpManager.executeTool = vi.fn().mockResolvedValue('success'); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--file_read', + { path: '/test' }, + 'test-call-id' + ); + + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + expect(mockMcpManager.executeTool).toHaveBeenCalled(); + expect(result).toEqual({ result: 'success' }); + }); + + it('should auto-deny when mode is auto-deny', async () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'auto-deny', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const error = (await toolManager + .executeTool('mcp--file_read', { path: '/test' }, 'test-call-id') + .catch((e) => e)) as DextoRuntimeError; + + expect(error).toBeInstanceOf(DextoRuntimeError); + expect(error.code).toBe(ToolErrorCode.EXECUTION_DENIED); + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + }); + + describe('Cache Management Logic', () => { + it('should cache tool discovery results', async () => { + const tools = { + test_tool: { name: 'test_tool', description: 'Test', parameters: {} }, + }; + mockMcpManager.getAllTools = vi.fn().mockResolvedValue(tools); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + // First call + await toolManager.getAllTools(); + // Second call should use cache + await toolManager.getAllTools(); + + expect(mockMcpManager.getAllTools).toHaveBeenCalledTimes(1); + }); + + it('should invalidate cache on refresh', async () => { + const tools = { + test_tool: { name: 'test_tool', description: 'Test', parameters: {} }, + }; + mockMcpManager.getAllTools = vi.fn().mockResolvedValue(tools); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + // First call + await toolManager.getAllTools(); + + // Refresh should invalidate cache + await toolManager.refresh(); + + // Second call should fetch again + await toolManager.getAllTools(); + + expect(mockMcpManager.getAllTools).toHaveBeenCalledTimes(2); + }); + }); + + describe('Tool Statistics Logic', () => { + it('should calculate statistics correctly', async () => { + const mcpTools = { + tool1: { name: 'tool1', description: 'Tool 1', parameters: {} }, + tool2: { name: 'tool2', description: 'Tool 2', parameters: {} }, + }; + + mockMcpManager.getAllTools = vi.fn().mockResolvedValue(mcpTools); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const stats = await toolManager.getToolStats(); + + expect(stats).toEqual({ + total: 2, + mcp: 2, + internal: 0, + custom: 0, + }); + }); + + it('should handle empty tool sets', async () => { + mockMcpManager.getAllTools = vi.fn().mockResolvedValue({}); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const stats = await toolManager.getToolStats(); + + expect(stats).toEqual({ + total: 0, + mcp: 0, + internal: 0, + custom: 0, + }); + }); + + it('should handle MCP errors gracefully in statistics', async () => { + mockMcpManager.getAllTools = vi.fn().mockRejectedValue(new Error('MCP failed')); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const stats = await toolManager.getToolStats(); + + expect(stats).toEqual({ + total: 0, + mcp: 0, + internal: 0, + custom: 0, + }); + }); + }); + + describe('Tool Existence Checking Logic', () => { + it('should check MCP tool existence correctly', async () => { + mockMcpManager.getToolClient = vi.fn().mockReturnValue({}); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const exists = await toolManager.hasTool('mcp--file_read'); + + expect(mockMcpManager.getToolClient).toHaveBeenCalledWith('file_read'); + expect(exists).toBe(true); + }); + + it('should return false for non-existent MCP tools', async () => { + mockMcpManager.getToolClient = vi.fn().mockReturnValue(undefined); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const exists = await toolManager.hasTool('mcp--nonexistent'); + + expect(exists).toBe(false); + }); + + it('should return false for tools without proper prefix', async () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const exists = await toolManager.hasTool('invalid_tool'); + + expect(exists).toBe(false); + }); + }); + + describe('Error Propagation Logic', () => { + it('should propagate MCP tool execution errors', async () => { + const executionError = new Error('Tool execution failed'); + mockMcpManager.executeTool = vi.fn().mockRejectedValue(executionError); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await expect( + toolManager.executeTool('mcp--file_read', { path: '/test' }, 'test-call-id') + ).rejects.toThrow('Tool execution failed'); + }); + + it('should propagate approval manager errors', async () => { + const approvalError = new Error('Approval request failed'); + mockApprovalManager.requestToolConfirmation = vi.fn().mockRejectedValue(approvalError); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await expect( + toolManager.executeTool('mcp--file_read', { path: '/test' }, 'test-call-id') + ).rejects.toThrow('Approval request failed'); + }); + }); + + describe('Tool Policies (Allow/Deny Lists)', () => { + beforeEach(() => { + // Reset mocks for policy tests + mockMcpManager.executeTool = vi.fn().mockResolvedValue('success'); + mockAllowedToolsProvider.isToolAllowed = vi.fn().mockResolvedValue(false); + }); + + describe('Precedence Logic', () => { + it('should deny tools in alwaysDeny list (highest precedence)', async () => { + const toolPolicies = { + alwaysAllow: [], + alwaysDeny: ['mcp--filesystem--delete_file'], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await expect( + toolManager.executeTool( + 'mcp--filesystem--delete_file', + { path: '/test' }, + 'test-call-id' + ) + ).rejects.toThrow(); + + // Should not reach approval manager or execution + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + + it('should deny even if tool is in alwaysAllow list', async () => { + const toolPolicies = { + alwaysAllow: ['mcp--filesystem--delete_file'], + alwaysDeny: ['mcp--filesystem--delete_file'], // Deny takes precedence + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await expect( + toolManager.executeTool( + 'mcp--filesystem--delete_file', + { path: '/test' }, + 'test-call-id' + ) + ).rejects.toThrow(); + + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + + it('should allow tools in alwaysAllow list without confirmation', async () => { + const toolPolicies = { + alwaysAllow: ['mcp--filesystem--read_file'], + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--filesystem--read_file', + { path: '/test' }, + 'test-call-id' + ); + + expect(result).toEqual({ result: 'success' }); + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + expect(mockMcpManager.executeTool).toHaveBeenCalledWith( + 'filesystem--read_file', + { + path: '/test', + }, + undefined + ); + }); + + it('should check dynamic allowed list after static policies', async () => { + mockAllowedToolsProvider.isToolAllowed = vi.fn().mockResolvedValue(true); + + const toolPolicies = { + alwaysAllow: [], + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--filesystem--read_file', + { path: '/test' }, + 'test-call-id' + ); + + expect(result).toEqual({ result: 'success' }); + expect(mockAllowedToolsProvider.isToolAllowed).toHaveBeenCalledWith( + 'mcp--filesystem--read_file', + undefined + ); + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + }); + + it('should fall back to approval mode when no policies match', async () => { + mockApprovalManager.requestToolConfirmation = vi.fn().mockResolvedValue({ + approvalId: 'test-approval', + status: 'approved', + data: {}, + }); + + const toolPolicies = { + alwaysAllow: ['internal--ask_user'], + alwaysDeny: ['mcp--filesystem--delete_file'], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--filesystem--read_file', + { path: '/test' }, + 'test-call-id' + ); + + expect(result).toEqual({ + result: 'success', + requireApproval: true, + approvalStatus: 'approved', + }); + expect(mockApprovalManager.requestToolConfirmation).toHaveBeenCalled(); + }); + }); + + describe('Auto-approve with Deny List', () => { + it('should deny tools in alwaysDeny even with auto-approve mode', async () => { + const toolPolicies = { + alwaysAllow: [], + alwaysDeny: ['mcp--filesystem--delete_file'], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await expect( + toolManager.executeTool( + 'mcp--filesystem--delete_file', + { path: '/test' }, + 'test-call-id' + ) + ).rejects.toThrow(); + + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + + it('should auto-approve tools not in deny list', async () => { + const toolPolicies = { + alwaysAllow: [], + alwaysDeny: ['mcp--filesystem--delete_file'], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'auto-approve', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--filesystem--read_file', + { path: '/test' }, + 'test-call-id' + ); + + expect(result).toEqual({ result: 'success' }); + expect(mockMcpManager.executeTool).toHaveBeenCalled(); + }); + }); + + describe('Auto-deny with Allow List', () => { + it('should allow tools in alwaysAllow even with auto-deny mode', async () => { + const toolPolicies = { + alwaysAllow: ['mcp--filesystem--read_file'], + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'auto-deny', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--filesystem--read_file', + { path: '/test' }, + 'test-call-id' + ); + + expect(result).toEqual({ result: 'success' }); + expect(mockMcpManager.executeTool).toHaveBeenCalled(); + }); + + it('should auto-deny tools not in allow list', async () => { + const toolPolicies = { + alwaysAllow: ['mcp--filesystem--read_file'], + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'auto-deny', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await expect( + toolManager.executeTool( + 'mcp--filesystem--write_file', + { path: '/test' }, + 'test-call-id' + ) + ).rejects.toThrow(); + + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + }); + + describe('No Policies Configured', () => { + it('should work normally when no policies are provided', async () => { + mockApprovalManager.requestToolConfirmation = vi.fn().mockResolvedValue({ + approvalId: 'test-approval', + status: 'approved', + data: {}, + }); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, // No policies + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--filesystem--read_file', + { path: '/test' }, + 'test-call-id' + ); + + expect(result).toEqual({ + result: 'success', + requireApproval: true, + approvalStatus: 'approved', + }); + expect(mockApprovalManager.requestToolConfirmation).toHaveBeenCalled(); + }); + + it('should work normally when empty policies are provided', async () => { + mockApprovalManager.requestToolConfirmation = vi.fn().mockResolvedValue({ + approvalId: 'test-approval', + status: 'approved', + data: {}, + }); + + const toolPolicies = { + alwaysAllow: [], + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--filesystem--read_file', + { path: '/test' }, + 'test-call-id' + ); + + expect(result).toEqual({ + result: 'success', + requireApproval: true, + approvalStatus: 'approved', + }); + expect(mockApprovalManager.requestToolConfirmation).toHaveBeenCalled(); + }); + }); + + describe('Internal Tools with Policies', () => { + it('should respect policies for internal tools', async () => { + const toolPolicies = { + alwaysAllow: ['internal--ask_user'], + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { + internalToolsServices: {}, + internalToolsConfig: ['ask_user'], + }, + mockLogger + ); + + // Should not throw since internal tools provider will be initialized + // This tests that the policy check happens before tool routing + expect(toolManager).toBeDefined(); + }); + }); + + describe('Dual Matching (Exact + Suffix)', () => { + beforeEach(() => { + mockMcpManager.executeTool = vi.fn().mockResolvedValue('success'); + mockAllowedToolsProvider.isToolAllowed = vi.fn().mockResolvedValue(false); + }); + + it('should match exact tool names in allow list', async () => { + const toolPolicies = { + alwaysAllow: ['mcp--read_file'], + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const result = await toolManager.executeTool( + 'mcp--read_file', + { path: '/test' }, + 'test-call-id' + ); + + expect(result).toEqual({ result: 'success' }); + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + }); + + it('should match qualified names with suffix matching in allow list', async () => { + const toolPolicies = { + alwaysAllow: ['mcp--read_file'], // Simple policy + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + // Tool with server prefix should match simple policy + const result = await toolManager.executeTool( + 'mcp--filesystem--read_file', + { path: '/test' }, + 'test-call-id' + ); + + expect(result).toEqual({ result: 'success' }); + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + expect(mockMcpManager.executeTool).toHaveBeenCalledWith( + 'filesystem--read_file', + { + path: '/test', + }, + undefined + ); + }); + + it('should match exact tool names in deny list', async () => { + const toolPolicies = { + alwaysAllow: [], + alwaysDeny: ['mcp--delete_file'], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + await expect( + toolManager.executeTool('mcp--delete_file', { path: '/test' }, 'test-call-id') + ).rejects.toThrow(); + + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + + it('should match qualified names with suffix matching in deny list', async () => { + const toolPolicies = { + alwaysAllow: [], + alwaysDeny: ['mcp--delete_file'], // Simple policy + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + // Tool with server prefix should match simple policy + await expect( + toolManager.executeTool( + 'mcp--filesystem--delete_file', + { path: '/test' }, + 'test-call-id' + ) + ).rejects.toThrow(); + + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + + it('should not match unrelated tools with similar names', async () => { + mockApprovalManager.requestToolConfirmation = vi.fn().mockResolvedValue({ + approvalId: 'test-approval', + status: 'approved', + data: {}, + }); + + const toolPolicies = { + alwaysAllow: ['mcp--read_file'], + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + // This should NOT match because it doesn't end with --read_file + const result = await toolManager.executeTool( + 'mcp--read_file_metadata', + {}, + 'test-call-id' + ); + + expect(result).toEqual({ + result: 'success', + requireApproval: true, + approvalStatus: 'approved', + }); + // Should require approval since it doesn't match the policy + expect(mockApprovalManager.requestToolConfirmation).toHaveBeenCalled(); + }); + + it('should only apply suffix matching to MCP tools, not internal tools', async () => { + mockApprovalManager.requestToolConfirmation = vi.fn().mockResolvedValue({ + approvalId: 'test-approval', + status: 'approved', + data: {}, + }); + + const toolPolicies = { + // Policy for a simple tool name + alwaysAllow: ['mcp--read_file'], + alwaysDeny: [], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + // MCP tool with suffix should match (suffix matching works) + await toolManager.executeTool('mcp--filesystem--read_file', {}, 'test-call-id'); + expect(mockMcpManager.executeTool).toHaveBeenLastCalledWith( + 'filesystem--read_file', + {}, + undefined + ); + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + + // But if an internal tool had a similar pattern, it shouldn't match via suffix + // (This is conceptual - internal tools don't have server prefixes in practice) + // The point is that suffix matching logic only applies to mcp-- prefixed patterns + }); + + it('should handle multiple policies with mixed matching', async () => { + mockApprovalManager.requestToolConfirmation = vi.fn().mockResolvedValue({ + approvalId: 'test-approval', + status: 'approved', + data: {}, + }); + + const toolPolicies = { + alwaysAllow: ['mcp--read_file', 'mcp--list_directory', 'internal--ask_user'], + alwaysDeny: ['mcp--delete_file', 'mcp--execute_script'], + }; + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + toolPolicies, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + // Test various matching scenarios + await toolManager.executeTool('mcp--read_file', {}, 'test-call-id'); + expect(mockMcpManager.executeTool).toHaveBeenLastCalledWith( + 'read_file', + {}, + undefined + ); + + await toolManager.executeTool('mcp--filesystem--read_file', {}, 'test-call-id'); + expect(mockMcpManager.executeTool).toHaveBeenLastCalledWith( + 'filesystem--read_file', + {}, + undefined + ); + + await toolManager.executeTool('mcp--server2--list_directory', {}, 'test-call-id'); + expect(mockMcpManager.executeTool).toHaveBeenLastCalledWith( + 'server2--list_directory', + {}, + undefined + ); + + // Deny should still work + await expect( + toolManager.executeTool('mcp--filesystem--delete_file', {}, 'test-call-id') + ).rejects.toThrow(); + + // None of these should have triggered approval + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + }); + }); + }); + + describe('Session Auto-Approve Tools (Skill allowed-tools)', () => { + describe('Basic CRUD Operations', () => { + it('should set and get session auto-approve tools', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const sessionId = 'test-session-123'; + const tools = ['internal--bash', 'mcp--read_file']; + + toolManager.setSessionAutoApproveTools(sessionId, tools); + + expect(toolManager.hasSessionAutoApproveTools(sessionId)).toBe(true); + expect(toolManager.getSessionAutoApproveTools(sessionId)).toEqual(tools); + }); + + it('should return false/undefined for non-existent sessions', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + expect(toolManager.hasSessionAutoApproveTools('non-existent')).toBe(false); + expect(toolManager.getSessionAutoApproveTools('non-existent')).toBeUndefined(); + }); + + it('should clear session auto-approve tools', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const sessionId = 'test-session-123'; + toolManager.setSessionAutoApproveTools(sessionId, ['internal--bash']); + + expect(toolManager.hasSessionAutoApproveTools(sessionId)).toBe(true); + + toolManager.clearSessionAutoApproveTools(sessionId); + + expect(toolManager.hasSessionAutoApproveTools(sessionId)).toBe(false); + expect(toolManager.getSessionAutoApproveTools(sessionId)).toBeUndefined(); + }); + + it('should handle multiple sessions independently', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const session1 = 'session-1'; + const session2 = 'session-2'; + + toolManager.setSessionAutoApproveTools(session1, ['internal--bash']); + toolManager.setSessionAutoApproveTools(session2, [ + 'mcp--read_file', + 'mcp--write_file', + ]); + + expect(toolManager.getSessionAutoApproveTools(session1)).toEqual([ + 'internal--bash', + ]); + expect(toolManager.getSessionAutoApproveTools(session2)).toEqual([ + 'mcp--read_file', + 'mcp--write_file', + ]); + + // Clearing one session should not affect the other + toolManager.clearSessionAutoApproveTools(session1); + + expect(toolManager.hasSessionAutoApproveTools(session1)).toBe(false); + expect(toolManager.hasSessionAutoApproveTools(session2)).toBe(true); + }); + + it('should overwrite existing tools when setting again', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const sessionId = 'test-session'; + toolManager.setSessionAutoApproveTools(sessionId, ['tool1']); + toolManager.setSessionAutoApproveTools(sessionId, ['tool2', 'tool3']); + + expect(toolManager.getSessionAutoApproveTools(sessionId)).toEqual([ + 'tool2', + 'tool3', + ]); + }); + + it('should clear auto-approvals when setting empty array', () => { + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const sessionId = 'test-session'; + + // First set some tools + toolManager.setSessionAutoApproveTools(sessionId, ['internal--bash']); + expect(toolManager.hasSessionAutoApproveTools(sessionId)).toBe(true); + + // Setting empty array should clear auto-approvals + toolManager.setSessionAutoApproveTools(sessionId, []); + + expect(toolManager.hasSessionAutoApproveTools(sessionId)).toBe(false); + expect(toolManager.getSessionAutoApproveTools(sessionId)).toBeUndefined(); + }); + }); + + describe('Auto-Approve Precedence', () => { + it('should auto-approve tools in session auto-approve list', async () => { + (mockMcpManager.getAllTools as ReturnType).mockResolvedValue({ + test_tool: { + name: 'test_tool', + description: 'A test tool', + inputSchema: {}, + }, + }); + (mockMcpManager.executeTool as ReturnType).mockResolvedValue( + 'success' + ); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', // Manual mode - normally requires approval + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const sessionId = 'test-session'; + toolManager.setSessionAutoApproveTools(sessionId, ['mcp--test_tool']); + + // Execute tool with sessionId + await toolManager.executeTool('mcp--test_tool', {}, 'call-1', sessionId); + + // Should NOT have requested approval (auto-approved by session config) + expect(mockApprovalManager.requestToolConfirmation).not.toHaveBeenCalled(); + expect(mockMcpManager.executeTool).toHaveBeenCalledWith('test_tool', {}, sessionId); + }); + + it('should still require approval for tools NOT in session auto-approve list', async () => { + (mockMcpManager.getAllTools as ReturnType).mockResolvedValue({ + allowed_tool: { + name: 'allowed_tool', + description: 'Allowed tool', + inputSchema: {}, + }, + other_tool: { + name: 'other_tool', + description: 'Other tool', + inputSchema: {}, + }, + }); + (mockMcpManager.executeTool as ReturnType).mockResolvedValue( + 'success' + ); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const sessionId = 'test-session'; + toolManager.setSessionAutoApproveTools(sessionId, ['mcp--allowed_tool']); + + // Execute a tool NOT in the auto-approve list + await toolManager.executeTool('mcp--other_tool', {}, 'call-1', sessionId); + + // Should have requested approval + expect(mockApprovalManager.requestToolConfirmation).toHaveBeenCalled(); + }); + + it('should respect alwaysDeny even if tool is in session auto-approve', async () => { + (mockMcpManager.getAllTools as ReturnType).mockResolvedValue({ + dangerous_tool: { + name: 'dangerous_tool', + description: 'A dangerous tool', + inputSchema: {}, + }, + }); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: ['mcp--dangerous_tool'] }, // In deny list (full qualified name) + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + const sessionId = 'test-session'; + toolManager.setSessionAutoApproveTools(sessionId, ['mcp--dangerous_tool']); + + // Should throw because alwaysDeny takes precedence + await expect( + toolManager.executeTool('mcp--dangerous_tool', {}, 'call-1', sessionId) + ).rejects.toThrow(); + + expect(mockMcpManager.executeTool).not.toHaveBeenCalled(); + }); + + it('should not auto-approve if sessionId does not match', async () => { + (mockMcpManager.getAllTools as ReturnType).mockResolvedValue({ + test_tool: { + name: 'test_tool', + description: 'A test tool', + inputSchema: {}, + }, + }); + (mockMcpManager.executeTool as ReturnType).mockResolvedValue( + 'success' + ); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + // Set auto-approve for session-1 + toolManager.setSessionAutoApproveTools('session-1', ['mcp--test_tool']); + + // Execute with different session + await toolManager.executeTool('mcp--test_tool', {}, 'call-1', 'session-2'); + + // Should have requested approval (different session) + expect(mockApprovalManager.requestToolConfirmation).toHaveBeenCalled(); + }); + + it('should not auto-approve when no sessionId provided', async () => { + (mockMcpManager.getAllTools as ReturnType).mockResolvedValue({ + test_tool: { + name: 'test_tool', + description: 'A test tool', + inputSchema: {}, + }, + }); + (mockMcpManager.executeTool as ReturnType).mockResolvedValue( + 'success' + ); + + const toolManager = new ToolManager( + mockMcpManager, + mockApprovalManager, + mockAllowedToolsProvider, + 'manual', + mockAgentEventBus, + { alwaysAllow: [], alwaysDeny: [] }, + { internalToolsConfig: [], internalToolsServices: {} as any }, + mockLogger + ); + + toolManager.setSessionAutoApproveTools('session-1', ['mcp--test_tool']); + + // Execute without sessionId + await toolManager.executeTool('mcp--test_tool', {}, 'call-1'); + + // Should have requested approval (no sessionId means no session auto-approve) + expect(mockApprovalManager.requestToolConfirmation).toHaveBeenCalled(); + }); + }); + }); +}); diff --git a/dexto/packages/core/src/tools/tool-manager.ts b/dexto/packages/core/src/tools/tool-manager.ts new file mode 100644 index 00000000..8466e398 --- /dev/null +++ b/dexto/packages/core/src/tools/tool-manager.ts @@ -0,0 +1,1358 @@ +import { MCPManager } from '../mcp/manager.js'; +import { InternalToolsProvider } from './internal-tools/provider.js'; +import { InternalToolsServices } from './internal-tools/registry.js'; +import type { InternalToolsConfig, CustomToolsConfig, ToolPolicies } from './schemas.js'; +import { ToolSet, ToolExecutionContext } from './types.js'; +import type { ToolDisplayData } from './display-types.js'; +import { ToolError } from './errors.js'; +import { ToolErrorCode } from './error-codes.js'; +import { DextoRuntimeError } from '../errors/index.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import { DextoLogComponent } from '../logger/v2/types.js'; +import type { AgentEventBus } from '../events/index.js'; +import type { ApprovalManager } from '../approval/manager.js'; +import { ApprovalStatus, ApprovalType, DenialReason } from '../approval/types.js'; +import type { ApprovalRequest, ToolConfirmationMetadata } from '../approval/types.js'; +import type { IAllowedToolsProvider } from './confirmation/allowed-tools-provider/types.js'; +import type { PluginManager } from '../plugins/manager.js'; +import type { PromptManager } from '../prompts/prompt-manager.js'; +import type { SessionManager } from '../session/index.js'; +import type { AgentStateManager } from '../agent/state-manager.js'; +import type { BeforeToolCallPayload, AfterToolResultPayload } from '../plugins/types.js'; +import { InstrumentClass } from '../telemetry/decorators.js'; +import { + generateBashPatternKey, + generateBashPatternSuggestions, + isDangerousCommand, +} from './bash-pattern-utils.js'; +/** + * Options for internal tools configuration in ToolManager + */ +export interface InternalToolsOptions { + internalToolsServices?: InternalToolsServices; + internalToolsConfig?: InternalToolsConfig; + customToolsConfig?: CustomToolsConfig; +} + +/** + * Unified Tool Manager - Single interface for all tool operations + * + * This class acts as the single point of contact between the LLM and all tool sources. + * It aggregates tools from MCP servers and internal tools, providing a unified interface + * for tool discovery, aggregation, and execution. + * + * Responsibilities: + * - Aggregate tools from MCP servers and internal tools with conflict resolution + * - Route tool execution to appropriate source (MCP vs Internal) + * - Provide unified tool interface to LLM + * - Manage tool confirmation and security via ApprovalManager + * - Handle cross-source naming conflicts (internal tools have precedence) + * + * Architecture: + * LLMService → ToolManager → [MCPManager, InternalToolsProvider] + * ↓ + * ApprovalManager (for confirmations) + * + * TODO (Telemetry): Add OpenTelemetry metrics collection + * - Tool execution counters (by tool name, source: MCP/internal) + * - Tool execution latency histograms + * - Tool success/failure rate counters + * - Tool approval/denial counters + * See feature-plans/telemetry.md for details + */ +@InstrumentClass({ + prefix: 'tool', + excludeMethods: [ + 'setPluginManager', + 'setStateManager', + 'getApprovalManager', + 'getAllowedToolsProvider', + ], +}) +export class ToolManager { + private mcpManager: MCPManager; + private internalToolsProvider?: InternalToolsProvider; + private approvalManager: ApprovalManager; + private allowedToolsProvider: IAllowedToolsProvider; + private approvalMode: 'manual' | 'auto-approve' | 'auto-deny'; + private agentEventBus: AgentEventBus; + private toolPolicies: ToolPolicies | undefined; + + // Plugin support - set after construction to avoid circular dependencies + private pluginManager?: PluginManager; + private sessionManager?: SessionManager; + private stateManager?: AgentStateManager; + + // Tool source prefixing - ALL tools get prefixed by source + private static readonly MCP_TOOL_PREFIX = 'mcp--'; + private static readonly INTERNAL_TOOL_PREFIX = 'internal--'; + private static readonly CUSTOM_TOOL_PREFIX = 'custom--'; + + // Tool caching for performance + private toolsCache: ToolSet = {}; + private cacheValid: boolean = false; + private logger: IDextoLogger; + + // Session-level auto-approve tools for skills + // When a skill with allowedTools is invoked, those tools are auto-approved (skip confirmation) + // This is ADDITIVE - other tools are NOT blocked, they just go through normal approval flow + private sessionAutoApproveTools: Map = new Map(); + + constructor( + mcpManager: MCPManager, + approvalManager: ApprovalManager, + allowedToolsProvider: IAllowedToolsProvider, + approvalMode: 'manual' | 'auto-approve' | 'auto-deny', + agentEventBus: AgentEventBus, + toolPolicies: ToolPolicies, + options: InternalToolsOptions, + logger: IDextoLogger + ) { + this.mcpManager = mcpManager; + this.approvalManager = approvalManager; + this.allowedToolsProvider = allowedToolsProvider; + this.approvalMode = approvalMode; + this.agentEventBus = agentEventBus; + this.toolPolicies = toolPolicies; + this.logger = logger.createChild(DextoLogComponent.TOOLS); + + // Initialize internal tools and/or custom tools if configured + if ( + (options?.internalToolsConfig && options.internalToolsConfig.length > 0) || + (options?.customToolsConfig && options.customToolsConfig.length > 0) + ) { + // Include approvalManager in services for internal tools + const internalToolsServices = { + ...options.internalToolsServices, + approvalManager, + }; + this.internalToolsProvider = new InternalToolsProvider( + internalToolsServices, + options.internalToolsConfig || [], + options.customToolsConfig || [], + this.logger + ); + } + + // Set up event listeners for surgical cache updates + this.setupNotificationListeners(); + + this.logger.debug('ToolManager initialized'); + } + + /** + * Initialize the ToolManager and its components + */ + async initialize(): Promise { + if (this.internalToolsProvider) { + await this.internalToolsProvider.initialize(); + } + this.logger.debug('ToolManager initialization complete'); + } + + /** + * Set plugin support services (called after construction to avoid circular dependencies) + */ + setPluginSupport( + pluginManager: PluginManager, + sessionManager: SessionManager, + stateManager: AgentStateManager + ): void { + this.pluginManager = pluginManager; + this.sessionManager = sessionManager; + this.stateManager = stateManager; + this.logger.debug('Plugin support configured for ToolManager'); + } + + /** + * Set agent reference for custom tools (called after construction to avoid circular dependencies) + * Must be called before initialize() if custom tools are configured + */ + setAgent(agent: any): void { + if (this.internalToolsProvider) { + this.internalToolsProvider.setAgent(agent); + this.logger.debug('Agent reference configured for custom tools'); + } + } + + /** + * Set prompt manager for invoke_skill tool (called after construction to avoid circular dependencies) + * Must be called before initialize() if invoke_skill tool is enabled + */ + setPromptManager(promptManager: PromptManager): void { + if (this.internalToolsProvider) { + this.internalToolsProvider.setPromptManager(promptManager); + this.logger.debug('PromptManager reference configured for invoke_skill tool'); + } + } + + /** + * Set task forker for context:fork skill execution (late-binding) + * Called by agent-spawner custom tool provider after RuntimeService is created. + * This enables invoke_skill to fork execution to an isolated subagent. + */ + setTaskForker(taskForker: import('./internal-tools/registry.js').TaskForker): void { + if (this.internalToolsProvider) { + this.internalToolsProvider.setTaskForker(taskForker); + this.logger.debug( + 'TaskForker reference configured for invoke_skill (context:fork support)' + ); + } + } + + // ============= SESSION AUTO-APPROVE TOOLS ============= + + /** + * Set session-level auto-approve tools. + * When set, these tools will skip confirmation prompts for this session. + * This is ADDITIVE - other tools are NOT blocked, they just go through normal approval flow. + * + * @param sessionId The session ID + * @param autoApproveTools Array of tool names to auto-approve (e.g., ['custom--bash_exec', 'custom--read_file']) + */ + setSessionAutoApproveTools(sessionId: string, autoApproveTools: string[]): void { + // Empty array = no auto-approvals, same as clearing + if (autoApproveTools.length === 0) { + this.clearSessionAutoApproveTools(sessionId); + return; + } + this.sessionAutoApproveTools.set(sessionId, autoApproveTools); + this.logger.info( + `Session auto-approve tools set for '${sessionId}': ${autoApproveTools.length} tools` + ); + this.logger.debug(`Auto-approve tools: ${autoApproveTools.join(', ')}`); + } + + /** + * Clear session-level auto-approve tools. + * Call this when the session ends or when the skill completes. + * + * @param sessionId The session ID to clear auto-approve tools for + */ + clearSessionAutoApproveTools(sessionId: string): void { + const hadAutoApprove = this.sessionAutoApproveTools.has(sessionId); + this.sessionAutoApproveTools.delete(sessionId); + if (hadAutoApprove) { + this.logger.info(`Session auto-approve tools cleared for '${sessionId}'`); + } + } + + /** + * Check if a session has auto-approve tools set. + * + * @param sessionId The session ID to check + * @returns true if the session has auto-approve tools + */ + hasSessionAutoApproveTools(sessionId: string): boolean { + return this.sessionAutoApproveTools.has(sessionId); + } + + /** + * Get the auto-approve tools for a session. + * + * @param sessionId The session ID to check + * @returns Array of auto-approve tool names, or undefined if none set + */ + getSessionAutoApproveTools(sessionId: string): string[] | undefined { + return this.sessionAutoApproveTools.get(sessionId); + } + + /** + * Check if a tool should be auto-approved for a session. + * Returns true if the tool is in the session's auto-approve list. + * + * @param sessionId The session ID + * @param toolName The tool name to check + * @returns true if the tool should be auto-approved + */ + private isToolAutoApprovedForSession(sessionId: string, toolName: string): boolean { + const autoApproveTools = this.sessionAutoApproveTools.get(sessionId); + if (!autoApproveTools) { + return false; + } + // Check if tool matches any auto-approve pattern + return autoApproveTools.some((pattern) => this.matchesToolPolicy(toolName, pattern)); + } + + /** + * Invalidate the tools cache when tool sources change + */ + private invalidateCache(): void { + this.cacheValid = false; + this.toolsCache = {}; + } + + /** + * Set up listeners for MCP notifications to invalidate cache on changes + */ + private setupNotificationListeners(): void { + // Listen for MCP server connection changes that affect tools + this.agentEventBus.on('mcp:server-connected', async (payload) => { + if (payload.success) { + this.logger.debug( + `🔄 MCP server connected, invalidating tool cache: ${payload.name}` + ); + this.invalidateCache(); + } + }); + + this.agentEventBus.on('mcp:server-removed', async (payload) => { + this.logger.debug( + `🔄 MCP server removed: ${payload.serverName}, invalidating tool cache` + ); + this.invalidateCache(); + }); + + // Auto-clear session auto-approve tools when a run completes + // This ensures skill auto-approve tools don't persist beyond their intended scope + this.agentEventBus.on('run:complete', (payload) => { + if (this.hasSessionAutoApproveTools(payload.sessionId)) { + this.logger.debug( + `🔓 Run complete, clearing session auto-approve tools for '${payload.sessionId}'` + ); + this.clearSessionAutoApproveTools(payload.sessionId); + } + }); + } + + // ==================== Bash Pattern Approval Helpers ==================== + + /** + * Check if a tool name represents a bash execution tool + */ + private isBashTool(toolName: string): boolean { + return ( + toolName === 'bash_exec' || + toolName === 'internal--bash_exec' || + toolName === 'custom--bash_exec' + ); + } + + /** + * Check if a bash command is covered by any approved pattern. + * Generates a pattern key from the command, then checks if it's covered by stored patterns. + * + * Returns approval info if covered, or pattern suggestions if not. + */ + private checkBashPatternApproval(command: string): { + approved: boolean; + suggestedPatterns?: string[]; + } { + // Generate the pattern key for this command + const patternKey = generateBashPatternKey(command); + + if (!patternKey) { + // Dangerous command - no pattern, require explicit approval each time + if (isDangerousCommand(command)) { + this.logger.debug( + `Skipping pattern generation for dangerous command: ${command.split(/\s+/)[0]}` + ); + } + return { approved: false, suggestedPatterns: [] }; + } + + // Check if this pattern key is covered by any approved pattern + if (this.approvalManager.matchesBashPattern(patternKey)) { + return { approved: true }; + } + + // Generate broader pattern suggestions for the UI + return { + approved: false, + suggestedPatterns: generateBashPatternSuggestions(command), + }; + } + + /** + * Auto-approve pending tool confirmation requests for the same tool. + * Called after a user selects "remember choice" for a tool. + * This handles the case where parallel tool calls come in before the first one is approved. + * + * @param toolName The tool name that was just remembered + * @param sessionId The session ID for which the tool was allowed + */ + private autoApprovePendingToolRequests(toolName: string, sessionId?: string): void { + const count = this.approvalManager.autoApprovePendingRequests( + (request: ApprovalRequest) => { + // Only match tool confirmation requests + if (request.type !== ApprovalType.TOOL_CONFIRMATION) { + return false; + } + + // Only match requests for the same session + if (request.sessionId !== sessionId) { + return false; + } + + // Check if it's the same tool + const metadata = request.metadata as ToolConfirmationMetadata; + return metadata.toolName === toolName; + }, + { rememberChoice: false } // Don't propagate remember choice to auto-approved requests + ); + + if (count > 0) { + this.logger.info( + `Auto-approved ${count} parallel request(s) for tool '${toolName}' after user selected "remember choice"` + ); + } + } + + /** + * Auto-approve pending bash command requests that match a pattern. + * Called after a user selects "remember pattern" for a bash command. + * This handles the case where parallel bash commands come in before the first one is approved. + * + * @param pattern The bash pattern that was just remembered + * @param sessionId The session ID for context + */ + private autoApprovePendingBashRequests(pattern: string, sessionId?: string): void { + const count = this.approvalManager.autoApprovePendingRequests( + (request: ApprovalRequest) => { + // Only match tool confirmation requests + if (request.type !== ApprovalType.TOOL_CONFIRMATION) { + return false; + } + + // Only match requests for the same session + if (request.sessionId !== sessionId) { + return false; + } + + // Check if it's a bash tool + const metadata = request.metadata as ToolConfirmationMetadata; + if (!this.isBashTool(metadata.toolName)) { + return false; + } + + // Check if the command matches the pattern + const command = metadata.args?.command as string | undefined; + if (!command) { + return false; + } + + // Generate pattern key for this command and check if it matches + const patternKey = generateBashPatternKey(command); + if (!patternKey) { + return false; + } + + // Check if this command would now be approved with the new pattern + // The pattern was just added, so we can use matchesBashPattern + return this.approvalManager.matchesBashPattern(patternKey); + }, + { rememberPattern: undefined } // Don't propagate pattern to auto-approved requests + ); + + if (count > 0) { + this.logger.info( + `Auto-approved ${count} parallel bash command(s) matching pattern '${pattern}'` + ); + } + } + + getMcpManager(): MCPManager { + return this.mcpManager; + } + + /** + * Get all MCP tools (delegates to mcpManager.getAllTools()) + * This provides access to MCP tools while maintaining separation of concerns + */ + async getMcpTools(): Promise { + return await this.mcpManager.getAllTools(); + } + + /** + * Build all tools from sources with universal prefixing + * ALL tools get prefixed by their source - no exceptions + * + * TODO: Rethink tool naming convention for more consistency + * Current issue: MCP tools have dynamic naming based on conflicts: + * - No conflict: mcp--toolName + * - With conflict: mcp--serverName--toolName + * This makes policy configuration fragile. Consider: + * 1. Always including server name: mcp--serverName--toolName (breaking change) + * 2. Using a different delimiter pattern that's more predictable + * 3. Providing a tool discovery command to help users find exact names + * Related: Tool policies now support dual matching (exact + suffix) as a workaround + */ + private async buildAllTools(): Promise { + const allTools: ToolSet = {}; + + // Get tools from all sources (already in final JSON Schema format) + let mcpTools: ToolSet = {}; + let internalTools: ToolSet = {}; + let customTools: ToolSet = {}; + + try { + mcpTools = await this.mcpManager.getAllTools(); + } catch (error) { + this.logger.error( + `Failed to get MCP tools: ${error instanceof Error ? error.message : String(error)}` + ); + mcpTools = {}; + } + + try { + internalTools = this.internalToolsProvider?.getInternalTools() || {}; + } catch (error) { + this.logger.error( + `Failed to get internal tools: ${error instanceof Error ? error.message : String(error)}` + ); + internalTools = {}; + } + + try { + customTools = this.internalToolsProvider?.getCustomTools() || {}; + } catch (error) { + this.logger.error( + `Failed to get custom tools: ${error instanceof Error ? error.message : String(error)}` + ); + customTools = {}; + } + + // Add internal tools with 'internal--' prefix + for (const [toolName, toolDef] of Object.entries(internalTools)) { + const qualifiedName = `${ToolManager.INTERNAL_TOOL_PREFIX}${toolName}`; + allTools[qualifiedName] = { + ...toolDef, + name: qualifiedName, + description: `${toolDef.description || 'No description provided'} (internal tool)`, + }; + } + + // Add custom tools with 'custom--' prefix + for (const [toolName, toolDef] of Object.entries(customTools)) { + const qualifiedName = `${ToolManager.CUSTOM_TOOL_PREFIX}${toolName}`; + allTools[qualifiedName] = { + ...toolDef, + name: qualifiedName, + description: `${toolDef.description || 'No description provided'} (custom tool)`, + }; + } + + // Add MCP tools with 'mcp--' prefix + for (const [toolName, toolDef] of Object.entries(mcpTools)) { + const qualifiedName = `${ToolManager.MCP_TOOL_PREFIX}${toolName}`; + allTools[qualifiedName] = { + ...toolDef, + name: qualifiedName, + description: `${toolDef.description || 'No description provided'} (via MCP servers)`, + }; + } + + const totalTools = Object.keys(allTools).length; + const mcpCount = Object.keys(mcpTools).length; + const internalCount = Object.keys(internalTools).length; + const customCount = Object.keys(customTools).length; + + this.logger.debug( + `🔧 Unified tool discovery: ${totalTools} total tools (${mcpCount} MCP, ${internalCount} internal, ${customCount} custom)` + ); + + return allTools; + } + + /** + * Get all available tools from all sources with conflict resolution + * This is the single interface the LLM uses to discover tools + * Uses caching to avoid rebuilding on every call + */ + async getAllTools(): Promise { + if (this.cacheValid) { + return this.toolsCache; + } + + this.toolsCache = await this.buildAllTools(); + this.cacheValid = true; + return this.toolsCache; + } + + /** + * Execute a tool by routing based on universal prefix + * ALL tools must have source prefix - no exceptions + * + * @param toolName The fully qualified tool name (e.g., "internal--edit_file") + * @param args The arguments for the tool + * @param toolCallId The unique tool call ID for tracking (from LLM or generated for direct calls) + * @param sessionId Optional session ID for context + * @param abortSignal Optional abort signal for cancellation support + */ + async executeTool( + toolName: string, + args: Record, + toolCallId: string, + sessionId?: string, + abortSignal?: AbortSignal + ): Promise { + this.logger.debug(`🔧 Tool execution requested: '${toolName}' (toolCallId: ${toolCallId})`); + this.logger.debug(`Tool args: ${JSON.stringify(args, null, 2)}`); + + // IMPORTANT: Emit llm:tool-call FIRST, before approval handling. + // This ensures correct event ordering - llm:tool-call must arrive before approval:request + // in the CLI's event stream. + // + // Why this is needed: The Vercel SDK enqueues tool-call to the stream BEFORE calling execute(), + // but our async iterator hasn't read from the queue yet. Meanwhile, execute() runs synchronously + // until its first await, and EventEmitter.emit() is synchronous. So events emitted here arrive + // before our iterator processes the queued tool-call. By emitting here, we guarantee llm:tool-call + // arrives before approval:request. + if (sessionId) { + this.agentEventBus.emit('llm:tool-call', { + toolName, + args, + callId: toolCallId, + sessionId, + }); + } + + // Handle approval/confirmation flow - returns whether approval was required + const { requireApproval, approvalStatus } = await this.handleToolApproval( + toolName, + args, + toolCallId, + sessionId + ); + + this.logger.debug(`✅ Tool execution approved: ${toolName}`); + this.logger.info( + `🔧 Tool execution started for ${toolName}, sessionId: ${sessionId ?? 'global'}` + ); + + // Emit tool:running event - tool is now actually executing (after approval if needed) + // Only emit when sessionId is provided (LLM flow) - direct API calls don't need UI updates + if (sessionId) { + this.agentEventBus.emit('tool:running', { + toolName, + toolCallId, + sessionId, + }); + } + + const startTime = Date.now(); + + // Execute beforeToolCall plugins if available + if (this.pluginManager && this.sessionManager && this.stateManager) { + const beforePayload: BeforeToolCallPayload = { + toolName, + args, + ...(sessionId !== undefined && { sessionId }), + }; + + const modifiedPayload = await this.pluginManager.executePlugins( + 'beforeToolCall', + beforePayload, + { + sessionManager: this.sessionManager, + mcpManager: this.mcpManager, + toolManager: this, + stateManager: this.stateManager, + ...(sessionId !== undefined && { sessionId }), + } + ); + + // Use modified payload for execution + args = modifiedPayload.args; + } + + try { + let result: unknown; + + // Route to MCP tools + if (toolName.startsWith(ToolManager.MCP_TOOL_PREFIX)) { + this.logger.debug(`🔧 Detected MCP tool: '${toolName}'`); + const actualToolName = toolName.substring(ToolManager.MCP_TOOL_PREFIX.length); + if (actualToolName.length === 0) { + throw ToolError.invalidName(toolName, 'tool name cannot be empty after prefix'); + } + this.logger.debug(`🎯 MCP routing: '${toolName}' -> '${actualToolName}'`); + result = await this.mcpManager.executeTool(actualToolName, args, sessionId); + } + // Route to internal tools + else if (toolName.startsWith(ToolManager.INTERNAL_TOOL_PREFIX)) { + this.logger.debug(`🔧 Detected internal tool: '${toolName}'`); + const actualToolName = toolName.substring(ToolManager.INTERNAL_TOOL_PREFIX.length); + if (actualToolName.length === 0) { + throw ToolError.invalidName(toolName, 'tool name cannot be empty after prefix'); + } + if (!this.internalToolsProvider) { + throw ToolError.internalToolsNotInitialized(toolName); + } + this.logger.debug(`🎯 Internal routing: '${toolName}' -> '${actualToolName}'`); + result = await this.internalToolsProvider.executeTool( + actualToolName, + args, + sessionId, + abortSignal, + toolCallId + ); + } + // Route to custom tools + else if (toolName.startsWith(ToolManager.CUSTOM_TOOL_PREFIX)) { + this.logger.debug(`🔧 Detected custom tool: '${toolName}'`); + const actualToolName = toolName.substring(ToolManager.CUSTOM_TOOL_PREFIX.length); + if (actualToolName.length === 0) { + throw ToolError.invalidName(toolName, 'tool name cannot be empty after prefix'); + } + if (!this.internalToolsProvider) { + throw ToolError.internalToolsNotInitialized(toolName); + } + this.logger.debug(`🎯 Custom routing: '${toolName}' -> '${actualToolName}'`); + result = await this.internalToolsProvider.executeTool( + actualToolName, + args, + sessionId, + abortSignal, + toolCallId + ); + } + // Tool doesn't have proper prefix + else { + this.logger.debug(`🔧 Detected tool without proper prefix: '${toolName}'`); + const stats = await this.getToolStats(); + this.logger.error( + `❌ Tool missing source prefix: '${toolName}' (expected '${ToolManager.MCP_TOOL_PREFIX}*', '${ToolManager.INTERNAL_TOOL_PREFIX}*', or '${ToolManager.CUSTOM_TOOL_PREFIX}*')` + ); + this.logger.debug( + `Available: ${stats.mcp} MCP, ${stats.internal} internal, ${stats.custom} custom tools` + ); + throw ToolError.notFound(toolName); + } + + const duration = Date.now() - startTime; + this.logger.debug(`🎯 Tool execution completed in ${duration}ms: '${toolName}'`); + this.logger.info( + `✅ Tool execution completed successfully for ${toolName} in ${duration}ms, sessionId: ${sessionId ?? 'global'}` + ); + + // Execute afterToolResult plugins if available + if (this.pluginManager && this.sessionManager && this.stateManager) { + const afterPayload: AfterToolResultPayload = { + toolName, + result, + success: true, + ...(sessionId !== undefined && { sessionId }), + }; + + const modifiedPayload = await this.pluginManager.executePlugins( + 'afterToolResult', + afterPayload, + { + sessionManager: this.sessionManager, + mcpManager: this.mcpManager, + toolManager: this, + stateManager: this.stateManager, + ...(sessionId !== undefined && { sessionId }), + } + ); + + // Use modified result + result = modifiedPayload.result; + } + + return { + result, + ...(requireApproval && { requireApproval, approvalStatus }), + }; + } catch (error) { + const duration = Date.now() - startTime; + this.logger.error( + `❌ Tool execution failed for ${toolName} after ${duration}ms, sessionId: ${sessionId ?? 'global'}: ${error instanceof Error ? error.message : String(error)}` + ); + + // Execute afterToolResult plugins for error case if available + if (this.pluginManager && this.sessionManager && this.stateManager) { + const afterPayload: AfterToolResultPayload = { + toolName, + result: error instanceof Error ? error.message : String(error), + success: false, + ...(sessionId !== undefined && { sessionId }), + }; + + // Note: We still execute plugins even on error, but we don't use the modified result + // Plugins can log, track metrics, etc. but cannot suppress the error + await this.pluginManager.executePlugins('afterToolResult', afterPayload, { + sessionManager: this.sessionManager, + mcpManager: this.mcpManager, + toolManager: this, + stateManager: this.stateManager, + ...(sessionId !== undefined && { sessionId }), + }); + } + + throw error; + } + } + + /** + * Check if a tool exists (must have proper source prefix) + */ + async hasTool(toolName: string): Promise { + // Check MCP tools + if (toolName.startsWith(ToolManager.MCP_TOOL_PREFIX)) { + const actualToolName = toolName.substring(ToolManager.MCP_TOOL_PREFIX.length); + return this.mcpManager.getToolClient(actualToolName) !== undefined; + } + + // Check internal tools + if (toolName.startsWith(ToolManager.INTERNAL_TOOL_PREFIX)) { + const actualToolName = toolName.substring(ToolManager.INTERNAL_TOOL_PREFIX.length); + return this.internalToolsProvider?.hasInternalTool(actualToolName) ?? false; + } + + // Check custom tools + if (toolName.startsWith(ToolManager.CUSTOM_TOOL_PREFIX)) { + const actualToolName = toolName.substring(ToolManager.CUSTOM_TOOL_PREFIX.length); + return this.internalToolsProvider?.hasCustomTool(actualToolName) ?? false; + } + + // Tool without proper prefix doesn't exist + return false; + } + + /** + * Get tool statistics across all sources + */ + async getToolStats(): Promise<{ + total: number; + mcp: number; + internal: number; + custom: number; + }> { + let mcpTools: ToolSet = {}; + let internalTools: ToolSet = {}; + let customTools: ToolSet = {}; + + try { + mcpTools = await this.mcpManager.getAllTools(); + } catch (error) { + this.logger.error( + `Failed to get MCP tools for stats: ${error instanceof Error ? error.message : String(error)}` + ); + mcpTools = {}; + } + + try { + internalTools = this.internalToolsProvider?.getInternalTools() || {}; + } catch (error) { + this.logger.error( + `Failed to get internal tools for stats: ${error instanceof Error ? error.message : String(error)}` + ); + internalTools = {}; + } + + try { + customTools = this.internalToolsProvider?.getCustomTools() || {}; + } catch (error) { + this.logger.error( + `Failed to get custom tools for stats: ${error instanceof Error ? error.message : String(error)}` + ); + customTools = {}; + } + + const mcpCount = Object.keys(mcpTools).length; + const internalCount = Object.keys(internalTools).length; + const customCount = Object.keys(customTools).length; + + return { + total: mcpCount + internalCount + customCount, + mcp: mcpCount, + internal: internalCount, + custom: customCount, + }; + } + + /** + * Get the source of a tool (mcp, internal, custom, or unknown) + * @param toolName The name of the tool to check + * @returns The source of the tool + */ + getToolSource(toolName: string): 'mcp' | 'internal' | 'custom' | 'unknown' { + if ( + toolName.startsWith(ToolManager.MCP_TOOL_PREFIX) && + toolName.length > ToolManager.MCP_TOOL_PREFIX.length + ) { + return 'mcp'; + } + if ( + toolName.startsWith(ToolManager.INTERNAL_TOOL_PREFIX) && + toolName.length > ToolManager.INTERNAL_TOOL_PREFIX.length + ) { + return 'internal'; + } + if ( + toolName.startsWith(ToolManager.CUSTOM_TOOL_PREFIX) && + toolName.length > ToolManager.CUSTOM_TOOL_PREFIX.length + ) { + return 'custom'; + } + return 'unknown'; + } + + /** + * Check if a tool matches a policy pattern + * Supports both exact matching and suffix matching for MCP tools with server prefixes + * + * Examples: + * - Policy "mcp--read_file" matches "mcp--read_file" (exact) + * - Policy "mcp--read_file" matches "mcp--filesystem--read_file" (suffix) + * - Policy "internal--ask_user" matches "internal--ask_user" (exact only) + * + * @param toolName The fully qualified tool name (e.g., "mcp--filesystem--read_file") + * @param policyPattern The policy pattern to match against (e.g., "mcp--read_file") + * @returns true if the tool matches the policy pattern + */ + private matchesToolPolicy(toolName: string, policyPattern: string): boolean { + // Exact match + if (toolName === policyPattern) { + return true; + } + + // Suffix match for MCP tools with server conflicts + // Policy "mcp--read_file" should match "mcp--filesystem--read_file" + // Note: MCP server delimiter is '--' (defined in MCPManager.SERVER_DELIMITER) + if (policyPattern.startsWith(ToolManager.MCP_TOOL_PREFIX)) { + // Extract the base tool name without mcp-- prefix + const baseName = policyPattern.substring(ToolManager.MCP_TOOL_PREFIX.length); + + // Check if the tool name ends with --{baseName} and starts with mcp-- + // This handles: mcp--filesystem--read_file matching policy mcp--read_file + if ( + toolName.endsWith(`--${baseName}`) && + toolName.startsWith(ToolManager.MCP_TOOL_PREFIX) + ) { + return true; + } + } + + return false; + } + + /** + * Check if a tool is in the static alwaysDeny list + * Supports both exact and suffix matching (e.g., "mcp--read_file" matches "mcp--server--read_file") + * @param toolName The fully qualified tool name to check + * @returns true if the tool is in the deny list + */ + private isInAlwaysDenyList(toolName: string): boolean { + if (!this.toolPolicies?.alwaysDeny) { + return false; + } + return this.toolPolicies.alwaysDeny.some((pattern) => + this.matchesToolPolicy(toolName, pattern) + ); + } + + /** + * Check if a tool is in the static alwaysAllow list + * Supports both exact and suffix matching (e.g., "mcp--read_file" matches "mcp--server--read_file") + * @param toolName The fully qualified tool name to check + * @returns true if the tool is in the allow list + */ + private isInAlwaysAllowList(toolName: string): boolean { + if (!this.toolPolicies?.alwaysAllow) { + return false; + } + return this.toolPolicies.alwaysAllow.some((pattern) => + this.matchesToolPolicy(toolName, pattern) + ); + } + + /** + * Check if a tool has a custom approval override and handle it. + * Tools can implement getApprovalOverride() to request specialized approval flows + * (e.g., directory access approval for file tools) instead of default tool confirmation. + * + * @param toolName The fully qualified tool name + * @param args The tool arguments + * @param sessionId Optional session ID + * @returns { handled: true } if custom approval was processed, { handled: false } to continue normal flow + */ + private async checkCustomApprovalOverride( + toolName: string, + args: Record, + sessionId?: string + ): Promise<{ handled: boolean }> { + // Get the actual tool name without prefix + let actualToolName: string | undefined; + if (toolName.startsWith(ToolManager.INTERNAL_TOOL_PREFIX)) { + actualToolName = toolName.substring(ToolManager.INTERNAL_TOOL_PREFIX.length); + } else if (toolName.startsWith(ToolManager.CUSTOM_TOOL_PREFIX)) { + actualToolName = toolName.substring(ToolManager.CUSTOM_TOOL_PREFIX.length); + } + + if (!actualToolName || !this.internalToolsProvider) { + return { handled: false }; + } + + // Get the tool and check if it has custom approval override + const tool = this.internalToolsProvider.getTool(actualToolName); + if (!tool?.getApprovalOverride) { + return { handled: false }; + } + + // Get the custom approval request from the tool (may be async) + const approvalRequest = await tool.getApprovalOverride(args); + if (!approvalRequest) { + // Tool decided no custom approval needed, continue normal flow + return { handled: false }; + } + + this.logger.debug( + `Tool '${toolName}' requested custom approval: type=${approvalRequest.type}` + ); + + // Add sessionId to the approval request if not already present + if (sessionId && !approvalRequest.sessionId) { + approvalRequest.sessionId = sessionId; + } + + // Request the custom approval through ApprovalManager + const response = await this.approvalManager.requestApproval(approvalRequest); + + if (response.status === ApprovalStatus.APPROVED) { + // Let the tool handle the approved response (e.g., remember directory) + if (tool.onApprovalGranted) { + tool.onApprovalGranted(response); + } + + this.logger.info( + `Custom approval granted for '${toolName}', type=${approvalRequest.type}, session=${sessionId ?? 'global'}` + ); + return { handled: true }; + } + + // Handle denial - throw appropriate error based on approval type + this.logger.info( + `Custom approval denied for '${toolName}', type=${approvalRequest.type}, reason=${response.reason ?? 'unknown'}` + ); + + // For directory access, throw specific error + if (approvalRequest.type === 'directory_access') { + const metadata = approvalRequest.metadata as { parentDir?: string } | undefined; + throw ToolError.directoryAccessDenied( + metadata?.parentDir ?? 'unknown directory', + sessionId + ); + } + + // For other custom approval types, throw generic execution denied + throw ToolError.executionDenied(toolName, sessionId); + } + + /** + * Handle tool approval flow. Checks various precedence levels to determine + * if a tool should be auto-approved, denied, or requires manual approval. + */ + private async handleToolApproval( + toolName: string, + args: Record, + toolCallId: string, + sessionId?: string + ): Promise<{ requireApproval: boolean; approvalStatus?: 'approved' | 'rejected' }> { + // Try quick resolution first (auto-approve/deny based on policies) + const quickResult = await this.tryQuickApprovalResolution(toolName, args, sessionId); + if (quickResult !== null) { + return quickResult; + } + + // Fall back to manual approval flow + return this.requestManualApproval(toolName, args, toolCallId, sessionId); + } + + /** + * Try to resolve tool approval quickly based on policies and cached permissions. + * Returns null if manual approval is needed. + * + * Precedence order (highest to lowest): + * 1. Static deny list (security - always blocks) + * 2. Custom approval override (tool-specific approval flows) + * 3. Session auto-approve (skill allowed-tools) + * 4. Static allow list + * 5. Dynamic "remembered" allowed list + * 6. Bash command patterns + * 7. Approval mode (auto-approve/auto-deny) + */ + private async tryQuickApprovalResolution( + toolName: string, + args: Record, + sessionId?: string + ): Promise<{ requireApproval: boolean; approvalStatus?: 'approved' } | null> { + // 1. Check static alwaysDeny list (highest priority - security-first) + if (this.isInAlwaysDenyList(toolName)) { + this.logger.info( + `Tool '${toolName}' is in static deny list – blocking execution (session: ${sessionId ?? 'global'})` + ); + throw ToolError.executionDenied(toolName, sessionId); + } + + // 2. Check custom approval override (e.g., directory access for file tools) + const customApprovalResult = await this.checkCustomApprovalOverride( + toolName, + args, + sessionId + ); + if (customApprovalResult.handled) { + return { requireApproval: true, approvalStatus: 'approved' }; + } + + // 3. Check session auto-approve (skill allowed-tools) + if (sessionId && this.isToolAutoApprovedForSession(sessionId, toolName)) { + this.logger.info( + `Tool '${toolName}' is in session's auto-approve list – skipping confirmation (session: ${sessionId})` + ); + return { requireApproval: false }; + } + + // 4. Check static alwaysAllow list + if (this.isInAlwaysAllowList(toolName)) { + this.logger.info( + `Tool '${toolName}' is in static allow list – skipping confirmation (session: ${sessionId ?? 'global'})` + ); + return { requireApproval: false }; + } + + // 5. Check dynamic "remembered" allowed list + if (await this.allowedToolsProvider.isToolAllowed(toolName, sessionId)) { + this.logger.info( + `Tool '${toolName}' already allowed for session '${sessionId ?? 'global'}' – skipping confirmation.` + ); + return { requireApproval: false }; + } + + // 6. Check bash command patterns + if (this.isBashTool(toolName)) { + const command = args.command as string | undefined; + if (command) { + const bashResult = this.checkBashPatternApproval(command); + if (bashResult.approved) { + this.logger.info( + `Bash command '${command}' matched approved pattern – skipping confirmation.` + ); + return { requireApproval: false }; + } + } + } + + // 7. Check approval mode + if (this.approvalMode === 'auto-approve') { + this.logger.debug(`🟢 Auto-approving tool execution: ${toolName}`); + return { requireApproval: false }; + } + + if (this.approvalMode === 'auto-deny') { + this.logger.debug(`🚫 Auto-denying tool execution: ${toolName}`); + throw ToolError.executionDenied(toolName, sessionId); + } + + // Needs manual approval + return null; + } + + /** + * Request manual approval from the user for a tool execution. + * Generates preview, sends approval request, and handles the response. + */ + private async requestManualApproval( + toolName: string, + args: Record, + toolCallId: string, + sessionId?: string + ): Promise<{ requireApproval: boolean; approvalStatus: 'approved' | 'rejected' }> { + this.logger.info( + `Tool confirmation requested for ${toolName}, sessionId: ${sessionId ?? 'global'}` + ); + + try { + // Generate preview for approval UI + const displayPreview = await this.generateToolPreview( + toolName, + args, + toolCallId, + sessionId + ); + + // Get suggested bash patterns if applicable + const suggestedPatterns = this.getBashSuggestedPatterns(toolName, args); + + // Build and send approval request + const response = await this.approvalManager.requestToolConfirmation({ + toolName, + toolCallId, + args, + ...(sessionId !== undefined && { sessionId }), + ...(displayPreview !== undefined && { displayPreview }), + ...(suggestedPatterns !== undefined && { suggestedPatterns }), + }); + + // Handle "remember" choices if approved + if (response.status === ApprovalStatus.APPROVED && response.data) { + await this.handleRememberChoice(toolName, response, sessionId); + } + + // Process response + if (response.status !== ApprovalStatus.APPROVED) { + this.handleApprovalDenied(toolName, response, sessionId); + } + + this.logger.info( + `Tool confirmation approved for ${toolName}, sessionId: ${sessionId ?? 'global'}` + ); + return { requireApproval: true, approvalStatus: 'approved' }; + } catch (error) { + this.logger.error( + `Tool confirmation error for ${toolName}: ${error instanceof Error ? error.message : String(error)}` + ); + throw error; + } + } + + /** + * Generate a preview for the tool approval UI if the tool supports it. + */ + private async generateToolPreview( + toolName: string, + args: Record, + toolCallId: string, + sessionId?: string + ): Promise { + const actualToolName = toolName.replace(/^internal--/, '').replace(/^custom--/, ''); + const internalTool = this.internalToolsProvider?.getTool(actualToolName); + + if (!internalTool?.generatePreview) { + return undefined; + } + + try { + const context: ToolExecutionContext = { sessionId, toolCallId }; + const preview = await internalTool.generatePreview(args, context); + this.logger.debug(`Generated preview for ${toolName}`); + return preview ?? undefined; + } catch (previewError) { + // Validation errors should fail before approval + if ( + previewError instanceof DextoRuntimeError && + previewError.code === ToolErrorCode.VALIDATION_FAILED + ) { + this.logger.debug(`Validation failed for ${toolName}: ${previewError.message}`); + throw previewError; + } + // Other errors should not block approval + this.logger.debug( + `Preview generation failed for ${toolName}: ${previewError instanceof Error ? previewError.message : String(previewError)}` + ); + return undefined; + } + } + + /** + * Get suggested bash patterns for the approval UI. + */ + private getBashSuggestedPatterns( + toolName: string, + args: Record + ): string[] | undefined { + if (!this.isBashTool(toolName)) { + return undefined; + } + const command = args.command as string | undefined; + if (!command) { + return undefined; + } + const result = this.checkBashPatternApproval(command); + return result.suggestedPatterns?.length ? result.suggestedPatterns : undefined; + } + + /** + * Handle "remember choice" or "remember pattern" when user approves a tool. + */ + private async handleRememberChoice( + toolName: string, + response: { status: ApprovalStatus; data?: unknown; sessionId?: string | undefined }, + sessionId?: string + ): Promise { + const data = response.data as Record | undefined; + if (!data) return; + + const rememberChoice = data.rememberChoice as boolean | undefined; + const rememberPattern = data.rememberPattern as string | undefined; + + if (rememberChoice) { + const allowSessionId = sessionId ?? response.sessionId; + await this.allowedToolsProvider.allowTool(toolName, allowSessionId); + this.logger.info( + `Tool '${toolName}' added to allowed tools for session '${allowSessionId ?? 'global'}' (remember choice selected)` + ); + this.autoApprovePendingToolRequests(toolName, allowSessionId); + } else if (rememberPattern && this.isBashTool(toolName)) { + this.approvalManager.addBashPattern(rememberPattern); + this.logger.info(`Bash pattern '${rememberPattern}' added for session approval`); + this.autoApprovePendingBashRequests(rememberPattern, sessionId); + } + } + + /** + * Handle approval denied/timeout - throws appropriate error. + */ + private handleApprovalDenied( + toolName: string, + response: { + status: ApprovalStatus; + reason?: DenialReason | undefined; + message?: string | undefined; + timeoutMs?: number | undefined; + }, + sessionId?: string + ): never { + if ( + response.status === ApprovalStatus.CANCELLED && + response.reason === DenialReason.TIMEOUT + ) { + this.logger.info( + `Tool confirmation timed out for ${toolName}, sessionId: ${sessionId ?? 'global'}` + ); + throw ToolError.executionTimeout(toolName, response.timeoutMs ?? 0, sessionId); + } + + this.logger.info( + `Tool confirmation denied for ${toolName}, sessionId: ${sessionId ?? 'global'}, reason: ${response.reason ?? 'unknown'}` + ); + throw ToolError.executionDenied(toolName, sessionId, response.message); + } + + /** + * Refresh tool discovery (call when MCP servers change) + * Refreshes both MCPManager's cache (server capabilities) and ToolManager's cache (combined tools) + */ + async refresh(): Promise { + // First: Refresh MCPManager's cache to get fresh data from MCP servers + await this.mcpManager.refresh(); + + // Then: Invalidate our cache so next getAllTools() rebuilds from fresh MCP data + this.invalidateCache(); + + this.logger.debug('ToolManager refreshed (including MCP server capabilities)'); + } + + /** + * Get list of pending confirmation requests + */ + getPendingConfirmations(): string[] { + return this.approvalManager.getPendingApprovals(); + } + + /** + * Cancel a pending confirmation request + */ + cancelConfirmation(approvalId: string): void { + this.approvalManager.cancelApproval(approvalId); + } + + /** + * Cancel all pending confirmation requests + */ + cancelAllConfirmations(): void { + this.approvalManager.cancelAllApprovals(); + } +} diff --git a/dexto/packages/core/src/tools/types.ts b/dexto/packages/core/src/tools/types.ts new file mode 100644 index 00000000..d96d1161 --- /dev/null +++ b/dexto/packages/core/src/tools/types.ts @@ -0,0 +1,142 @@ +// ============================================================================ +// SIMPLIFIED TOOL TYPES - Essential interfaces only +// ============================================================================ + +import type { JSONSchema7 } from 'json-schema'; +import type { ZodSchema } from 'zod'; +import type { ToolDisplayData } from './display-types.js'; +import type { ApprovalRequestDetails, ApprovalResponse } from '../approval/types.js'; + +/** + * Context passed to tool execution + */ +export interface ToolExecutionContext { + /** Session ID if available */ + sessionId?: string | undefined; + /** Abort signal for cancellation support */ + abortSignal?: AbortSignal | undefined; + /** Unique tool call ID for tracking parallel tool calls */ + toolCallId?: string | undefined; +} + +/** + * Result of tool execution, including approval metadata + */ +export interface ToolExecutionResult { + /** The actual result data from tool execution */ + result: unknown; + /** Whether this tool required user approval before execution */ + requireApproval?: boolean; + /** The approval status (only present if requireApproval is true) */ + approvalStatus?: 'approved' | 'rejected'; +} + +// ============================================================================ +// CORE TOOL INTERFACES +// ============================================================================ + +/** + * Internal tool interface - for tools implemented within Dexto + */ +export interface InternalTool { + /** Unique identifier for the tool */ + id: string; + + /** Human-readable description of what the tool does */ + description: string; + + /** Zod schema defining the input parameters */ + inputSchema: ZodSchema; + + /** The actual function that executes the tool - input is validated by Zod before execution */ + execute: (input: unknown, context?: ToolExecutionContext) => Promise | unknown; + + /** + * Optional preview generator for approval UI. + * Called before requesting user approval to generate display data (e.g., diff preview). + * Returns null if no preview is available. + */ + generatePreview?: ( + input: unknown, + context?: ToolExecutionContext + ) => Promise; + + /** + * Optional custom approval override. + * If present and returns non-null, this approval request is used instead of + * the default tool confirmation. Allows tools to request specialized approval + * flows (e.g., directory access approval for file tools). + * + * @param args The validated input arguments for the tool + * @returns ApprovalRequestDetails for custom approval, or null to use default tool confirmation + * + * @example + * ```typescript + * // File tool requesting directory access approval for external paths + * getApprovalOverride: async (args) => { + * const filePath = (args as {file_path: string}).file_path; + * if (!await isPathWithinAllowed(filePath)) { + * return { + * type: ApprovalType.DIRECTORY_ACCESS, + * metadata: { path: filePath, operation: 'read', ... } + * }; + * } + * return null; // Use default tool confirmation + * } + * ``` + */ + getApprovalOverride?: ( + args: unknown + ) => Promise | ApprovalRequestDetails | null; + + /** + * Optional callback invoked when custom approval is granted. + * Allows tools to handle approval responses (e.g., remember approved directories). + * Only called when getApprovalOverride returned non-null and approval was granted. + * + * @param response The approval response from ApprovalManager + * + * @example + * ```typescript + * onApprovalGranted: (response) => { + * if (response.data?.rememberDirectory) { + * directoryApproval.addApproved(parentDir, 'session'); + * } + * } + * ``` + */ + onApprovalGranted?: (response: ApprovalResponse) => void; +} + +/** + * Standard tool set interface - used by AI/LLM services + * Each tool entry contains JSON Schema parameters + */ +export interface ToolSet { + [key: string]: { + name?: string; + description?: string; + parameters: JSONSchema7; // JSON Schema v7 specification + }; +} + +// ============================================================================ +// TOOL EXECUTION AND RESULTS +// ============================================================================ + +/** + * Tool execution result + */ +export interface ToolResult { + success: boolean; + data?: any; + error?: string; +} + +/** + * Interface for any provider of tools + */ +export interface ToolProvider { + getTools(): Promise; + callTool(toolName: string, args: Record): Promise; +} diff --git a/dexto/packages/core/src/utils/api-key-resolver.ts b/dexto/packages/core/src/utils/api-key-resolver.ts new file mode 100644 index 00000000..ed72f743 --- /dev/null +++ b/dexto/packages/core/src/utils/api-key-resolver.ts @@ -0,0 +1,72 @@ +import type { LLMProvider } from '../llm/types.js'; + +/** + * Utility for resolving API keys from environment variables. + * This consolidates the API key resolution logic used across CLI and core components. + */ + +// Map the provider to its corresponding API key name (in order of preference) +export const PROVIDER_API_KEY_MAP: Record = { + openai: ['OPENAI_API_KEY', 'OPENAI_KEY'], + 'openai-compatible': ['OPENAI_API_KEY', 'OPENAI_KEY'], // Uses same keys as openai + anthropic: ['ANTHROPIC_API_KEY', 'ANTHROPIC_KEY', 'CLAUDE_API_KEY'], + google: ['GOOGLE_GENERATIVE_AI_API_KEY', 'GOOGLE_API_KEY', 'GEMINI_API_KEY'], + groq: ['GROQ_API_KEY'], + cohere: ['COHERE_API_KEY'], + xai: ['XAI_API_KEY', 'X_AI_API_KEY'], + openrouter: ['OPENROUTER_API_KEY'], + litellm: ['LITELLM_API_KEY', 'LITELLM_KEY'], + glama: ['GLAMA_API_KEY'], + // Vertex uses ADC (Application Default Credentials), not API keys + // GOOGLE_APPLICATION_CREDENTIALS points to service account JSON (optional) + // Primary config is GOOGLE_VERTEX_PROJECT (required) + GOOGLE_VERTEX_LOCATION (optional) + vertex: [], + // Bedrock supports two auth methods: + // 1. AWS_BEARER_TOKEN_BEDROCK - Bedrock API key (simplest) + // 2. AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_REGION (IAM credentials) + // AWS_SESSION_TOKEN (optional, for temporary credentials) + bedrock: ['AWS_BEARER_TOKEN_BEDROCK'], + // Local providers don't require API keys + local: [], // Native node-llama-cpp execution + ollama: [], // Ollama server (no authentication required) + // Dexto gateway - requires key from `dexto login` + dexto: ['DEXTO_API_KEY'], + // perplexity: ['PERPLEXITY_API_KEY'], + // together: ['TOGETHER_API_KEY'], + // fireworks: ['FIREWORKS_API_KEY'], + // deepseek: ['DEEPSEEK_API_KEY'], +}; + +/** + * Resolves API key for a given provider from environment variables. + * + * @param provider The LLM provider + * @returns Resolved API key or undefined if not found + */ +export function resolveApiKeyForProvider(provider: LLMProvider): string | undefined { + const envVars = PROVIDER_API_KEY_MAP[provider]; + if (!envVars) { + return undefined; + } + + // Try each environment variable in order of preference + for (const envVar of envVars) { + const value = process.env[envVar]; + if (value && value.trim()) { + return value.trim(); + } + } + + return undefined; +} + +/** + * Gets the primary environment variable name for a provider (for display/error messages). + * + * @param provider The LLM provider + * @returns Primary environment variable name + */ +export function getPrimaryApiKeyEnvVar(provider: LLMProvider): string { + const envVars = PROVIDER_API_KEY_MAP[provider]; + return envVars?.[0] || `${provider.toUpperCase()}_API_KEY`; +} diff --git a/dexto/packages/core/src/utils/async-context.ts b/dexto/packages/core/src/utils/async-context.ts new file mode 100644 index 00000000..2ff1cbfa --- /dev/null +++ b/dexto/packages/core/src/utils/async-context.ts @@ -0,0 +1,97 @@ +// TODO: Add fallback strategy for non-Node.js environments (browsers, edge workers) +// For now, this will work in Node.js (CLI API server, standalone deployments). +// Future: Consider session metadata fallback when AsyncLocalStorage is unavailable. + +import { AsyncLocalStorage } from 'async_hooks'; + +/** + * Context data stored in AsyncLocalStorage + * Used for multi-tenant deployments to propagate tenant/user information + */ +export interface AsyncContext { + /** Tenant ID for multi-tenant deployments */ + tenantId?: string; + + /** User ID for tracking which user is making the request */ + userId?: string; +} + +/** + * AsyncLocalStorage instance for storing request context + * This automatically propagates across async boundaries in Node.js + */ +const asyncContext = new AsyncLocalStorage(); + +/** + * Set the current async context + * Should be called at the entry point of a request (e.g., Express middleware) + * + * @param ctx - Context to set + * + * @example + * ```typescript + * // In Express middleware + * app.use((req, res, next) => { + * const { tenantId, userId } = extractAuthFromRequest(req); + * setContext({ tenantId, userId }); + * next(); + * }); + * ``` + */ +export function setContext(ctx: AsyncContext): void { + asyncContext.enterWith(ctx); +} + +/** + * Get the current async context + * Returns undefined if no context is set + * + * @returns Current context or undefined + * + * @example + * ```typescript + * // In plugin or service + * const ctx = getContext(); + * if (ctx?.tenantId) { + * // Use tenant ID for scoped operations + * } + * ``` + */ +export function getContext(): AsyncContext | undefined { + return asyncContext.getStore(); +} + +/** + * Run a function with a specific context + * Useful for testing or when you need to override context temporarily + * + * @param ctx - Context to run with + * @param fn - Function to execute + * @returns Result of the function + * + * @example + * ```typescript + * await runWithContext({ tenantId: 'test-tenant' }, async () => { + * // This code runs with the specified context + * await someOperation(); + * }); + * ``` + */ +export async function runWithContext(ctx: AsyncContext, fn: () => Promise): Promise { + return asyncContext.run(ctx, fn); +} + +/** + * Check if AsyncLocalStorage is available in the current environment + * Returns false in non-Node.js environments (browsers, edge workers) + * + * @returns true if AsyncLocalStorage is available + */ +export function isAsyncContextAvailable(): boolean { + try { + // Check if async_hooks module exists + return typeof AsyncLocalStorage !== 'undefined'; + } catch { + return false; + } +} diff --git a/dexto/packages/core/src/utils/debug.ts b/dexto/packages/core/src/utils/debug.ts new file mode 100644 index 00000000..53e15bd2 --- /dev/null +++ b/dexto/packages/core/src/utils/debug.ts @@ -0,0 +1,13 @@ +export function shouldIncludeRawToolResult(): boolean { + const flag = process.env.DEXTO_DEBUG_TOOL_RESULT_RAW; + if (!flag) return false; + switch (flag.trim().toLowerCase()) { + case '1': + case 'true': + case 'yes': + case 'on': + return true; + default: + return false; + } +} diff --git a/dexto/packages/core/src/utils/defer.test.ts b/dexto/packages/core/src/utils/defer.test.ts new file mode 100644 index 00000000..d50b5423 --- /dev/null +++ b/dexto/packages/core/src/utils/defer.test.ts @@ -0,0 +1,207 @@ +import { describe, it, expect, vi } from 'vitest'; +import { defer } from './defer.js'; + +describe('defer', () => { + describe('sync dispose (using keyword)', () => { + it('should call cleanup on normal scope exit', () => { + const cleanup = vi.fn(); + + function testScope(): void { + using _ = defer(cleanup); + // Normal exit + } + + testScope(); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it('should call cleanup on early return', () => { + const cleanup = vi.fn(); + + function testScope(): string { + using _ = defer(cleanup); + return 'early'; + } + + const result = testScope(); + expect(result).toBe('early'); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it('should call cleanup on throw', () => { + const cleanup = vi.fn(); + + function testScope(): void { + using _ = defer(cleanup); + throw new Error('test error'); + } + + expect(() => testScope()).toThrow('test error'); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it('should execute multiple defers in LIFO order', () => { + const order: number[] = []; + + function testScope(): void { + using _a = defer(() => { + order.push(1); + }); + using _b = defer(() => { + order.push(2); + }); + using _c = defer(() => { + order.push(3); + }); + } + + testScope(); + expect(order).toEqual([3, 2, 1]); + }); + + it('should handle async cleanup function in sync context', async () => { + const cleanup = vi.fn().mockResolvedValue(undefined); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + + function testScope(): void { + using _ = defer(cleanup); + } + + testScope(); + expect(cleanup).toHaveBeenCalledTimes(1); + + // Give time for any promise rejections to surface + await new Promise((resolve) => setTimeout(resolve, 10)); + consoleError.mockRestore(); + }); + }); + + describe('async dispose (await using keyword)', () => { + it('should call async cleanup on normal scope exit', async () => { + const cleanup = vi.fn().mockResolvedValue(undefined); + + async function testScope(): Promise { + await using _ = defer(cleanup); + // Normal exit + } + + await testScope(); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it('should call async cleanup on throw', async () => { + const cleanup = vi.fn().mockResolvedValue(undefined); + + async function testScope(): Promise { + await using _ = defer(cleanup); + throw new Error('async error'); + } + + await expect(testScope()).rejects.toThrow('async error'); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it('should await async cleanup function', async () => { + let cleanupCompleted = false; + const cleanup = vi.fn(async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + cleanupCompleted = true; + }); + + async function testScope(): Promise { + await using _ = defer(cleanup); + } + + await testScope(); + expect(cleanup).toHaveBeenCalledTimes(1); + expect(cleanupCompleted).toBe(true); + }); + + it('should execute multiple async defers in LIFO order', async () => { + const order: number[] = []; + + async function testScope(): Promise { + await using _a = defer(async () => { + order.push(1); + }); + await using _b = defer(async () => { + order.push(2); + }); + await using _c = defer(async () => { + order.push(3); + }); + } + + await testScope(); + expect(order).toEqual([3, 2, 1]); + }); + }); + + describe('Symbol.dispose interface', () => { + it('should implement Symbol.dispose', () => { + const cleanup = vi.fn(); + const deferred = defer(cleanup); + + expect(typeof deferred[Symbol.dispose]).toBe('function'); + + deferred[Symbol.dispose](); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it('should implement Symbol.asyncDispose', async () => { + const cleanup = vi.fn().mockResolvedValue(undefined); + const deferred = defer(cleanup); + + expect(typeof deferred[Symbol.asyncDispose]).toBe('function'); + + await deferred[Symbol.asyncDispose](); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + }); + + describe('error handling', () => { + it('should propagate errors from sync cleanup in sync context', () => { + const cleanup = vi.fn(() => { + throw new Error('cleanup error'); + }); + + function testScope(): void { + using _ = defer(cleanup); + } + + expect(() => testScope()).toThrow('cleanup error'); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it('should propagate errors from async cleanup in async context', async () => { + const cleanup = vi.fn().mockRejectedValue(new Error('async cleanup error')); + + async function testScope(): Promise { + await using _ = defer(cleanup); + } + + await expect(testScope()).rejects.toThrow('async cleanup error'); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it('should log error when async cleanup fails in sync context', async () => { + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + const cleanup = vi.fn().mockRejectedValue(new Error('async fail')); + + function testScope(): void { + using _ = defer(cleanup); + } + + testScope(); + + // Wait for the promise rejection to be caught + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(consoleError).toHaveBeenCalledWith( + 'Deferred async cleanup failed (used sync dispose):', + expect.any(Error) + ); + consoleError.mockRestore(); + }); + }); +}); diff --git a/dexto/packages/core/src/utils/defer.ts b/dexto/packages/core/src/utils/defer.ts new file mode 100644 index 00000000..11cc6625 --- /dev/null +++ b/dexto/packages/core/src/utils/defer.ts @@ -0,0 +1,81 @@ +/** + * TC39 Explicit Resource Management pattern. + * Similar to Go's `defer`, Python's `with`, C#'s `using`. + * + * Benefits: + * - Can't forget cleanup (automatic on scope exit) + * - Works with early returns, throws, aborts + * - Multiple defers execute in LIFO order + * - Cleaner than try/finally chains + * + * @see https://github.com/tc39/proposal-explicit-resource-management + */ + +/** + * Type for cleanup functions - can be sync or async. + */ +export type CleanupFunction = () => void | Promise; + +/** + * Return type for defer() - implements both Disposable and AsyncDisposable. + */ +export interface DeferredCleanup extends Disposable, AsyncDisposable { + [Symbol.dispose]: () => void; + [Symbol.asyncDispose]: () => Promise; +} + +/** + * Creates a deferred cleanup resource. + * + * When used with the `using` keyword, the cleanup function is automatically + * called when the enclosing scope exits - whether normally, via return, + * or via thrown exception. + * + * @param cleanupFn - The function to call on cleanup. Can be sync or async. + * @returns A disposable resource for use with `using` keyword + * + * @example Synchronous cleanup + * ```typescript + * function processData(): void { + * using _ = defer(() => console.log('cleanup')); + * // ... work ... + * // 'cleanup' is logged when scope exits + * } + * ``` + * + * @example Async cleanup with await using + * ```typescript + * async function execute(): Promise { + * await using _ = defer(async () => { + * await closeConnection(); + * }); + * // ... work ... + * } + * ``` + * + * @example Multiple defers (LIFO order) + * ```typescript + * function example(): void { + * using a = defer(() => console.log('first')); + * using b = defer(() => console.log('second')); + * // Logs: 'second' then 'first' (LIFO) + * } + * ``` + */ +export function defer(cleanupFn: CleanupFunction): DeferredCleanup { + return { + [Symbol.dispose](): void { + const result = cleanupFn(); + // If cleanup returns a promise in sync context, fire-and-forget with error logging + if (result instanceof Promise) { + result.catch((err) => { + console.error('Deferred async cleanup failed (used sync dispose):', err); + }); + } + }, + + [Symbol.asyncDispose](): Promise { + return Promise.resolve(cleanupFn()); + }, + }; +} diff --git a/dexto/packages/core/src/utils/error-conversion.ts b/dexto/packages/core/src/utils/error-conversion.ts new file mode 100644 index 00000000..44b45ee5 --- /dev/null +++ b/dexto/packages/core/src/utils/error-conversion.ts @@ -0,0 +1,81 @@ +/** + * Utility functions for converting various error types to proper Error instances + * with meaningful messages instead of "[object Object]" + */ + +import { safeStringify } from './safe-stringify.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; + +/** + * Converts any error value to an Error instance with a meaningful message + * + * @param error - The error value to convert (can be Error, object, string, etc.) + * @returns Error instance with extracted or serialized message + */ +export function toError(error: unknown, logger: IDextoLogger): Error { + if (error instanceof Error) { + logger.info(`error is already an Error: ${error.message}`); + return error; + } + + if (error && typeof error === 'object') { + const errorObj = error as any; + + // Handle Vercel AI SDK error format: parse error.error.responseBody JSON + // TODO: this is a workaround because vercel's ai sdk errors returned are untyped garbage + // Summary: onError callback returns this weird shape. If we try to catch the promise.all block + // we get a useless error (added comments near the catch block). + // Improve this once vercel ai sdk errors are typed properly. + + // Handle Vercel AI SDK error format: error.error.data.error.message + if (errorObj.error?.data?.error?.message) { + logger.info( + `Extracted error from error.error.data.error.message: ${errorObj.error.data.error.message}` + ); + return new Error(errorObj.error.data.error.message, { cause: error }); + } + + if (errorObj.error?.responseBody && typeof errorObj.error.responseBody === 'string') { + try { + const parsed = JSON.parse(errorObj.error.responseBody); + if (parsed?.error?.message) { + logger.info( + `Extracted error from error.error.responseBody: ${parsed.error.message}` + ); + return new Error(parsed.error.message, { cause: error }); + } + } catch { + logger.info(`Failed to parse error.error.responseBody as JSON`); + // Failed to parse, continue to other checks + } + } + + // Try to extract meaningful message from error object + if ('message' in error && typeof (error as { message?: unknown }).message === 'string') { + return new Error((error as { message: string }).message, { cause: error }); + } + if ('error' in error && typeof (error as { error?: unknown }).error === 'string') { + return new Error((error as { error: string }).error, { cause: error }); + } + if ('details' in error && typeof (error as { details?: unknown }).details === 'string') { + return new Error((error as { details: string }).details, { cause: error }); + } + if ( + 'description' in error && + typeof (error as { description?: unknown }).description === 'string' + ) { + return new Error((error as { description: string }).description, { cause: error }); + } + // Fallback to safe serialization for complex objects + const serialized = safeStringify(error); // Uses truncation + circular-ref safety + logger.info(`falling back to safe serialization for complex objects: ${serialized}`); + return new Error(serialized); + } + + if (typeof error === 'string') { + return new Error(error, { cause: error }); + } + + // For primitives and other types + return new Error(String(error), { cause: error as unknown }); +} diff --git a/dexto/packages/core/src/utils/execution-context.ts b/dexto/packages/core/src/utils/execution-context.ts new file mode 100644 index 00000000..25fd9262 --- /dev/null +++ b/dexto/packages/core/src/utils/execution-context.ts @@ -0,0 +1,93 @@ +// packages/core/src/utils/execution-context.ts +// TODO: (migration) This file is duplicated in @dexto/agent-management for short-term compatibility +// Remove from core once all services accept paths via initialization options + +import { walkUpDirectories } from './fs-walk.js'; +import { readFileSync } from 'fs'; +import * as path from 'path'; + +export type ExecutionContext = 'dexto-source' | 'dexto-project' | 'global-cli'; + +/** + * Check if directory is the dexto source code itself + * @param dirPath Directory to check + * @returns True if directory contains the dexto source monorepo (top-level). + */ +function isDextoSourceDirectory(dirPath: string): boolean { + const packageJsonPath = path.join(dirPath, 'package.json'); + + try { + const pkg = JSON.parse(readFileSync(packageJsonPath, 'utf-8')); + // Monorepo root must be named 'dexto-monorepo'. No other names are treated as source root. + return pkg.name === 'dexto-monorepo'; + } catch { + return false; + } +} + +/** + * Check if directory is a project that uses dexto as dependency (but is not dexto source) + * @param dirPath Directory to check + * @returns True if directory has dexto as dependency but is not dexto source + */ +function isDextoProjectDirectory(dirPath: string): boolean { + const packageJsonPath = path.join(dirPath, 'package.json'); + + try { + const pkg = JSON.parse(readFileSync(packageJsonPath, 'utf-8')); + + // Not internal dexto packages themselves + if (pkg.name === 'dexto' || pkg.name === '@dexto/core' || pkg.name === '@dexto/webui') { + return false; + } + + // Check if has dexto or @dexto/core as dependency + const allDeps = { + ...(pkg.dependencies ?? {}), + ...(pkg.devDependencies ?? {}), + ...(pkg.peerDependencies ?? {}), + }; + + return 'dexto' in allDeps || '@dexto/core' in allDeps; + } catch { + return false; + } +} + +/** + * Find dexto source root directory + * @param startPath Starting directory path + * @returns Dexto source root directory or null if not found + */ +export function findDextoSourceRoot(startPath: string = process.cwd()): string | null { + return walkUpDirectories(startPath, isDextoSourceDirectory); +} + +/** + * Find dexto project root directory (projects using dexto as dependency) + * @param startPath Starting directory path + * @returns Dexto project root directory or null if not found + */ +export function findDextoProjectRoot(startPath: string = process.cwd()): string | null { + return walkUpDirectories(startPath, isDextoProjectDirectory); +} + +/** + * Detect current execution context - standardized across codebase + * @param startPath Starting directory path (defaults to process.cwd()) + * @returns Execution context + */ +export function getExecutionContext(startPath: string = process.cwd()): ExecutionContext { + // Check for Dexto source context first (most specific) + if (findDextoSourceRoot(startPath)) { + return 'dexto-source'; + } + + // Check for Dexto project context + if (findDextoProjectRoot(startPath)) { + return 'dexto-project'; + } + + // Default to global CLI context + return 'global-cli'; +} diff --git a/dexto/packages/core/src/utils/fs-walk.ts b/dexto/packages/core/src/utils/fs-walk.ts new file mode 100644 index 00000000..a252b951 --- /dev/null +++ b/dexto/packages/core/src/utils/fs-walk.ts @@ -0,0 +1,30 @@ +// TODO: (migration) This file is duplicated in @dexto/agent-management for short-term compatibility +// Remove from core once path utilities are fully migrated + +import * as path from 'path'; + +/** + * Generic directory walker that searches up the directory tree + * @param startPath Starting directory path + * @param predicate Function that returns true when the desired condition is found + * @returns The directory path where the condition was met, or null if not found + */ +export function walkUpDirectories( + startPath: string, + predicate: (dirPath: string) => boolean +): string | null { + let currentPath = path.resolve(startPath); + const rootPath = path.parse(currentPath).root; + + while (true) { + if (predicate(currentPath)) { + return currentPath; + } + if (currentPath === rootPath) break; + const parent = path.dirname(currentPath); + if (parent === currentPath) break; // safety for exotic paths + currentPath = parent; + } + + return null; +} diff --git a/dexto/packages/core/src/utils/index.ts b/dexto/packages/core/src/utils/index.ts new file mode 100644 index 00000000..64fc6c48 --- /dev/null +++ b/dexto/packages/core/src/utils/index.ts @@ -0,0 +1,26 @@ +// TODO: (migration) path.js, execution-context.js, fs-walk.js, env-file.js +// are duplicated in @dexto/agent-management for Node-specific environment management. +// Core still needs these for FilePromptProvider, MCPClient, and FileContributor functionality. +// These will remain in core until we refactor those features to be dependency-injected. + +export * from './path.js'; +export * from './service-initializer.js'; +export * from './zod-schema-converter.js'; +export * from './result.js'; +export * from './error-conversion.js'; +export * from './execution-context.js'; +export * from './fs-walk.js'; +export * from './redactor.js'; +export * from './debug.js'; +export * from './safe-stringify.js'; +export * from './api-key-resolver.js'; +export * from './defer.js'; +export * from './async-context.js'; + +// API key STORAGE has been moved to @dexto/agent-management +// These functions write to .env files and are CLI/server concerns, not core runtime +// Import from '@dexto/agent-management' instead: +// - updateEnvFile +// - saveProviderApiKey +// - getProviderKeyStatus +// - listProviderKeyStatus diff --git a/dexto/packages/core/src/utils/path.test.ts b/dexto/packages/core/src/utils/path.test.ts new file mode 100644 index 00000000..3f870053 --- /dev/null +++ b/dexto/packages/core/src/utils/path.test.ts @@ -0,0 +1,392 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { tmpdir, homedir } from 'os'; +import { getDextoPath, getDextoGlobalPath, getDextoEnvPath, findPackageRoot } from './path.js'; +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; + +function createTempDir() { + return fs.mkdtempSync(path.join(tmpdir(), 'dexto-test-')); +} + +function createTempDirStructure(structure: Record, baseDir?: string): string { + const tempDir = baseDir || createTempDir(); + + for (const [filePath, content] of Object.entries(structure)) { + const fullPath = path.join(tempDir, filePath); + const dir = path.dirname(fullPath); + + // Create directory if it doesn't exist + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + if (typeof content === 'string') { + fs.writeFileSync(fullPath, content); + } else if (typeof content === 'object') { + fs.writeFileSync(fullPath, JSON.stringify(content, null, 2)); + } + } + + return tempDir; +} + +describe('getDextoPath', () => { + let tempDir: string; + + afterEach(() => { + if (tempDir) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + describe('in dexto project', () => { + beforeEach(() => { + tempDir = createTempDirStructure({ + 'package.json': { + name: 'test-project', + dependencies: { dexto: '^1.0.0' }, + }, + }); + }); + + it('returns project-local path for logs', () => { + const result = getDextoPath('logs', 'test.log', tempDir); + expect(result).toBe(path.join(tempDir, '.dexto', 'logs', 'test.log')); + }); + + it('returns project-local path for database', () => { + const result = getDextoPath('database', 'dexto.db', tempDir); + expect(result).toBe(path.join(tempDir, '.dexto', 'database', 'dexto.db')); + }); + + it('returns directory path when no filename provided', () => { + const result = getDextoPath('config', undefined, tempDir); + expect(result).toBe(path.join(tempDir, '.dexto', 'config')); + }); + + it('works from nested directories', () => { + const nestedDir = path.join(tempDir, 'src', 'app'); + fs.mkdirSync(nestedDir, { recursive: true }); + + const result = getDextoPath('logs', 'app.log', nestedDir); + expect(result).toBe(path.join(tempDir, '.dexto', 'logs', 'app.log')); + }); + }); + + describe('outside dexto project (global)', () => { + beforeEach(() => { + tempDir = createTempDirStructure({ + 'package.json': { + name: 'regular-project', + dependencies: { express: '^4.0.0' }, + }, + }); + }); + + it('returns global path when not in dexto project', () => { + const originalCwd = process.cwd(); + try { + process.chdir(tempDir); + const result = getDextoPath('logs', 'global.log'); + expect(result).toContain('.dexto'); + expect(result).toContain('logs'); + expect(result).toContain('global.log'); + expect(result).not.toContain(tempDir); + } finally { + process.chdir(originalCwd); + } + }); + }); +}); + +describe('getDextoGlobalPath', () => { + let tempDir: string; + + afterEach(() => { + if (tempDir) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + describe('basic functionality', () => { + it('returns global agents directory', () => { + const result = getDextoGlobalPath('agents'); + expect(result).toContain('.dexto'); + expect(result).toContain('agents'); + expect(path.isAbsolute(result)).toBe(true); + }); + + it('returns global path with filename', () => { + const result = getDextoGlobalPath('agents', 'database-agent'); + expect(result).toContain('.dexto'); + expect(result).toContain('agents'); + expect(result).toContain('database-agent'); + expect(path.isAbsolute(result)).toBe(true); + }); + + it('handles different types correctly', () => { + const agents = getDextoGlobalPath('agents'); + const logs = getDextoGlobalPath('logs'); + const cache = getDextoGlobalPath('cache'); + + expect(agents).toContain('agents'); + expect(logs).toContain('logs'); + expect(cache).toContain('cache'); + }); + }); + + describe('in dexto project context', () => { + beforeEach(() => { + tempDir = createTempDirStructure({ + 'package.json': { + name: 'test-project', + dependencies: { dexto: '^1.0.0' }, + }, + }); + }); + + it('always returns global path, never project-relative', () => { + // getDextoPath returns project-relative + const projectPath = getDextoPath('agents', 'test-agent', tempDir); + expect(projectPath).toBe(path.join(tempDir, '.dexto', 'agents', 'test-agent')); + + // getDextoGlobalPath should ALWAYS return global, never project-relative + const globalPath = getDextoGlobalPath('agents', 'test-agent'); + expect(globalPath).toContain('.dexto'); + expect(globalPath).toContain('agents'); + expect(globalPath).toContain('test-agent'); + expect(globalPath).not.toContain(tempDir); // Key difference! + expect(path.isAbsolute(globalPath)).toBe(true); + }); + }); + + describe('outside dexto project context', () => { + beforeEach(() => { + tempDir = createTempDirStructure({ + 'package.json': { + name: 'regular-project', + dependencies: { express: '^4.0.0' }, + }, + }); + }); + + it('returns global path (same as in project context)', () => { + const globalPath = getDextoGlobalPath('agents', 'test-agent'); + expect(globalPath).toContain('.dexto'); + expect(globalPath).toContain('agents'); + expect(globalPath).toContain('test-agent'); + expect(globalPath).not.toContain(tempDir); + expect(path.isAbsolute(globalPath)).toBe(true); + }); + }); +}); + +describe('findPackageRoot', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = createTempDir(); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it('returns null if no package.json found', () => { + const result = findPackageRoot(tempDir); + expect(result).toBeNull(); + }); + + it('returns the directory containing package.json', () => { + fs.writeFileSync(path.join(tempDir, 'package.json'), JSON.stringify({ name: 'test-pkg' })); + const result = findPackageRoot(tempDir); + expect(result).toBe(tempDir); + }); + + it('finds package.json by walking up directories', () => { + const nestedDir = path.join(tempDir, 'nested', 'deep'); + fs.mkdirSync(nestedDir, { recursive: true }); + fs.writeFileSync(path.join(tempDir, 'package.json'), JSON.stringify({ name: 'test-pkg' })); + + const result = findPackageRoot(nestedDir); + expect(result).toBe(tempDir); + }); +}); + +// resolveBundledScript tests have been moved to @dexto/agent-management + +describe('getDextoEnvPath', () => { + describe('in dexto project', () => { + let tempDir: string; + let originalCwd: string; + + beforeEach(() => { + originalCwd = process.cwd(); + tempDir = createTempDirStructure({ + 'package.json': { + name: 'test-project', + dependencies: { dexto: '^1.0.0' }, + }, + }); + }); + + afterEach(() => { + process.chdir(originalCwd); + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it('returns project root .env path', () => { + process.chdir(tempDir); + const result = getDextoEnvPath(tempDir); + expect(result).toBe(path.join(tempDir, '.env')); + }); + }); + + describe('in dexto source', () => { + let tempDir: string; + let originalCwd: string; + const originalDevMode = process.env.DEXTO_DEV_MODE; + + beforeEach(() => { + originalCwd = process.cwd(); + tempDir = createTempDirStructure({ + 'package.json': { + name: 'dexto-monorepo', + version: '1.0.0', + }, + 'agents/default-agent.yml': 'mcpServers: {}', + }); + }); + + afterEach(() => { + process.chdir(originalCwd); + fs.rmSync(tempDir, { recursive: true, force: true }); + // Restore original env + if (originalDevMode === undefined) { + delete process.env.DEXTO_DEV_MODE; + } else { + process.env.DEXTO_DEV_MODE = originalDevMode; + } + }); + + it('returns repo .env when DEXTO_DEV_MODE=true', () => { + process.chdir(tempDir); + process.env.DEXTO_DEV_MODE = 'true'; + const result = getDextoEnvPath(tempDir); + expect(result).toBe(path.join(tempDir, '.env')); + }); + + it('returns global ~/.dexto/.env when DEXTO_DEV_MODE is not set', () => { + process.chdir(tempDir); + delete process.env.DEXTO_DEV_MODE; + const result = getDextoEnvPath(tempDir); + expect(result).toBe(path.join(homedir(), '.dexto', '.env')); + }); + + it('returns global ~/.dexto/.env when DEXTO_DEV_MODE=false', () => { + process.chdir(tempDir); + process.env.DEXTO_DEV_MODE = 'false'; + const result = getDextoEnvPath(tempDir); + expect(result).toBe(path.join(homedir(), '.dexto', '.env')); + }); + }); + + describe('in global-cli context', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = createTempDirStructure({ + 'package.json': { + name: 'regular-project', + dependencies: { express: '^4.0.0' }, + }, + }); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it('returns global ~/.dexto/.env path', () => { + const result = getDextoEnvPath(tempDir); + expect(result).toBe(path.join(homedir(), '.dexto', '.env')); + }); + }); +}); + +describe('real-world execution contexts', () => { + describe('SDK usage in project', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = createTempDirStructure({ + 'package.json': { + name: 'my-app', + dependencies: { dexto: '^1.0.0' }, + }, + 'src/dexto/agents/default-agent.yml': 'mcpServers: {}', + }); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it('uses project-local storage', () => { + const logPath = getDextoPath('logs', 'dexto.log', tempDir); + const dbPath = getDextoPath('database', 'dexto.db', tempDir); + + expect(logPath).toBe(path.join(tempDir, '.dexto', 'logs', 'dexto.log')); + expect(dbPath).toBe(path.join(tempDir, '.dexto', 'database', 'dexto.db')); + }); + }); + + describe('CLI in dexto source', () => { + let tempDir: string; + const originalDevMode = process.env.DEXTO_DEV_MODE; + + beforeEach(() => { + tempDir = createTempDirStructure({ + 'package.json': { + name: 'dexto-monorepo', + version: '1.0.0', + }, + 'agents/default-agent.yml': 'mcpServers: {}', + }); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + // Restore original env + if (originalDevMode === undefined) { + delete process.env.DEXTO_DEV_MODE; + } else { + process.env.DEXTO_DEV_MODE = originalDevMode; + } + }); + + it('uses local repo storage when DEXTO_DEV_MODE=true', () => { + process.env.DEXTO_DEV_MODE = 'true'; + const logPath = getDextoPath('logs', 'dexto.log', tempDir); + expect(logPath).toBe(path.join(tempDir, '.dexto', 'logs', 'dexto.log')); + }); + + it('uses global storage when DEXTO_DEV_MODE is not set', () => { + delete process.env.DEXTO_DEV_MODE; + const logPath = getDextoPath('logs', 'dexto.log', tempDir); + expect(logPath).toContain('.dexto'); + expect(logPath).toContain('logs'); + expect(logPath).toContain('dexto.log'); + expect(logPath).not.toContain(tempDir); // Should be global, not local + }); + + it('uses global storage when DEXTO_DEV_MODE=false', () => { + process.env.DEXTO_DEV_MODE = 'false'; + const logPath = getDextoPath('logs', 'dexto.log', tempDir); + expect(logPath).toContain('.dexto'); + expect(logPath).toContain('logs'); + expect(logPath).toContain('dexto.log'); + expect(logPath).not.toContain(tempDir); // Should be global, not local + }); + }); +}); diff --git a/dexto/packages/core/src/utils/path.ts b/dexto/packages/core/src/utils/path.ts new file mode 100644 index 00000000..747f8871 --- /dev/null +++ b/dexto/packages/core/src/utils/path.ts @@ -0,0 +1,195 @@ +// TODO: (migration) This file is duplicated in @dexto/agent-management for short-term compatibility +// Remove from core once all services accept paths via initialization options + +import * as path from 'path'; +import { existsSync } from 'fs'; +import { promises as fs } from 'fs'; +import { homedir } from 'os'; +import { walkUpDirectories } from './fs-walk.js'; +import { + getExecutionContext, + findDextoSourceRoot, + findDextoProjectRoot, +} from './execution-context.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; + +/** + * Standard path resolver for logs/db/config/anything in dexto projects + * Context-aware with dev mode support: + * - dexto-source + DEXTO_DEV_MODE=true: Use local repo .dexto (isolated testing) + * - dexto-source (normal): Use global ~/.dexto (user experience) + * - dexto-project: Use project-local .dexto + * - global-cli: Use global ~/.dexto + * @param type Path type (logs, database, config, etc.) + * @param filename Optional filename to append + * @param startPath Starting directory for project detection + * @returns Absolute path to the requested location + */ +export function getDextoPath(type: string, filename?: string, startPath?: string): string { + const context = getExecutionContext(startPath); + + let basePath: string; + + switch (context) { + case 'dexto-source': { + // Dev mode: use local repo .dexto for isolated testing + // Normal mode: use global ~/.dexto for user experience + const isDevMode = process.env.DEXTO_DEV_MODE === 'true'; + if (isDevMode) { + const sourceRoot = findDextoSourceRoot(startPath); + if (!sourceRoot) { + throw new Error('Not in dexto source context'); + } + basePath = path.join(sourceRoot, '.dexto', type); + } else { + basePath = path.join(homedir(), '.dexto', type); + } + break; + } + case 'dexto-project': { + const projectRoot = findDextoProjectRoot(startPath); + if (!projectRoot) { + throw new Error('Not in dexto project context'); + } + basePath = path.join(projectRoot, '.dexto', type); + break; + } + case 'global-cli': { + basePath = path.join(homedir(), '.dexto', type); + break; + } + default: { + throw new Error(`Unknown execution context: ${context}`); + } + } + + return filename ? path.join(basePath, filename) : basePath; +} + +/** + * Global path resolver that ALWAYS returns paths in the user's home directory + * Used for agent registry and other global-only resources that should not be project-relative + * @param type Path type (agents, cache, etc.) + * @param filename Optional filename to append + * @returns Absolute path to the global location (~/.dexto/...) + */ +export function getDextoGlobalPath(type: string, filename?: string): string { + // ALWAYS return global path, ignore project context + const basePath = path.join(homedir(), '.dexto', type); + return filename ? path.join(basePath, filename) : basePath; +} + +/** + * Copy entire directory recursively + * @param src Source directory path + * @param dest Destination directory path + */ +export async function copyDirectory(src: string, dest: string): Promise { + await fs.mkdir(dest, { recursive: true }); + + const entries = await fs.readdir(src, { withFileTypes: true }); + + for (const entry of entries) { + const srcPath = path.join(src, entry.name); + const destPath = path.join(dest, entry.name); + + if (entry.isDirectory()) { + await copyDirectory(srcPath, destPath); + } else { + await fs.copyFile(srcPath, destPath); + } + } +} + +/** + * Check if string looks like a file path vs registry name + * @param str String to check + * @returns True if looks like a path, false if looks like a registry name + */ +export function isPath(str: string): boolean { + // Absolute paths + if (path.isAbsolute(str)) return true; + + // Relative paths with separators + if (/[\\/]/.test(str)) return true; + + // File extensions + if (/\.(ya?ml|json)$/i.test(str)) return true; + + return false; +} + +/** + * Find package root (for other utilities) + * @param startPath Starting directory path + * @returns Directory containing package.json or null + */ +export function findPackageRoot(startPath: string = process.cwd()): string | null { + return walkUpDirectories(startPath, (dirPath) => { + const pkgPath = path.join(dirPath, 'package.json'); + return existsSync(pkgPath); + }); +} + +// resolveBundledScript has been moved to @dexto/agent-management +// Core no longer needs to resolve bundled script paths - users should use +// ${{dexto.agent_dir}} template variables in their configs instead + +/** + * Ensure ~/.dexto directory exists for global storage + */ +export async function ensureDextoGlobalDirectory(): Promise { + const dextoDir = path.join(homedir(), '.dexto'); + try { + await fs.mkdir(dextoDir, { recursive: true }); + } catch (error) { + // Directory might already exist, ignore EEXIST errors + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') { + throw error; + } + } +} + +/** + * Get the appropriate .env file path for saving API keys. + * Uses the same project detection logic as other dexto paths. + * + * @param startPath Starting directory for project detection + * @param logger Optional logger instance for logging + * @returns Absolute path to .env file for saving + */ +export function getDextoEnvPath(startPath: string = process.cwd(), logger?: IDextoLogger): string { + const context = getExecutionContext(startPath); + let envPath = ''; + switch (context) { + case 'dexto-source': { + // Dev mode: use local repo .env for isolated testing + // Normal mode: use global ~/.dexto/.env for user experience + const isDevMode = process.env.DEXTO_DEV_MODE === 'true'; + if (isDevMode) { + const sourceRoot = findDextoSourceRoot(startPath); + if (!sourceRoot) { + throw new Error('Not in dexto source context'); + } + envPath = path.join(sourceRoot, '.env'); + } else { + envPath = path.join(homedir(), '.dexto', '.env'); + } + break; + } + case 'dexto-project': { + const projectRoot = findDextoProjectRoot(startPath); + if (!projectRoot) { + throw new Error('Not in dexto project context'); + } + envPath = path.join(projectRoot, '.env'); + break; + } + case 'global-cli': { + envPath = path.join(homedir(), '.dexto', '.env'); + break; + } + } + logger?.debug(`Dexto env path: ${envPath}, context: ${context}`); + return envPath; +} diff --git a/dexto/packages/core/src/utils/redactor.test.ts b/dexto/packages/core/src/utils/redactor.test.ts new file mode 100644 index 00000000..b1f3625f --- /dev/null +++ b/dexto/packages/core/src/utils/redactor.test.ts @@ -0,0 +1,196 @@ +import { describe, test, expect } from 'vitest'; +import { redactSensitiveData as redact } from './redactor.js'; + +describe('redact', () => { + // Basic field redaction + test('should redact a single sensitive field', () => { + expect(redact({ apiKey: 'secret' })).toEqual({ apiKey: '[REDACTED]' }); + }); + + test('should redact multiple sensitive fields', () => { + expect(redact({ apiKey: 'secret', password: 'pass' })).toEqual({ + apiKey: '[REDACTED]', + password: '[REDACTED]', + }); + }); + + test('should perform case-insensitive field matching', () => { + expect(redact({ ApiKey: 'secret', PASSWORD: 'pass' })).toEqual({ + ApiKey: '[REDACTED]', + PASSWORD: '[REDACTED]', + }); + }); + + test('should handle mixed sensitive and non-sensitive fields', () => { + expect(redact({ apiKey: 'secret', name: 'john' })).toEqual({ + apiKey: '[REDACTED]', + name: 'john', + }); + }); + + test('should handle field names with underscores', () => { + expect(redact({ api_key: 'secret', access_token: 'token' })).toEqual({ + api_key: '[REDACTED]', + access_token: '[REDACTED]', + }); + }); + + // Array Processing + test('should handle array of objects with sensitive fields', () => { + expect(redact([{ apiKey: 'secret' }, { password: 'pass' }])).toEqual([ + { apiKey: '[REDACTED]' }, + { password: '[REDACTED]' }, + ]); + }); + + test('should handle array of strings with patterns', () => { + expect(redact(['sk-thisisafakekeyofsufficientlength', 'normal string'])).toEqual([ + '[REDACTED]', + 'normal string', + ]); + }); + + test('should handle mixed array types', () => { + expect(redact([{ apiKey: 'secret' }, 'sk-thisisafakekeyofsufficientlength', 42])).toEqual([ + { apiKey: '[REDACTED]' }, + '[REDACTED]', + 42, + ]); + }); + + test('should handle an empty array', () => { + expect(redact([])).toEqual([]); + }); + + test('should handle nested arrays', () => { + expect(redact([[{ apiKey: 'secret' }]])).toEqual([[{ apiKey: '[REDACTED]' }]]); + }); + + // Object Nesting + test('should handle deeply nested sensitive fields', () => { + expect(redact({ user: { config: { apiKey: 'secret' } } })).toEqual({ + user: { config: { apiKey: '[REDACTED]' } }, + }); + }); + + test('should handle mixed nesting levels', () => { + expect(redact({ apiKey: 'secret', user: { password: 'pass' } })).toEqual({ + apiKey: '[REDACTED]', + user: { password: '[REDACTED]' }, + }); + }); + + test('should handle array within object', () => { + expect(redact({ users: [{ apiKey: 'secret' }] })).toEqual({ + users: [{ apiKey: '[REDACTED]' }], + }); + }); + + test('should handle object within array', () => { + expect(redact([{ nested: { apiKey: 'secret' } }])).toEqual([ + { nested: { apiKey: '[REDACTED]' } }, + ]); + }); + + // Primitive Types + test('should return primitives unchanged', () => { + expect(redact(null)).toBeNull(); + expect(redact(undefined)).toBeUndefined(); + expect(redact(42)).toBe(42); + expect(redact(true)).toBe(true); + const s = Symbol('foo'); + expect(redact(s)).toBe(s); + }); + + // Sensitive Patterns + describe('Sensitive Patterns in Strings', () => { + test('should redact OpenAI API keys', () => { + const text = 'My API key is sk-thisisafakekeyofsufficientlength'; + expect(redact(text)).toBe('My API key is [REDACTED]'); + }); + + test('should redact Bearer tokens', () => { + const text = 'Authorization: Bearer my-secret-token-123'; + expect(redact(text)).toBe('Authorization: [REDACTED]'); + }); + + test('should redact emails', () => { + const text = 'Contact me at test@example.com'; + expect(redact(text)).toBe('Contact me at [REDACTED]'); + }); + + test('should not redact normal strings', () => { + const text = 'This is a normal sentence.'; + expect(redact(text)).toBe(text); + }); + + test('should redact standalone JWT tokens', () => { + const jwt = + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; + expect(redact(jwt)).toBe('[REDACTED]'); + }); + }); + + // Signed URLs (should NOT be redacted) + describe('Signed URLs', () => { + test('should NOT redact Supabase signed URLs', () => { + const url = + 'https://xxx.supabase.co/storage/v1/object/sign/bucket/file.dat?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; + expect(redact(url)).toBe(url); + }); + + test('should NOT redact AWS S3 presigned URLs', () => { + const url = + 'https://bucket.s3.us-east-1.amazonaws.com/file.dat?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=xxx'; + expect(redact(url)).toBe(url); + }); + + test('should NOT redact Google Cloud Storage signed URLs', () => { + const url = + 'https://storage.googleapis.com/bucket/file.dat?Expires=123&GoogleAccessId=xxx&Signature=xxx'; + expect(redact(url)).toBe(url); + }); + + test('should still redact JWT tokens in non-URL contexts', () => { + const text = + 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; + expect(redact(text)).toBe('[REDACTED]'); + }); + }); + + // Circular References + describe('Circular References', () => { + test('should handle circular references in objects', () => { + const obj: any = { a: 1 }; + obj.b = obj; // Circular reference + const redacted = redact(obj); + expect(redacted).toEqual({ a: 1, b: '[REDACTED_CIRCULAR]' }); + }); + + test('should handle circular references in arrays', () => { + const arr: any[] = [1]; + arr.push(arr); // Circular reference + const redacted = redact(arr); + expect(redacted).toEqual([1, '[REDACTED_CIRCULAR]']); + }); + + test('should handle complex circular references', () => { + const obj1: any = { name: 'obj1' }; + const obj2: any = { name: 'obj2' }; + obj1.child = obj2; + obj2.parent = obj1; // Circular reference + const redacted = redact(obj1); + expect(redacted).toEqual({ + name: 'obj1', + child: { name: 'obj2', parent: '[REDACTED_CIRCULAR]' }, + }); + }); + + test('should handle circular references in nested arrays', () => { + const arr: any[] = [1, [2]]; + (arr[1] as any[]).push(arr); + const redacted = redact(arr); + expect(redacted).toEqual([1, [2, '[REDACTED_CIRCULAR]']]); + }); + }); +}); diff --git a/dexto/packages/core/src/utils/redactor.ts b/dexto/packages/core/src/utils/redactor.ts new file mode 100644 index 00000000..c83292ff --- /dev/null +++ b/dexto/packages/core/src/utils/redactor.ts @@ -0,0 +1,140 @@ +/** + * Utility to redact sensitive information from objects, arrays, and strings. + * - Redacts by field name (e.g., apiKey, token, password, etc.) + * - Redacts by value pattern (e.g., OpenAI keys, Bearer tokens, emails) + * - Handles deeply nested structures and circular references + * - Recursive and preserves structure + * - Easy to extend + */ + +// List of sensitive field names to redact (case-insensitive) +const SENSITIVE_FIELDS = [ + 'apikey', + 'api_key', + 'token', + 'access_token', + 'refresh_token', + 'password', + 'secret', +]; + +// List of file data field names that should be truncated for logging +const FILE_DATA_FIELDS = [ + 'base64', + 'filedata', + 'file_data', + 'imagedata', + 'image_data', + 'audiodata', + 'audio_data', + 'data', +]; + +// List of regex patterns to redact sensitive values +const SENSITIVE_PATTERNS: RegExp[] = [ + /\bsk-[A-Za-z0-9]{20,}\b/g, // OpenAI API keys (at least 20 chars after sk-) + /\bBearer\s+[A-Za-z0-9\-_.=]+\b/gi, // Bearer tokens + /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, // Emails +]; + +// JWT pattern - applied selectively (not to signed URLs) +const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]*\.[A-Za-z0-9_-]*\.[A-Za-z0-9_-]*/g; + +// Patterns that indicate a URL contains a signed token that should NOT be redacted +// These are legitimate shareable URLs, not sensitive credentials +const SIGNED_URL_PATTERNS = [ + /supabase\.co\/storage\/.*\?token=/i, // Supabase signed URLs + /\.r2\.cloudflarestorage\.com\/.*\?/i, // Cloudflare R2 signed URLs + /\.s3\..*amazonaws\.com\/.*\?(X-Amz-|AWSAccessKeyId)/i, // AWS S3 presigned URLs + /storage\.googleapis\.com\/.*\?/i, // Google Cloud Storage signed URLs +]; + +const REDACTED = '[REDACTED]'; +const REDACTED_CIRCULAR = '[REDACTED_CIRCULAR]'; +const FILE_DATA_TRUNCATED = '[FILE_DATA_TRUNCATED]'; + +/** + * Determines if a string looks like base64-encoded file data + * @param value - String to check + * @returns true if it appears to be large base64 data + */ +function isLargeBase64Data(value: string): boolean { + // Check if it's a long string that looks like base64 + return value.length > 1000 && /^[A-Za-z0-9+/=]{1000,}$/.test(value.substring(0, 1000)); +} + +/** + * Truncates large file data for logging purposes + * @param value - The value to potentially truncate + * @param key - The field name + * @param parent - The parent object for context checking + * @returns Truncated value with metadata or original value + */ +function truncateFileData(value: unknown, key: string, parent?: Record): unknown { + if (typeof value !== 'string') return value; + const lowerKey = key.toLowerCase(); + // Gate "data" by presence of file-ish sibling metadata to avoid false positives + const hasFileContext = + !!parent && ('mimeType' in parent || 'filename' in parent || 'fileName' in parent); + const looksLikeFileField = + FILE_DATA_FIELDS.includes(lowerKey) || (lowerKey === 'data' && hasFileContext); + if (looksLikeFileField && isLargeBase64Data(value)) { + // Only log a concise marker + size; no content preview to prevent leakage + return `${FILE_DATA_TRUNCATED} (${value.length} chars)`; + } + return value; +} + +/** + * Redacts sensitive data from an object, array, or string. + * Handles circular references gracefully. + * @param input - The data to redact + * @param seen - Internal set to track circular references + * @returns The redacted data + */ +/** + * Checks if a string is a signed URL that should not have its token redacted + */ +function isSignedUrl(value: string): boolean { + return SIGNED_URL_PATTERNS.some((pattern) => pattern.test(value)); +} + +export function redactSensitiveData(input: unknown, seen = new WeakSet()): unknown { + if (typeof input === 'string') { + let result = input; + for (const pattern of SENSITIVE_PATTERNS) { + result = result.replace(pattern, REDACTED); + } + // Only redact JWTs if they're not part of a signed URL + // Signed URLs are meant to be shared and their tokens are not credentials + if (!isSignedUrl(result)) { + result = result.replace(JWT_PATTERN, REDACTED); + } + return result; + } + if (Array.isArray(input)) { + if (seen.has(input)) return REDACTED_CIRCULAR; + seen.add(input); + return input.map((item) => redactSensitiveData(item, seen)); + } + if (input && typeof input === 'object') { + if (seen.has(input)) return REDACTED_CIRCULAR; + seen.add(input); + const result: any = {}; + for (const [key, value] of Object.entries(input)) { + if (SENSITIVE_FIELDS.includes(key.toLowerCase())) { + result[key] = REDACTED; + } else { + // First truncate file data (with parent context), then recursively redact + const truncatedValue = truncateFileData( + value, + key, + input as Record + ); + result[key] = redactSensitiveData(truncatedValue, seen); + } + } + return result; + } + return input; +} diff --git a/dexto/packages/core/src/utils/result.test.ts b/dexto/packages/core/src/utils/result.test.ts new file mode 100644 index 00000000..271c335b --- /dev/null +++ b/dexto/packages/core/src/utils/result.test.ts @@ -0,0 +1,296 @@ +import { describe, test, expect } from 'vitest'; +import { z, ZodError } from 'zod'; +import { zodToIssues, ok, fail, hasErrors, splitIssues } from './result.js'; +import { ErrorScope, ErrorType } from '../errors/index.js'; +import type { Issue } from '../errors/index.js'; + +// Helper to create test issues with less boilerplate +const makeIssue = ( + code: string, + severity: 'error' | 'warning', + message = `Test ${severity}` +): Issue => ({ + code, + message, + severity, + scope: ErrorScope.AGENT, + type: ErrorType.USER, + context: {}, +}); + +describe('zodToIssues', () => { + describe('standard error handling', () => { + test('should convert basic Zod validation error', () => { + const schema = z.object({ + name: z.string(), + age: z.number(), + }); + + const result = schema.safeParse({ name: 'John', age: 'invalid' }); + expect(result.success).toBe(false); + + if (!result.success) { + const issues = zodToIssues(result.error); + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + code: 'schema_validation', + message: 'Expected number, received string', + path: ['age'], + severity: 'error', + scope: ErrorScope.AGENT, + type: ErrorType.USER, + }); + } + }); + + test('should handle multiple validation errors', () => { + const schema = z.object({ + email: z.string().email(), + age: z.number().positive(), + }); + + const result = schema.safeParse({ email: 'invalid', age: -5 }); + expect(result.success).toBe(false); + + if (!result.success) { + const issues = zodToIssues(result.error); + expect(issues).toHaveLength(2); + expect(issues[0]?.path).toEqual(['email']); + expect(issues[1]?.path).toEqual(['age']); + } + }); + + test('should respect severity parameter', () => { + const schema = z.string(); + const result = schema.safeParse(123); + + if (!result.success) { + const warningIssues = zodToIssues(result.error, 'warning'); + expect(warningIssues[0]?.severity).toBe('warning'); + + const errorIssues = zodToIssues(result.error, 'error'); + expect(errorIssues[0]?.severity).toBe('error'); + } + }); + }); + + describe('union error handling', () => { + test('should collect errors from 2-member union', () => { + const schema = z.union([ + z.object({ type: z.literal('a'), value: z.string() }), + z.object({ type: z.literal('b'), count: z.number() }), + ]); + + const result = schema.safeParse({ type: 'a', value: 123 }); + expect(result.success).toBe(false); + + if (!result.success) { + const issues = zodToIssues(result.error); + // Should have issues from both union branches + expect(issues.length).toBeGreaterThan(0); + // At least one issue should mention the validation failure + expect( + issues.some((i) => i.path?.includes('value') || i.path?.includes('type')) + ).toBe(true); + } + }); + + test('should collect errors from 4-member union', () => { + // Simulates ApprovalResponseSchema structure + const schema = z.union([ + z.object({ type: z.literal('tool'), toolId: z.string() }), + z.object({ type: z.literal('command'), commandId: z.string() }), + z.object({ type: z.literal('elicit'), question: z.string() }), + z.object({ type: z.literal('custom'), data: z.object({}) }), + ]); + + const result = schema.safeParse({ type: 'tool', toolId: 123 }); + expect(result.success).toBe(false); + + if (!result.success) { + const issues = zodToIssues(result.error); + // Should collect errors from all union branches, not just the first two + expect(issues.length).toBeGreaterThan(0); + } + }); + + test('should handle deeply nested union errors', () => { + const innerSchema = z.union([z.string(), z.number()]); + const outerSchema = z.object({ + field: innerSchema, + }); + + const result = outerSchema.safeParse({ field: true }); + expect(result.success).toBe(false); + + if (!result.success) { + const issues = zodToIssues(result.error); + expect(issues.length).toBeGreaterThan(0); + expect(issues.some((i) => i.path?.includes('field'))).toBe(true); + } + }); + + test('should handle union with all failing branches (no match)', () => { + const schema = z.union([ + z.object({ type: z.literal('a'), data: z.string() }), + z.object({ type: z.literal('b'), data: z.number() }), + z.object({ type: z.literal('c'), data: z.boolean() }), + ]); + + // Input doesn't match ANY branch - all 3 should fail + const result = schema.safeParse({ type: 'x', data: 'invalid' }); + expect(result.success).toBe(false); + + if (!result.success) { + const issues = zodToIssues(result.error); + // Should collect errors from all 3 failed branches + expect(issues.length).toBeGreaterThan(0); + // Verify we got errors from multiple branches (not just the first) + const uniquePaths = new Set(issues.map((i) => JSON.stringify(i.path))); + expect(uniquePaths.size).toBeGreaterThan(0); + } + }); + + test('should handle very deeply nested unions (3+ levels)', () => { + // Union inside union inside union + const innerUnion = z.union([z.string(), z.number()]); + const middleUnion = z.union([innerUnion, z.boolean()]); + const outerUnion = z.union([ + middleUnion, + z.object({ foo: z.string() }), + z.array(z.number()), + ]); + + // This fails at multiple nesting levels + const result = outerUnion.safeParse({ foo: 123 }); + expect(result.success).toBe(false); + + if (!result.success) { + const issues = zodToIssues(result.error); + // Should collect errors from deeply nested union branches + expect(issues.length).toBeGreaterThan(0); + // Should have errors mentioning the nested field + expect(issues.some((i) => i.path?.includes('foo'))).toBe(true); + } + }); + + test('should handle fallback when no union errors are collected', () => { + // Create a manual ZodError with invalid_union but empty unionErrors + const error = new ZodError([ + { + code: 'invalid_union', + unionErrors: [] as ZodError[], + path: ['field'], + message: 'Invalid union type', + } as any, + ]); + + const issues = zodToIssues(error); + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + code: 'schema_validation', + message: 'Invalid union type', + path: ['field'], + severity: 'error', + }); + }); + }); + + describe('discriminated union error handling', () => { + test('should handle discriminated union errors', () => { + const schema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('success'), data: z.string() }), + z.object({ type: z.literal('error'), code: z.number() }), + ]); + + const result = schema.safeParse({ type: 'success', data: 123 }); + expect(result.success).toBe(false); + + if (!result.success) { + const issues = zodToIssues(result.error); + expect(issues.length).toBeGreaterThan(0); + expect(issues.some((i) => i.path?.includes('data'))).toBe(true); + } + }); + }); +}); + +describe('Result helper functions', () => { + describe('ok', () => { + test('should create successful result without issues', () => { + const result = ok({ value: 42 }); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data).toEqual({ value: 42 }); + } + expect(result.issues).toEqual([]); + }); + + test('should create successful result with warnings', () => { + const result = ok({ value: 42 }, [makeIssue('test_warning', 'warning')]); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.data).toEqual({ value: 42 }); + } + expect(result.issues).toHaveLength(1); + expect(result.issues[0]?.severity).toBe('warning'); + }); + }); + + describe('fail', () => { + test('should create failed result', () => { + const result = fail([makeIssue('test_error', 'error')]); + expect(result.ok).toBe(false); + expect(result.issues).toHaveLength(1); + expect(result.issues[0]?.severity).toBe('error'); + }); + }); + + describe('hasErrors', () => { + test('should return true when issues contain errors', () => { + expect(hasErrors([makeIssue('test_error', 'error')])).toBe(true); + }); + + test('should return false when issues only contain warnings', () => { + expect(hasErrors([makeIssue('test_warning', 'warning')])).toBe(false); + }); + + test('should return false for empty array', () => { + expect(hasErrors([])).toBe(false); + }); + }); + + describe('splitIssues', () => { + test('should split errors and warnings', () => { + const issues = [makeIssue('test_error', 'error'), makeIssue('test_warning', 'warning')]; + const { errors, warnings } = splitIssues(issues); + + expect(errors).toHaveLength(1); + expect(warnings).toHaveLength(1); + expect(errors[0]?.severity).toBe('error'); + expect(warnings[0]?.severity).toBe('warning'); + }); + + test('should handle all errors', () => { + const issues = [ + makeIssue('err1', 'error', 'Error 1'), + makeIssue('err2', 'error', 'Error 2'), + ]; + const { errors, warnings } = splitIssues(issues); + + expect(errors).toHaveLength(2); + expect(warnings).toHaveLength(0); + }); + + test('should handle all warnings', () => { + const issues = [ + makeIssue('warn1', 'warning', 'Warning 1'), + makeIssue('warn2', 'warning', 'Warning 2'), + ]; + const { errors, warnings } = splitIssues(issues); + + expect(errors).toHaveLength(0); + expect(warnings).toHaveLength(2); + }); + }); +}); diff --git a/dexto/packages/core/src/utils/result.ts b/dexto/packages/core/src/utils/result.ts new file mode 100644 index 00000000..66ee2608 --- /dev/null +++ b/dexto/packages/core/src/utils/result.ts @@ -0,0 +1,280 @@ +// schemas/helpers.ts +import { z, type ZodError } from 'zod'; +import type { DextoErrorCode, Issue } from '@core/errors/types.js'; +import { ErrorScope, ErrorType } from '@core/errors/types.js'; + +/** Trim and require non-empty after trim */ +export const NonEmptyTrimmed = z + .string() + .transform((s) => s.trim()) + .refine((s) => s.length > 0, { message: 'Required' }); + +/** Simple URL check (so we don’t need preprocess JUST to trim before .url()) */ +function isValidUrl(s: string): boolean { + try { + // Allow only http/https (adjust if you want more) + const u = new URL(s); + return u.protocol === 'http:' || u.protocol === 'https:'; + } catch { + return false; + } +} + +export const OptionalURL = z + .string() + .transform((s) => s.trim()) + .refine((s) => s === '' || isValidUrl(s), { message: 'Invalid URL' }) + .transform((s) => (s === '' ? undefined : s)) + .optional(); + +// Expand $VAR and ${VAR} using the provided env, then trim. +export const EnvExpandedString = (env?: Record) => + z.string().transform((input) => { + if (typeof input !== 'string') return ''; + // Use current process.env if no env provided (reads fresh each time) + const envToUse = env ?? process.env; + const out = input.replace( + /\$([A-Z_][A-Z0-9_]*)|\${([A-Z_][A-Z0-9_]*)}/gi, + (_, a, b) => envToUse[a || b] ?? '' + ); + return out.trim(); + }); + +// Zod type for non-empty environment expanded string +export const NonEmptyEnvExpandedString = (env?: Record) => + EnvExpandedString(env).refine((s) => s.length > 0, { + message: 'Value is required', + }); + +// Zod type for URL that could be pulled from env variables +export const RequiredEnvURL = (env?: Record) => + EnvExpandedString(env).refine( + (s) => { + try { + const u = new URL(s); + return u.protocol === 'http:' || u.protocol === 'https:'; + } catch { + return false; + } + }, + { message: 'Invalid URL' } + ); + +/** + * A discriminated union result type that can be either successful or failed + * Provides type safety by ensuring data is only available on success + * @param T - The type of the data on success + * @param C - The type of the context for issues + */ +export type Result = + | { ok: true; data: T; issues: Issue[] } + | { ok: false; issues: Issue[] }; + +/** + * Create a successful result with validated data and optional warnings. + * + * **Usage Guidelines:** + * - Use for operations that completed successfully, even with warnings + * - Include warnings for non-blocking issues (API key too short, fallback model used, etc.) + * - DextoAgent methods should prefer this over throwing exceptions + * - API layer maps this to 2xx status codes + * + * @param data - The successfully validated/processed data + * @param issues - Optional warnings or informational issues (defaults to empty array) + * @returns A successful Result with ok: true + * + * @example + * ```typescript + * // Success with no warnings + * return ok(validatedConfig); + * + * // Success with warnings + * return ok(validatedConfig, [ + * { code: 'llm_short_api_key', message: 'API key seems short', severity: 'warning', context: {} } + * ]); + * ``` + */ +export const ok = (data: T, issues: Issue[] = []): Result => ({ + ok: true, + data, + issues, // warnings live alongside errors here +}); + +/** + * Create a failed result with blocking errors that prevent operation completion. + * + * **Usage Guidelines:** + * - Use for validation failures, business rule violations, or any error that should stop execution + * - Do NOT mix with exceptions - choose Result pattern OR throwing, not both + * - API layer maps this to 4xx status codes (user/validation errors) + * - Issues should have severity: 'error' for blocking problems + * + * @param issues - Array of error issues that caused the failure (must not be empty) + * @returns A failed Result with ok: false and no data + * + * @example + * ```typescript + * // Validation failure + * return fail([ + * { + * code: LLMErrorCode.SWITCH_INPUT_MISSING, + * message: 'At least model or provider must be specified', + * severity: 'error', + * context: {} + * } + * ]); + * + * // Multiple validation errors + * return fail([ + * { code: 'missing_api_key', message: 'API key required', severity: 'error', context: {} }, + * { code: 'invalid_model', message: 'Model not supported', severity: 'error', context: {} } + * ]); + * ``` + */ +export const fail = (issues: Issue[]): Result => ({ + ok: false, + issues, +}); + +/** + * Check if a list of issues contains any blocking errors (non-warning severity). + * + * **Usage Guidelines:** + * - Use to determine if a Result should be ok: false + * - Warnings don't count as errors - operations can succeed with warnings + * - Useful in validation functions to decide success vs failure + * + * @param issues - Array of issues to check + * @returns true if any issue has severity other than 'warning', false otherwise + * + * @example + * ```typescript + * const issues = [ + * { severity: 'warning', message: 'API key seems short' }, + * { severity: 'error', message: 'Model not found' } + * ]; + * + * if (hasErrors(issues)) { + * return fail(issues); // Contains errors, operation fails + * } else { + * return ok(data, issues); // Only warnings, operation succeeds + * } + * ``` + */ +export function hasErrors(issues: Issue[]) { + return issues.some((i) => i.severity !== 'warning'); +} + +/** + * Separate issues into errors (blocking) and warnings (non-blocking) for different handling. + * + * **Usage Guidelines:** + * - Use when you need to handle errors and warnings differently + * - Errors should block operation, warnings should be logged/reported but allow success + * - Useful in API responses to show both what failed and what succeeded with caveats + * + * @param issues - Array of mixed issues to categorize + * @returns Object with separate arrays for errors and warnings + * + * @example + * ```typescript + * const { errors, warnings } = splitIssues(allIssues); + * + * if (errors.length > 0) { + * logger.error('Validation failed:', errors); + * return fail(errors); + * } + * + * if (warnings.length > 0) { + * logger.warn('Validation succeeded with warnings:', warnings); + * } + * + * return ok(data, warnings); + * ``` + */ +export function splitIssues(issues: Issue[]) { + return { + errors: issues.filter((i) => i.severity !== 'warning'), + warnings: issues.filter((i) => i.severity === 'warning'), + }; +} + +/** + * Convert Zod validation errors to standardized Issue format for Result pattern. + * + * **Usage Guidelines:** + * - Use in schema validation functions to convert Zod errors to our Issue format + * - Allows custom error codes via Zod's params.code field in custom refinements + * - Falls back to SCHEMA_VALIDATION code for standard Zod validation errors + * - Typically used with severity: 'error' for blocking validation failures + * + * @param err - ZodError from failed schema validation + * @param severity - Issue severity level (defaults to 'error') + * @returns Array of Issues in our standardized format + * + * @example + * ```typescript + * // In a validation function + * const result = MySchema.safeParse(data); + * if (!result.success) { + * const issues = zodToIssues(result.error); + * return fail(issues); + * } + * + * // Custom error codes in Zod schema + * const schema = z.string().refine(val => val.length > 0, { + * message: 'Field is required', + * params: { code: LLMErrorCode.API_KEY_MISSING } + * }); + * ``` + */ +export function zodToIssues( + err: ZodError, + severity: 'error' | 'warning' = 'error' +): Issue[] { + const issues: Issue[] = []; + + for (const e of err.errors) { + // Handle invalid_union errors by extracting the actual validation errors from unionErrors + if (e.code === 'invalid_union' && (e as any).unionErrors) { + const unionErrors = (e as any).unionErrors as ZodError[]; + // Iterate through ALL union errors to capture validation issues from every union branch + let hasCollectedErrors = false; + for (const unionError of unionErrors) { + if (unionError && unionError.errors && unionError.errors.length > 0) { + // Recursively process each union branch's errors + issues.push(...zodToIssues(unionError, severity)); + hasCollectedErrors = true; + } + } + + // Fallback: if no union errors were collected, report the invalid_union error itself + if (!hasCollectedErrors) { + const params = (e as any).params || {}; + issues.push({ + code: (params.code ?? 'schema_validation') as DextoErrorCode, + message: e.message, + scope: params.scope ?? ErrorScope.AGENT, + type: params.type ?? ErrorType.USER, + path: e.path, + severity, + context: params as C, + }); + } + } else { + // Standard error processing + const params = (e as any).params || {}; + issues.push({ + code: (params.code ?? 'schema_validation') as DextoErrorCode, + message: e.message, + scope: params.scope ?? ErrorScope.AGENT, + type: params.type ?? ErrorType.USER, + path: e.path, + severity, + context: params as C, + }); + } + } + + return issues; +} diff --git a/dexto/packages/core/src/utils/safe-stringify.ts b/dexto/packages/core/src/utils/safe-stringify.ts new file mode 100644 index 00000000..97fdd75e --- /dev/null +++ b/dexto/packages/core/src/utils/safe-stringify.ts @@ -0,0 +1,45 @@ +import { redactSensitiveData } from './redactor.js'; + +/** + * Safe stringify that handles circular references and BigInt. + * Also redacts sensitive data to prevent PII leaks. + * + * @param value - Value to stringify + * @param maxLen - Optional maximum length. If provided, truncates with '…(truncated)' suffix. + */ +export function safeStringify(value: unknown, maxLen?: number): string { + try { + // Handle top-level BigInt without triggering JSON.stringify errors + if (typeof value === 'bigint') { + return value.toString(); + } + // First redact sensitive data to prevent PII leaks + const redacted = redactSensitiveData(value); + const str = JSON.stringify(redacted, (_, v) => { + if (v instanceof Error) { + return { name: v.name, message: v.message, stack: v.stack }; + } + if (typeof v === 'bigint') return v.toString(); + return v; + }); + if (typeof str === 'string') { + // Only truncate if maxLen is explicitly provided + if (maxLen !== undefined && maxLen > 0 && str.length > maxLen) { + const indicator = '…(truncated)'; + if (maxLen <= indicator.length) { + return str.slice(0, maxLen); + } + const sliceLen = maxLen - indicator.length; + return `${str.slice(0, sliceLen)}${indicator}`; + } + return str; + } + return String(value); + } catch { + try { + return String(value); + } catch { + return '[Unserializable value]'; + } + } +} diff --git a/dexto/packages/core/src/utils/schema-metadata.ts b/dexto/packages/core/src/utils/schema-metadata.ts new file mode 100644 index 00000000..2881d8b9 --- /dev/null +++ b/dexto/packages/core/src/utils/schema-metadata.ts @@ -0,0 +1,405 @@ +/** + * Schema metadata extraction utilities for Zod schemas + * + * This module provides utilities to extract metadata from Zod schemas at runtime. + * + * IMPORTANT: This uses Zod's private `._def` API which is not officially supported + * and may break in future Zod versions. We use this approach because: + * 1. No public Zod API exists for runtime schema introspection + * 2. Benefits of schema-driven UI metadata outweigh version risk + * 3. Changes would be caught by TypeScript/tests during upgrades + * + * TODO: Update web UI to use these helpers to reduce total code volume and improve maintainability. Also fix these helpers if needed. + * See packages/webui/components/AgentEditor/CustomizePanel.tsx for the UI side TODO tracking this same goal. + * + * If Zod provides official introspection APIs in the future, migrate to those. + */ + +import { z } from 'zod'; + +/** + * Metadata extracted from a Zod schema + */ +export interface SchemaMetadata { + /** Default values for each field */ + defaults: Record; + /** Required fields (not optional, not with defaults) */ + requiredFields: string[]; + /** Field type information */ + fieldTypes: Record; + /** Field descriptions from .describe() calls */ + descriptions: Record; + /** Enum values for enum fields (e.g., provider: ['openai', 'anthropic']) */ + enumValues: Record; +} + +/** + * Extract metadata from a discriminated union schema + * Handles schemas like McpServerConfigSchema (stdio | sse | http) + */ +export interface DiscriminatedUnionMetadata { + /** The discriminator field name (e.g., "type") */ + discriminator: string; + /** Possible discriminator values (e.g., ["stdio", "sse", "http"]) */ + options: string[]; + /** Metadata for each option */ + schemas: Record; +} + +/** + * Extract default value from a Zod schema + * Returns undefined if no default is set + */ +function extractDefault(def: any): unknown { + // Check for .default() + if (def.defaultValue !== undefined) { + return typeof def.defaultValue === 'function' ? def.defaultValue() : def.defaultValue; + } + + // Check for branded types (wraps the actual schema) + if (def.typeName === 'ZodBranded' && def.type) { + return extractDefault(def.type._def); + } + + // Check for optional types (unwrap to inner type) + if (def.typeName === 'ZodOptional' && def.innerType) { + return extractDefault(def.innerType._def); + } + + // Check for nullable types + if (def.typeName === 'ZodNullable' && def.innerType) { + return extractDefault(def.innerType._def); + } + + return undefined; +} + +/** + * Extract enum values from a Zod schema + * Returns undefined if not an enum + */ +function extractEnumValues(def: any): string[] | undefined { + // Handle branded types + if (def.typeName === 'ZodBranded' && def.type) { + return extractEnumValues(def.type._def); + } + + // Handle optional types + if (def.typeName === 'ZodOptional' && def.innerType) { + return extractEnumValues(def.innerType._def); + } + + // Handle nullable types + if (def.typeName === 'ZodNullable' && def.innerType) { + return extractEnumValues(def.innerType._def); + } + + // Handle effects (transforms, refinements, etc.) + if (def.typeName === 'ZodEffects' && def.schema) { + return extractEnumValues(def.schema._def); + } + + // Extract from ZodEnum + if (def.typeName === 'ZodEnum') { + return def.values as string[]; + } + + // Extract from ZodLiteral (single value enum) + if (def.typeName === 'ZodLiteral') { + return [String(def.value)]; + } + + return undefined; +} + +/** + * Extract field type name from Zod schema + */ +function extractTypeName(def: any): string { + // Handle branded types + if (def.typeName === 'ZodBranded' && def.type) { + return extractTypeName(def.type._def); + } + + // Handle optional types + if (def.typeName === 'ZodOptional' && def.innerType) { + return extractTypeName(def.innerType._def) + '?'; + } + + // Handle nullable types + if (def.typeName === 'ZodNullable' && def.innerType) { + return extractTypeName(def.innerType._def) + '?'; + } + + // Handle literal types + if (def.typeName === 'ZodLiteral') { + return `literal(${JSON.stringify(def.value)})`; + } + + // Handle enum types + if (def.typeName === 'ZodEnum') { + return `enum(${def.values.join('|')})`; + } + + // Handle array types + if (def.typeName === 'ZodArray') { + return `array<${extractTypeName(def.type._def)}>`; + } + + // Handle record types + if (def.typeName === 'ZodRecord') { + return `record<${extractTypeName(def.valueType._def)}>`; + } + + // Handle effects (transforms, refinements, etc.) + if (def.typeName === 'ZodEffects' && def.schema) { + return extractTypeName(def.schema._def); + } + + // Map Zod type names to simplified names + const typeMap: Record = { + ZodString: 'string', + ZodNumber: 'number', + ZodBoolean: 'boolean', + ZodObject: 'object', + ZodArray: 'array', + ZodRecord: 'record', + ZodUnion: 'union', + ZodDiscriminatedUnion: 'discriminatedUnion', + }; + + return typeMap[def.typeName] || def.typeName?.replace('Zod', '').toLowerCase() || 'unknown'; +} + +/** + * Check if a field is required (not optional, no default) + */ +function isFieldRequired(def: any): boolean { + // Has a default? Not required for user input + if (def.defaultValue !== undefined) { + return false; + } + + // Is optional? Not required + if (def.typeName === 'ZodOptional') { + return false; + } + + // Is nullable? Not required + if (def.typeName === 'ZodNullable') { + return false; + } + + // Handle branded types + if (def.typeName === 'ZodBranded' && def.type) { + return isFieldRequired(def.type._def); + } + + return true; +} + +/** + * Extract metadata from a Zod object schema + * + * @param schema - Zod schema to extract metadata from + * @returns SchemaMetadata object with defaults, required fields, types, and descriptions + */ +export function extractSchemaMetadata(schema: z.ZodTypeAny): SchemaMetadata { + const metadata: SchemaMetadata = { + defaults: {}, + requiredFields: [], + fieldTypes: {}, + descriptions: {}, + enumValues: {}, + }; + + let def = (schema as any)._def; + + // Unwrap branded types + if (def.typeName === 'ZodBranded' && def.type) { + def = def.type._def; + } + + // Handle object schemas + if (def.typeName !== 'ZodObject') { + throw new Error(`Expected ZodObject, got ${def.typeName}`); + } + + const shape = def.shape(); + + for (const [fieldName, fieldSchema] of Object.entries(shape)) { + const fieldDef = (fieldSchema as any)._def; + + // Extract default value + const defaultValue = extractDefault(fieldDef); + if (defaultValue !== undefined) { + metadata.defaults[fieldName] = defaultValue; + } + + // Check if required + if (isFieldRequired(fieldDef)) { + metadata.requiredFields.push(fieldName); + } + + // Extract type + metadata.fieldTypes[fieldName] = extractTypeName(fieldDef); + + // Extract description + if (fieldDef.description) { + metadata.descriptions[fieldName] = fieldDef.description; + } + + // Extract enum values + const enumVals = extractEnumValues(fieldDef); + if (enumVals) { + metadata.enumValues[fieldName] = enumVals; + } + } + + return metadata; +} + +/** + * Extract metadata from a discriminated union schema + * + * @param schema - Zod discriminated union schema + * @returns DiscriminatedUnionMetadata with info about each variant + */ +export function extractDiscriminatedUnionMetadata( + schema: z.ZodTypeAny +): DiscriminatedUnionMetadata { + let def = (schema as any)._def; + + // Unwrap branded types + if (def.typeName === 'ZodBranded' && def.type) { + def = def.type._def; + } + + // Handle effects (refinements, transforms, etc.) + if (def.typeName === 'ZodEffects' && def.schema) { + def = def.schema._def; + } + + if (def.typeName !== 'ZodDiscriminatedUnion') { + throw new Error(`Expected ZodDiscriminatedUnion, got ${def.typeName}`); + } + + const discriminator = def.discriminator; + const optionsMap = def.optionsMap; + + const metadata: DiscriminatedUnionMetadata = { + discriminator, + options: Array.from(optionsMap.keys()) as string[], + schemas: {}, + }; + + // Extract metadata for each option + for (const [optionValue, optionSchema] of optionsMap.entries()) { + metadata.schemas[optionValue as string] = extractSchemaMetadata(optionSchema); + } + + return metadata; +} + +/** + * Extract common fields from a discriminated union (fields present in all variants) + * Useful for extracting shared defaults like 'timeout' or 'connectionMode' + */ +export function extractCommonFields(metadata: DiscriminatedUnionMetadata): SchemaMetadata { + const schemas = Object.values(metadata.schemas); + if (schemas.length === 0) { + return { + defaults: {}, + requiredFields: [], + fieldTypes: {}, + descriptions: {}, + enumValues: {}, + }; + } + + const first = schemas[0]!; // Safe: we checked length > 0 + const rest = schemas.slice(1); + const common: SchemaMetadata = { + defaults: { ...first.defaults }, + requiredFields: [...first.requiredFields], + fieldTypes: { ...first.fieldTypes }, + descriptions: { ...first.descriptions }, + enumValues: { ...first.enumValues }, + }; + + // Only keep fields that exist in ALL schemas + for (const schema of rest) { + // Filter defaults + for (const key of Object.keys(common.defaults)) { + if (!(key in schema.defaults) || schema.defaults[key] !== common.defaults[key]) { + delete common.defaults[key]; + } + } + + // Filter required fields + common.requiredFields = common.requiredFields.filter((field) => + schema.requiredFields.includes(field) + ); + + // Filter field types (keep only if same in all schemas) + for (const key of Object.keys(common.fieldTypes)) { + if (!(key in schema.fieldTypes) || schema.fieldTypes[key] !== common.fieldTypes[key]) { + delete common.fieldTypes[key]; + } + } + + // Filter descriptions (keep only if same in all schemas) + for (const key of Object.keys(common.descriptions)) { + if ( + !(key in schema.descriptions) || + schema.descriptions[key] !== common.descriptions[key] + ) { + delete common.descriptions[key]; + } + } + + // Filter enum values (keep only if same in all schemas) + for (const key of Object.keys(common.enumValues)) { + if ( + !(key in schema.enumValues) || + JSON.stringify(schema.enumValues[key]) !== JSON.stringify(common.enumValues[key]) + ) { + delete common.enumValues[key]; + } + } + } + + return common; +} + +/** + * Get default value for a specific field in a discriminated union variant + * + * @param metadata - Discriminated union metadata + * @param discriminatorValue - The discriminator value (e.g., 'stdio', 'http') + * @param fieldName - The field to get default for + * @returns The default value or undefined + */ +export function getFieldDefault( + metadata: DiscriminatedUnionMetadata, + discriminatorValue: string, + fieldName: string +): unknown { + return metadata.schemas[discriminatorValue]?.defaults[fieldName]; +} + +/** + * Check if a field is required in a specific discriminated union variant + * + * @param metadata - Discriminated union metadata + * @param discriminatorValue - The discriminator value (e.g., 'stdio', 'http') + * @param fieldName - The field to check + * @returns true if required, false otherwise + */ +export function isFieldRequiredInVariant( + metadata: DiscriminatedUnionMetadata, + discriminatorValue: string, + fieldName: string +): boolean { + return metadata.schemas[discriminatorValue]?.requiredFields.includes(fieldName) ?? false; +} diff --git a/dexto/packages/core/src/utils/schema.ts b/dexto/packages/core/src/utils/schema.ts new file mode 100644 index 00000000..ef9c8e7d --- /dev/null +++ b/dexto/packages/core/src/utils/schema.ts @@ -0,0 +1,27 @@ +import { zodToJsonSchema } from 'zod-to-json-schema'; +import type { IDextoLogger } from '../logger/v2/types.js'; + +/** + * Convert Zod schema to JSON Schema format for tool parameters + * + * TODO: Replace zod-to-json-schema with Zod v4 native JSON schema support + * The zod-to-json-schema package is deprecated and adds ~19MB due to a packaging bug + * (includes test files with full Zod copies in dist-test-v3 and dist-test-v4 folders). + * Zod v4 has native toJsonSchema() support - migrate when upgrading to Zod v4. + * See: https://github.com/StefanTerdell/zod-to-json-schema + */ +export function convertZodSchemaToJsonSchema(zodSchema: any, logger: IDextoLogger): any { + try { + // Use proper library for Zod to JSON Schema conversion + return zodToJsonSchema(zodSchema); + } catch (error) { + logger.warn( + `Failed to convert Zod schema to JSON Schema: ${error instanceof Error ? error.message : String(error)}` + ); + // Return basic object schema as fallback + return { + type: 'object', + properties: {}, + }; + } +} diff --git a/dexto/packages/core/src/utils/service-initializer.ts b/dexto/packages/core/src/utils/service-initializer.ts new file mode 100644 index 00000000..f70dd13f --- /dev/null +++ b/dexto/packages/core/src/utils/service-initializer.ts @@ -0,0 +1,266 @@ +/* + * Service Initializer: Centralized Wiring for Dexto Core Services + * + * This module is responsible for initializing and wiring together all core agent services (LLM, client manager, message manager, event bus, etc.) + * for the Dexto application. It provides a single entry point for constructing the service graph, ensuring consistent dependency injection + * and configuration across CLI, web, and test environments. + * + * **Configuration Pattern:** + * - The primary source of configuration is the config file (e.g., `agent.yml`), which allows users to declaratively specify both high-level + * and low-level service options (such as compression strategies for ContextManager, LLM provider/model, etc.). + * - For most use cases, the config file is sufficient and preferred, as it enables environment-specific, auditable, and user-friendly customization. + * + * **Service Architecture:** + * - All services are initialized based on the provided configuration. + * - For testing scenarios, mock the service dependencies directly using test frameworks rather than relying on service injection patterns. + * + * **Best Practice:** + * - Use the config file for all user-facing and environment-specific configuration, including low-level service details. + * - For testing, use proper mocking frameworks rather than service injection to ensure clean, maintainable tests. + * + * This pattern ensures a clean, scalable, and maintainable architecture, balancing flexibility with simplicity. + */ + +import { MCPManager } from '../mcp/manager.js'; +import { ToolManager } from '../tools/tool-manager.js'; +import { SystemPromptManager } from '../systemPrompt/manager.js'; +import { AgentStateManager } from '../agent/state-manager.js'; +import { SessionManager } from '../session/index.js'; +import { SearchService } from '../search/index.js'; +import { dirname, resolve } from 'path'; +import { createStorageManager, StorageManager } from '../storage/index.js'; +import { createAllowedToolsProvider } from '../tools/confirmation/allowed-tools-provider/factory.js'; +import type { IDextoLogger } from '../logger/v2/types.js'; +import type { ValidatedAgentConfig } from '@core/agent/schemas.js'; +import { AgentEventBus } from '../events/index.js'; +import { ResourceManager } from '../resources/manager.js'; +import { ApprovalManager } from '../approval/manager.js'; +import { MemoryManager } from '../memory/index.js'; +import { PluginManager } from '../plugins/manager.js'; +import { registerBuiltInPlugins } from '../plugins/registrations/builtins.js'; + +/** + * Type for the core agent services returned by createAgentServices + */ +export type AgentServices = { + mcpManager: MCPManager; + toolManager: ToolManager; + systemPromptManager: SystemPromptManager; + agentEventBus: AgentEventBus; + stateManager: AgentStateManager; + sessionManager: SessionManager; + searchService: SearchService; + storageManager: StorageManager; + resourceManager: ResourceManager; + approvalManager: ApprovalManager; + memoryManager: MemoryManager; + pluginManager: PluginManager; +}; + +// High-level factory to load, validate, and wire up all agent services in one call +/** + * Initializes all agent services from a validated configuration. + * @param config The validated agent configuration object + * @param configPath Optional path to the config file (for relative path resolution) + * @param logger Logger instance for this agent (dependency injection) + * @param agentEventBus Pre-created event bus from DextoAgent constructor + * @returns All the initialized services required for a Dexto agent + */ +export async function createAgentServices( + config: ValidatedAgentConfig, + configPath: string | undefined, + logger: IDextoLogger, + agentEventBus: AgentEventBus +): Promise { + // 0. Initialize telemetry FIRST (before any decorated classes are instantiated) + // This must happen before creating any services that use @InstrumentClass decorator + if (config.telemetry?.enabled) { + const { Telemetry } = await import('../telemetry/telemetry.js'); + await Telemetry.init(config.telemetry); + logger.debug('Telemetry initialized'); + } + + // 1. Use the event bus provided by DextoAgent constructor + logger.debug('Using pre-created agent event bus'); + + // 2. Initialize storage manager (schema provides in-memory defaults, CLI enrichment adds filesystem paths) + logger.debug('Initializing storage manager'); + const storageManager = await createStorageManager(config.storage, logger); + + logger.debug('Storage manager initialized', { + cache: config.storage.cache.type, + database: config.storage.database.type, + }); + + // 3. Initialize approval system (generalized user approval) + // Created before MCP manager since MCP manager depends on it for elicitation support + logger.debug('Initializing approval manager'); + const approvalManager = new ApprovalManager( + { + toolConfirmation: { + mode: config.toolConfirmation.mode, + ...(config.toolConfirmation.timeout !== undefined && { + timeout: config.toolConfirmation.timeout, + }), + }, + elicitation: { + enabled: config.elicitation.enabled, + ...(config.elicitation.timeout !== undefined && { + timeout: config.elicitation.timeout, + }), + }, + }, + logger + ); + logger.debug('Approval system initialized'); + + // 4. Initialize MCP manager + const mcpManager = new MCPManager(logger); + await mcpManager.initializeFromConfig(config.mcpServers); + + // 4.1 - Wire approval manager into MCP manager for elicitation support + mcpManager.setApprovalManager(approvalManager); + logger.debug('Approval manager connected to MCP manager for elicitation support'); + + // 5. Initialize search service + const searchService = new SearchService(storageManager.getDatabase(), logger); + + // 6. Initialize memory manager + const memoryManager = new MemoryManager(storageManager.getDatabase(), logger); + logger.debug('Memory manager initialized'); + + // 6.5 Initialize plugin manager + const configDir = configPath ? dirname(resolve(configPath)) : process.cwd(); + const pluginManager = new PluginManager( + { + agentEventBus, + storageManager, + configDir, + }, + logger + ); + + // Register built-in plugins from registry + registerBuiltInPlugins({ pluginManager, config }); + logger.debug('Built-in plugins registered'); + + // Initialize plugin manager (loads custom and registry plugins, validates, calls initialize()) + await pluginManager.initialize(config.plugins.custom, config.plugins.registry); + logger.info('Plugin manager initialized'); + + // 7. Initialize resource manager (MCP + internal resources) + // Moved before tool manager so it can be passed to internal tools + const resourceManager = new ResourceManager( + mcpManager, + { + internalResourcesConfig: config.internalResources, + blobStore: storageManager.getBlobStore(), + }, + logger + ); + await resourceManager.initialize(); + + // 8. Initialize tool manager with internal tools options + // 8.1 - Create allowed tools provider based on configuration + const allowedToolsProvider = createAllowedToolsProvider( + { + type: config.toolConfirmation.allowedToolsStorage, + storageManager, + }, + logger + ); + + // 8.2 - Initialize tool manager with direct ApprovalManager integration + const toolManager = new ToolManager( + mcpManager, + approvalManager, + allowedToolsProvider, + config.toolConfirmation.mode, + agentEventBus, + config.toolConfirmation.toolPolicies, + { + internalToolsServices: { + searchService, + resourceManager, + }, + internalToolsConfig: config.internalTools, + customToolsConfig: config.customTools, + }, + logger + ); + // NOTE: toolManager.initialize() is called in DextoAgent.start() after agent reference is set + // This allows custom tools to access the agent for bidirectional communication + + const mcpServerCount = Object.keys(config.mcpServers).length; + if (mcpServerCount === 0) { + logger.info('Agent initialized without MCP servers - only built-in capabilities available'); + } else { + logger.debug(`MCPManager initialized with ${mcpServerCount} MCP server(s)`); + } + + if (config.internalTools.length === 0) { + logger.info('No internal tools enabled by configuration'); + } else { + logger.info(`Internal tools enabled: ${config.internalTools.join(', ')}`); + } + + // 9. Initialize prompt manager + logger.debug( + `[ServiceInitializer] Creating SystemPromptManager with configPath: ${configPath} → configDir: ${configDir}` + ); + const systemPromptManager = new SystemPromptManager( + config.systemPrompt, + configDir, + memoryManager, + config.memories, + logger + ); + + // 10. Initialize state manager for runtime state tracking + const stateManager = new AgentStateManager(config, agentEventBus, logger); + logger.debug('Agent state manager initialized'); + + // 11. Initialize session manager + const sessionManager = new SessionManager( + { + stateManager, + systemPromptManager, + toolManager, + agentEventBus, + storageManager, // Add storage manager to session services + resourceManager, // Add resource manager for blob storage + pluginManager, // Add plugin manager for plugin execution + mcpManager, // Add MCP manager for ChatSession + }, + { + maxSessions: config.sessions?.maxSessions, + sessionTTL: config.sessions?.sessionTTL, + }, + logger + ); + + // Initialize the session manager with persistent storage + await sessionManager.init(); + + logger.debug('Session manager initialized with storage support'); + + // 12.5 Wire up plugin support to ToolManager (after SessionManager is created) + toolManager.setPluginSupport(pluginManager, sessionManager, stateManager); + logger.debug('Plugin support connected to ToolManager'); + + // 13. Return the core services + return { + mcpManager, + toolManager, + systemPromptManager, + agentEventBus, + stateManager, + sessionManager, + searchService, + storageManager, + resourceManager, + approvalManager, + memoryManager, + pluginManager, + }; +} diff --git a/dexto/packages/core/src/utils/user-info.ts b/dexto/packages/core/src/utils/user-info.ts new file mode 100644 index 00000000..2c7affa6 --- /dev/null +++ b/dexto/packages/core/src/utils/user-info.ts @@ -0,0 +1,6 @@ +// Utility to get the current user ID +// TODO: Update this logic to support multi-tenancy (e.g., from session, DB, or web app state) + +export function getUserId(): string { + return 'default-user'; +} diff --git a/dexto/packages/core/src/utils/zod-schema-converter.ts b/dexto/packages/core/src/utils/zod-schema-converter.ts new file mode 100644 index 00000000..e0b97630 --- /dev/null +++ b/dexto/packages/core/src/utils/zod-schema-converter.ts @@ -0,0 +1,106 @@ +import { z } from 'zod'; + +/** + * Converts a JSON Schema object to a Zod raw shape. + * This is a simplified converter that handles common MCP tool schemas. + */ +export function jsonSchemaToZodShape(jsonSchema: any): z.ZodRawShape { + if (!jsonSchema || typeof jsonSchema !== 'object' || jsonSchema.type !== 'object') { + return {}; + } + + const shape: z.ZodRawShape = {}; + + if (jsonSchema.properties) { + for (const [key, property] of Object.entries(jsonSchema.properties)) { + const propSchema = property as any; + let zodType: z.ZodTypeAny; + switch (propSchema.type) { + case 'string': + zodType = z.string(); + break; + case 'number': + zodType = z.number(); + break; + case 'integer': + zodType = z.number().int(); + break; + case 'boolean': + zodType = z.boolean(); + break; + case 'array': + if (propSchema.items) { + const itemType = getZodTypeFromProperty(propSchema.items); + zodType = z.array(itemType); + } else { + zodType = z.array(z.any()); + } + break; + case 'object': + zodType = z.object(jsonSchemaToZodShape(propSchema)); + break; + default: + zodType = z.any(); + } + + // Add description if present using custom metadata + if (propSchema.description) { + // Try to add description as custom property (this might get picked up by the SDK) + (zodType as any)._def.description = propSchema.description; + zodType = zodType.describe(propSchema.description); + } + + // Make optional if not in required array + if (!jsonSchema.required || !jsonSchema.required.includes(key)) { + zodType = zodType.optional(); + } + + shape[key] = zodType; + } + } + + return shape; +} + +/** + * Helper function to get a Zod type from a property schema + */ +export function getZodTypeFromProperty(propSchema: any): z.ZodTypeAny { + let zodType: z.ZodTypeAny; + + switch (propSchema.type) { + case 'string': + zodType = z.string(); + break; + case 'number': + zodType = z.number(); + break; + case 'integer': + zodType = z.number().int(); + break; + case 'boolean': + zodType = z.boolean(); + break; + case 'object': + zodType = z.object(jsonSchemaToZodShape(propSchema)); + break; + case 'array': + if (propSchema.items) { + zodType = z.array(getZodTypeFromProperty(propSchema.items)); + } else { + zodType = z.array(z.any()); + } + break; + default: + zodType = z.any(); + } + + // Add description if present using custom metadata + if (propSchema.description) { + // Try to add description as custom property (this might get picked up by the SDK) + (zodType as any)._def.description = propSchema.description; + zodType = zodType.describe(propSchema.description); + } + + return zodType; +} diff --git a/dexto/packages/core/tsconfig.json b/dexto/packages/core/tsconfig.json new file mode 100644 index 00000000..b89e4385 --- /dev/null +++ b/dexto/packages/core/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": false, + "paths": { + "@core/*": ["src/*"] + } + }, + "include": ["src/**/*"], + "exclude": ["**/*.test.ts", "**/*.spec.ts", "**/*.integration.test.ts", "dist", "node_modules"] +} diff --git a/dexto/packages/core/tsconfig.typecheck.json b/dexto/packages/core/tsconfig.typecheck.json new file mode 100644 index 00000000..90a93a82 --- /dev/null +++ b/dexto/packages/core/tsconfig.typecheck.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["dist", "node_modules"] +} diff --git a/dexto/packages/core/tsup.config.ts b/dexto/packages/core/tsup.config.ts new file mode 100644 index 00000000..51637888 --- /dev/null +++ b/dexto/packages/core/tsup.config.ts @@ -0,0 +1,21 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig([ + { + entry: ['src/**/*.ts', '!src/**/*.test.ts', '!src/**/*.integration.test.ts'], + format: ['cjs', 'esm'], + outDir: 'dist', + dts: false, // Disable DTS generation in tsup to avoid worker memory issues + platform: 'node', + bundle: false, + clean: true, + tsconfig: './tsconfig.json', + esbuildOptions(options) { + // Suppress empty import meta warnings which tsup anyway fixes + options.logOverride = { + ...(options.logOverride ?? {}), + 'empty-import-meta': 'silent', + }; + }, + }, +]); diff --git a/dexto/packages/core/vitest.setup.ts b/dexto/packages/core/vitest.setup.ts new file mode 100644 index 00000000..18401b8d --- /dev/null +++ b/dexto/packages/core/vitest.setup.ts @@ -0,0 +1,14 @@ +import { config } from 'dotenv'; +import { resolve } from 'path'; + +// Load .env file for integration tests +// This ensures environment variables are available during test execution +// Note: override is set to false to preserve CI environment variables +const result = config({ + path: resolve(__dirname, '../../.env'), + override: false, +}); + +if (result.error && process.env.CI !== 'true') { + console.warn('Warning: Failed to load .env file for tests:', result.error.message); +} diff --git a/dexto/packages/image-bundler/CHANGELOG.md b/dexto/packages/image-bundler/CHANGELOG.md new file mode 100644 index 00000000..4bcebc7b --- /dev/null +++ b/dexto/packages/image-bundler/CHANGELOG.md @@ -0,0 +1,89 @@ +# @dexto/image-bundler + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +- Updated dependencies [042f4f0] + - @dexto/core@1.5.6 + +## 1.5.5 + +### Patch Changes + +- Updated dependencies [63fa083] +- Updated dependencies [6df3ca9] + - @dexto/core@1.5.5 + +## 1.5.4 + +### Patch Changes + +- Updated dependencies [0016cd3] +- Updated dependencies [499b890] +- Updated dependencies [aa2c9a0] + - @dexto/core@1.5.4 + +## 1.5.3 + +### Patch Changes + +- Updated dependencies [4f00295] +- Updated dependencies [69c944c] + - @dexto/core@1.5.3 + +## 1.5.2 + +### Patch Changes + +- Updated dependencies [8a85ea4] +- Updated dependencies [527f3f9] + - @dexto/core@1.5.2 + +## 1.5.1 + +### Patch Changes + +- Updated dependencies [bfcc7b1] +- Updated dependencies [4aabdb7] + - @dexto/core@1.5.1 + +## 1.5.0 + +### Minor Changes + +- e7722e5: Minor version bump for new release with bundler, custom tool pkgs, etc. + +### Patch Changes + +- 1e7e974: Added image bundler, @dexto/image-local and moved tool services outside core. Added registry providers to select core services. +- 5fa79fa: Renamed compression to compaction, added context-awareness to hono, updated cli tool display formatting and added integration test for image-local. +- ef40e60: Upgrades package versions and related changes to MCP SDK. CLI colors improved and token streaming added to status bar. + + Security: Resolve all Dependabot security vulnerabilities. Updated @modelcontextprotocol/sdk to 1.25.2, esbuild to 0.25.0, langchain to 0.3.37, and @langchain/core to 0.3.80. Added pnpm overrides for indirect vulnerabilities (preact@10.27.3, qs@6.14.1, jws@3.2.3, mdast-util-to-hast@13.2.1). Fixed type errors from MCP SDK breaking changes. + +- Updated dependencies [ee12727] +- Updated dependencies [1e7e974] +- Updated dependencies [4c05310] +- Updated dependencies [5fa79fa] +- Updated dependencies [ef40e60] +- Updated dependencies [e714418] +- Updated dependencies [e7722e5] +- Updated dependencies [7d5ab19] +- Updated dependencies [436a900] + - @dexto/core@1.5.0 diff --git a/dexto/packages/image-bundler/package.json b/dexto/packages/image-bundler/package.json new file mode 100644 index 00000000..c7aebf17 --- /dev/null +++ b/dexto/packages/image-bundler/package.json @@ -0,0 +1,33 @@ +{ + "name": "@dexto/image-bundler", + "version": "1.5.6", + "description": "Bundler for Dexto base images", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "dexto-bundle": "./dist/cli.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsup", + "dev": "tsup --watch", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@dexto/core": "workspace:*", + "commander": "^12.0.0", + "esbuild": "^0.25.0", + "picocolors": "^1.0.0" + }, + "devDependencies": { + "@types/node": "^20.11.5", + "tsup": "^8.0.1", + "typescript": "^5.3.3" + } +} diff --git a/dexto/packages/image-bundler/src/bundler.ts b/dexto/packages/image-bundler/src/bundler.ts new file mode 100644 index 00000000..6cb3e47e --- /dev/null +++ b/dexto/packages/image-bundler/src/bundler.ts @@ -0,0 +1,361 @@ +/** + * Main bundler logic + */ + +import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync } from 'node:fs'; +import { dirname, join, resolve, relative, extname } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { validateImageDefinition } from '@dexto/core'; +import type { ImageDefinition } from '@dexto/core'; +import type { BundleOptions, BundleResult } from './types.js'; +import { generateEntryPoint } from './generator.js'; +import ts from 'typescript'; + +/** + * Bundle a Dexto base image + */ +export async function bundle(options: BundleOptions): Promise { + const warnings: string[] = []; + + // 1. Load and validate image definition + console.log(`📦 Loading image definition from ${options.imagePath}`); + const definition = await loadImageDefinition(options.imagePath); + + console.log(`✅ Loaded image: ${definition.name} v${definition.version}`); + + // 2. Validate definition + console.log(`🔍 Validating image definition...`); + try { + validateImageDefinition(definition); + console.log(`✅ Image definition is valid`); + } catch (error) { + throw new Error(`Image validation failed: ${error}`); + } + + // 3. Get core version (from package.json) + const coreVersion = getCoreVersion(); + + // 3.5. Discover providers from convention-based folders + console.log(`🔍 Discovering providers from folders...`); + const imageDir = dirname(options.imagePath); + const discoveredProviders = discoverProviders(imageDir); + console.log(`✅ Discovered ${discoveredProviders.totalCount} provider(s)`); + + // 4. Generate code + console.log(`🔨 Generating entry point...`); + const generated = generateEntryPoint(definition, coreVersion, discoveredProviders); + + // 5. Ensure output directory exists + const outDir = resolve(options.outDir); + if (!existsSync(outDir)) { + mkdirSync(outDir, { recursive: true }); + } + + // 5.5. Compile provider category folders + console.log(`🔨 Compiling provider source files...`); + const categories = ['blob-store', 'tools', 'compaction', 'plugins']; + let compiledCount = 0; + + for (const category of categories) { + const categoryDir = join(imageDir, category); + if (existsSync(categoryDir)) { + compileSourceFiles(categoryDir, join(outDir, category)); + compiledCount++; + } + } + + if (compiledCount > 0) { + console.log( + `✅ Compiled ${compiledCount} provider categor${compiledCount === 1 ? 'y' : 'ies'}` + ); + } + + // 6. Write generated files + const entryFile = join(outDir, 'index.js'); + const typesFile = join(outDir, 'index.d.ts'); + + console.log(`📝 Writing ${entryFile}...`); + writeFileSync(entryFile, generated.js, 'utf-8'); + + console.log(`📝 Writing ${typesFile}...`); + writeFileSync(typesFile, generated.dts, 'utf-8'); + + // 7. Generate package.json exports + updatePackageJson(dirname(options.imagePath), outDir); + + console.log(`✨ Build complete!`); + console.log(` Entry: ${entryFile}`); + console.log(` Types: ${typesFile}`); + + const metadata = { + name: definition.name, + version: definition.version, + description: definition.description, + target: definition.target || 'custom', + constraints: definition.constraints || [], + builtAt: new Date().toISOString(), + coreVersion, + }; + + return { + entryFile, + typesFile, + metadata, + warnings, + }; +} + +/** + * Load image definition from file + */ +async function loadImageDefinition(imagePath: string): Promise { + const absolutePath = resolve(imagePath); + + if (!existsSync(absolutePath)) { + throw new Error(`Image file not found: ${absolutePath}`); + } + + try { + // Convert to file:// URL for ESM import + const fileUrl = pathToFileURL(absolutePath).href; + + // Dynamic import + const module = await import(fileUrl); + + // Get default export + const definition = module.default as ImageDefinition; + + if (!definition) { + throw new Error('Image file must have a default export'); + } + + return definition; + } catch (error) { + if (error instanceof Error) { + throw new Error(`Failed to load image definition: ${error.message}`); + } + throw error; + } +} + +/** + * Get @dexto/core version + */ +function getCoreVersion(): string { + try { + // Try to read from node_modules + const corePackageJson = join(process.cwd(), 'node_modules/@dexto/core/package.json'); + if (existsSync(corePackageJson)) { + const pkg = JSON.parse(readFileSync(corePackageJson, 'utf-8')); + return pkg.version; + } + + // Fallback to workspace version + return '1.0.0'; + } catch { + return '1.0.0'; + } +} + +/** + * Update or create package.json with proper exports + */ +function updatePackageJson(imageDir: string, outDir: string): void { + const packageJsonPath = join(imageDir, 'package.json'); + + if (!existsSync(packageJsonPath)) { + console.log(`⚠️ No package.json found, skipping exports update`); + return; + } + + try { + const pkg = JSON.parse(readFileSync(packageJsonPath, 'utf-8')); + + // Update exports + pkg.exports = { + '.': { + types: './dist/index.d.ts', + import: './dist/index.js', + }, + }; + + // Update main and types fields + pkg.main = './dist/index.js'; + pkg.types = './dist/index.d.ts'; + + writeFileSync(packageJsonPath, JSON.stringify(pkg, null, 2), 'utf-8'); + console.log(`✅ Updated package.json exports`); + } catch (error) { + console.warn(`⚠️ Failed to update package.json: ${error}`); + } +} + +/** + * Compile TypeScript source files to JavaScript + */ +function compileSourceFiles(srcDir: string, outDir: string): void { + // Find all .ts files + const tsFiles = findTypeScriptFiles(srcDir); + + if (tsFiles.length === 0) { + console.log(` No TypeScript files found in ${srcDir}`); + return; + } + + console.log(` Found ${tsFiles.length} TypeScript file(s) to compile`); + + // TypeScript compiler options + const compilerOptions: ts.CompilerOptions = { + target: ts.ScriptTarget.ES2022, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + outDir: outDir, + rootDir: srcDir, // Use srcDir as root + declaration: true, + esModuleInterop: true, + skipLibCheck: true, + strict: true, + resolveJsonModule: true, + }; + + // Create program + const program = ts.createProgram(tsFiles, compilerOptions); + + // Emit compiled files + const emitResult = program.emit(); + + // Check for errors + const allDiagnostics = ts.getPreEmitDiagnostics(program).concat(emitResult.diagnostics); + + if (allDiagnostics.length > 0) { + allDiagnostics.forEach((diagnostic) => { + if (diagnostic.file) { + const { line, character } = ts.getLineAndCharacterOfPosition( + diagnostic.file, + diagnostic.start! + ); + const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n'); + console.error( + ` ${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}` + ); + } else { + console.error( + ` ${ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n')}` + ); + } + }); + + if (emitResult.emitSkipped) { + throw new Error('TypeScript compilation failed'); + } + } +} + +/** + * Recursively find all TypeScript files in a directory + */ +function findTypeScriptFiles(dir: string): string[] { + const files: string[] = []; + + function walk(currentDir: string) { + const entries = readdirSync(currentDir); + + for (const entry of entries) { + const fullPath = join(currentDir, entry); + const stat = statSync(fullPath); + + if (stat.isDirectory()) { + walk(fullPath); + } else if (stat.isFile() && extname(entry) === '.ts') { + files.push(fullPath); + } + } + } + + walk(dir); + return files; +} + +/** + * Provider discovery result for a single category + */ +export interface DiscoveredProviders { + blobStore: string[]; + customTools: string[]; + compaction: string[]; + plugins: string[]; + totalCount: number; +} + +/** + * Discover providers from convention-based folder structure + * + * Convention (folder-based with index.ts): + * tools/ - CustomToolProvider folders + * weather/ - Provider folder + * index.ts - Provider implementation (auto-discovered) + * helpers.ts - Optional helper files + * types.ts - Optional type definitions + * blob-store/ - BlobStoreProvider folders + * compaction/ - CompactionProvider folders + * plugins/ - PluginProvider folders + * + * Naming Convention (Node.js standard): + * /index.ts - Auto-discovered and registered + * /other.ts - Ignored unless imported by index.ts + */ +function discoverProviders(imageDir: string): DiscoveredProviders { + const result: DiscoveredProviders = { + blobStore: [], + customTools: [], + compaction: [], + plugins: [], + totalCount: 0, + }; + + // Category mapping: folder name -> property name + const categories = { + 'blob-store': 'blobStore', + tools: 'customTools', + compaction: 'compaction', + plugins: 'plugins', + } as const; + + for (const [folderName, propName] of Object.entries(categories)) { + const categoryDir = join(imageDir, folderName); + + if (!existsSync(categoryDir)) { + continue; + } + + // Find all provider folders (those with index.ts) + const providerFolders = readdirSync(categoryDir) + .filter((entry) => { + const entryPath = join(categoryDir, entry); + const stat = statSync(entryPath); + + // Must be a directory + if (!stat.isDirectory()) { + return false; + } + + // Must contain index.ts + const indexPath = join(entryPath, 'index.ts'); + return existsSync(indexPath); + }) + .map((folder) => { + // Return relative path for imports + return `./${folderName}/${folder}/index.js`; + }); + + if (providerFolders.length > 0) { + result[propName as keyof Omit].push( + ...providerFolders + ); + result.totalCount += providerFolders.length; + console.log(` Found ${providerFolders.length} provider(s) in ${folderName}/`); + } + } + + return result; +} diff --git a/dexto/packages/image-bundler/src/cli.ts b/dexto/packages/image-bundler/src/cli.ts new file mode 100644 index 00000000..5d8b6a0c --- /dev/null +++ b/dexto/packages/image-bundler/src/cli.ts @@ -0,0 +1,93 @@ +#!/usr/bin/env node +/** + * CLI for bundling Dexto base images + */ + +// Suppress experimental warnings (e.g., Type Stripping) +process.removeAllListeners('warning'); +process.on('warning', (warning) => { + if (warning.name !== 'ExperimentalWarning') { + console.warn(warning); + } +}); + +import { Command } from 'commander'; +import { bundle } from './bundler.js'; +import { readFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import pc from 'picocolors'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Read version from package.json +const packageJson = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf-8')); + +const program = new Command(); + +program.name('dexto-bundle').description('Bundle Dexto base images').version(packageJson.version); + +program + .command('build') + .description('Build a base image from dexto.image.ts') + .option('-i, --image ', 'Path to dexto.image.ts file', 'dexto.image.ts') + .option('-o, --out ', 'Output directory', 'dist') + .option('--sourcemap', 'Generate source maps', false) + .option('--minify', 'Minify output', false) + .action(async (options) => { + try { + console.log(pc.cyan('🚀 Dexto Image Bundler\n')); + + const result = await bundle({ + imagePath: options.image, + outDir: options.out, + sourcemap: options.sourcemap, + minify: options.minify, + }); + + console.log(pc.green('\n✨ Build successful!\n')); + console.log(pc.bold('Image Details:')); + console.log(` Name: ${result.metadata.name}`); + console.log(` Version: ${result.metadata.version}`); + console.log(` Target: ${result.metadata.target}`); + console.log(` Built at: ${result.metadata.builtAt}`); + console.log(` Core: v${result.metadata.coreVersion}`); + + if (result.metadata.constraints.length > 0) { + console.log(` Constraints: ${result.metadata.constraints.join(', ')}`); + } + + if (result.warnings.length > 0) { + console.log(pc.yellow('\n⚠️ Warnings:')); + result.warnings.forEach((w) => console.log(` - ${w}`)); + } + + // Read package.json to get the actual package name + const packageJsonPath = join(process.cwd(), 'package.json'); + let packageName = result.metadata.name; + try { + if (readFileSync) { + const pkgJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8')); + packageName = pkgJson.name || result.metadata.name; + } + } catch { + // Use metadata name as fallback + } + + console.log(pc.green('\n✅ Image is ready to use!')); + console.log(' To use this image in an app:'); + console.log( + pc.dim( + ` 1. Install it: pnpm add ${packageName}@file:../${packageName.split('/').pop()}` + ) + ); + console.log(pc.dim(` 2. Import it: import { createAgent } from '${packageName}';`)); + console.log(pc.dim(`\n Or publish to npm and install normally.`)); + } catch (error) { + console.error(pc.red('\n❌ Build failed:'), error); + process.exit(1); + } + }); + +program.parse(); diff --git a/dexto/packages/image-bundler/src/generator.ts b/dexto/packages/image-bundler/src/generator.ts new file mode 100644 index 00000000..5bc01114 --- /dev/null +++ b/dexto/packages/image-bundler/src/generator.ts @@ -0,0 +1,336 @@ +/** + * Code generator for base images + * + * Transforms image definitions into importable packages with: + * - Side-effect provider registration + * - createAgent() factory + * - Utility exports + * - Metadata exports + */ + +import type { ImageDefinition } from '@dexto/core'; +import type { GeneratedCode } from './types.js'; +import type { DiscoveredProviders } from './bundler.js'; + +/** + * Generate JavaScript entry point for an image + */ +export function generateEntryPoint( + definition: ImageDefinition, + coreVersion: string, + discoveredProviders?: DiscoveredProviders +): GeneratedCode { + // Generate imports section + const imports = generateImports(definition, discoveredProviders); + + // Generate provider registration section + const registrations = generateProviderRegistrations(definition, discoveredProviders); + + // Generate factory function + const factory = generateFactory(); + + // Generate utility exports + const utilityExports = generateUtilityExports(definition); + + // Generate metadata export + const metadata = generateMetadata(definition, coreVersion); + + // Combine all sections + const js = `// AUTO-GENERATED by @dexto/bundler +// Do not edit this file directly. Edit dexto.image.ts instead. + +${imports} + +${registrations} + +${factory} + +${utilityExports} + +${metadata} +`; + + // Generate TypeScript definitions + const dts = generateTypeDefinitions(definition); + + return { js, dts }; +} + +function generateImports( + definition: ImageDefinition, + discoveredProviders?: DiscoveredProviders +): string { + const imports: string[] = []; + + // Import base image first (if extending) - triggers side-effect provider registration + if (definition.extends) { + imports.push(`// Import base image for provider registration (side effect)`); + imports.push(`import '${definition.extends}';`); + imports.push(``); + } + + // Core imports + imports.push(`import { DextoAgent } from '@dexto/core';`); + + // Always import all registries since we re-export them in generateFactory() + // This ensures the re-exports don't reference unimported identifiers + imports.push( + `import { customToolRegistry, pluginRegistry, compactionRegistry, blobStoreRegistry } from '@dexto/core';` + ); + + // Import discovered providers + if (discoveredProviders) { + const categories = [ + { key: 'blobStore', label: 'Blob Storage' }, + { key: 'customTools', label: 'Custom Tools' }, + { key: 'compaction', label: 'Compaction' }, + { key: 'plugins', label: 'Plugins' }, + ] as const; + + for (const { key, label } of categories) { + const providers = discoveredProviders[key]; + if (providers.length > 0) { + imports.push(``); + imports.push(`// ${label} providers (auto-discovered)`); + providers.forEach((path, index) => { + const varName = `${key}Provider${index}`; + imports.push(`import * as ${varName} from '${path}';`); + }); + } + } + } + + return imports.join('\n'); +} + +function generateProviderRegistrations( + definition: ImageDefinition, + discoveredProviders?: DiscoveredProviders +): string { + const registrations: string[] = []; + + if (definition.extends) { + registrations.push( + `// Base image providers already registered via import of '${definition.extends}'` + ); + registrations.push(''); + } + + registrations.push('// SIDE EFFECT: Register providers on import'); + registrations.push(''); + + // Auto-register discovered providers + if (discoveredProviders) { + const categoryMap = [ + { key: 'blobStore', registry: 'blobStoreRegistry', label: 'Blob Storage' }, + { key: 'customTools', registry: 'customToolRegistry', label: 'Custom Tools' }, + { key: 'compaction', registry: 'compactionRegistry', label: 'Compaction' }, + { key: 'plugins', registry: 'pluginRegistry', label: 'Plugins' }, + ] as const; + + for (const { key, registry, label } of categoryMap) { + const providers = discoveredProviders[key]; + if (providers.length === 0) continue; + + registrations.push(`// Auto-register ${label} providers`); + providers.forEach((path, index) => { + const varName = `${key}Provider${index}`; + registrations.push(`// From ${path}`); + registrations.push(`for (const exported of Object.values(${varName})) {`); + registrations.push( + ` if (exported && typeof exported === 'object' && 'type' in exported && 'create' in exported) {` + ); + registrations.push(` try {`); + registrations.push(` ${registry}.register(exported);`); + registrations.push( + ` console.log(\`✓ Registered ${key}: \${exported.type}\`);` + ); + registrations.push(` } catch (err) {`); + registrations.push(` // Ignore duplicate registration errors`); + registrations.push( + ` if (!err.message?.includes('already registered')) throw err;` + ); + registrations.push(` }`); + registrations.push(` }`); + registrations.push(`}`); + }); + registrations.push(''); + } + } + + // Handle manual registration functions (backwards compatibility) + for (const [category, config] of Object.entries(definition.providers)) { + if (!config) continue; + + if (config.register) { + // Async registration function with duplicate prevention + registrations.push(`// Register ${category} via custom function (from dexto.image.ts)`); + registrations.push(`await (async () => {`); + registrations.push(` try {`); + registrations.push( + ` ${config.register + .toString() + .replace(/^async\s*\(\)\s*=>\s*{/, '') + .replace(/}$/, '')}` + ); + registrations.push(` } catch (err) {`); + registrations.push(` // Ignore duplicate registration errors`); + registrations.push(` if (!err.message?.includes('already registered')) {`); + registrations.push(` throw err;`); + registrations.push(` }`); + registrations.push(` }`); + registrations.push(`})();`); + registrations.push(''); + } + } + + return registrations.join('\n'); +} + +function generateFactory(): string { + return `/** + * Create a Dexto agent using this image's registered providers. + * + * @param config - Agent configuration + * @param configPath - Optional path to config file + * @returns DextoAgent instance with providers already registered + */ +export function createAgent(config, configPath) { + return new DextoAgent(config, configPath); +} + +/** + * Re-export registries for runtime customization. + * This allows apps to add custom providers without depending on @dexto/core directly. + */ +export { + customToolRegistry, + pluginRegistry, + compactionRegistry, + blobStoreRegistry, +} from '@dexto/core';`; +} + +function generateUtilityExports(definition: ImageDefinition): string { + const sections: string[] = []; + + // Generate wildcard utility exports + if (definition.utils && Object.keys(definition.utils).length > 0) { + sections.push('// Utility exports'); + for (const [name, path] of Object.entries(definition.utils)) { + sections.push(`export * from '${path}';`); + } + } + + // Generate selective named exports (filter out type-only exports for runtime JS) + if (definition.exports && Object.keys(definition.exports).length > 0) { + if (sections.length > 0) sections.push(''); + sections.push('// Selective package re-exports'); + for (const [packageName, exports] of Object.entries(definition.exports)) { + // Check for wildcard re-export + if (exports.length === 1 && exports[0] === '*') { + sections.push(`export * from '${packageName}';`); + continue; + } + + // Filter out type-only exports (those starting with 'type ') + const runtimeExports = exports.filter((exp) => !exp.startsWith('type ')); + if (runtimeExports.length > 0) { + sections.push(`export {`); + sections.push(` ${runtimeExports.join(',\n ')}`); + sections.push(`} from '${packageName}';`); + } + } + } + + if (sections.length === 0) { + return '// No utilities or exports defined for this image'; + } + + return sections.join('\n'); +} + +function generateMetadata(definition: ImageDefinition, coreVersion: string): string { + const metadata: Record = { + name: definition.name, + version: definition.version, + description: definition.description, + target: definition.target || 'custom', + constraints: definition.constraints || [], + builtAt: new Date().toISOString(), + coreVersion: coreVersion, + }; + + // Include extends information if present + if (definition.extends) { + metadata.extends = definition.extends; + } + + // Include bundled plugins if present + if (definition.bundledPlugins && definition.bundledPlugins.length > 0) { + metadata.bundledPlugins = definition.bundledPlugins; + } + + return `/** + * Image metadata + * Generated at build time + */ +export const imageMetadata = ${JSON.stringify(metadata, null, 4)};`; +} + +function generateTypeDefinitions(definition: ImageDefinition): string { + const sections: string[] = []; + + // Wildcard utility exports + if (definition.utils && Object.keys(definition.utils).length > 0) { + sections.push('// Utility re-exports'); + for (const path of Object.values(definition.utils)) { + sections.push(`export * from '${path}';`); + } + } + + // Selective named exports + if (definition.exports && Object.keys(definition.exports).length > 0) { + if (sections.length > 0) sections.push(''); + sections.push('// Selective package re-exports'); + for (const [packageName, exports] of Object.entries(definition.exports)) { + // Check for wildcard re-export + if (exports.length === 1 && exports[0] === '*') { + sections.push(`export * from '${packageName}';`); + continue; + } + + sections.push(`export {`); + sections.push(` ${exports.join(',\n ')}`); + sections.push(`} from '${packageName}';`); + } + } + + const utilityExports = sections.length > 0 ? '\n\n' + sections.join('\n') : ''; + + return `// AUTO-GENERATED TypeScript definitions +// Do not edit this file directly + +import type { DextoAgent, AgentConfig, ImageMetadata } from '@dexto/core'; + +/** + * Create a Dexto agent using this image's registered providers. + */ +export declare function createAgent(config: AgentConfig, configPath?: string): DextoAgent; + +/** + * Image metadata + */ +export declare const imageMetadata: ImageMetadata; + +/** + * Re-exported registries for runtime customization + */ +export { + customToolRegistry, + pluginRegistry, + compactionRegistry, + blobStoreRegistry, +} from '@dexto/core';${utilityExports} +`; +} diff --git a/dexto/packages/image-bundler/src/index.ts b/dexto/packages/image-bundler/src/index.ts new file mode 100644 index 00000000..c610c5ca --- /dev/null +++ b/dexto/packages/image-bundler/src/index.ts @@ -0,0 +1,9 @@ +/** + * @dexto/bundler + * + * Bundles Dexto base images from dexto.image.ts definitions + * into importable packages with side-effect provider registration. + */ + +export { bundle } from './bundler.js'; +export type { BundleOptions, BundleResult, GeneratedCode } from './types.js'; diff --git a/dexto/packages/image-bundler/src/types.ts b/dexto/packages/image-bundler/src/types.ts new file mode 100644 index 00000000..3b65ddb6 --- /dev/null +++ b/dexto/packages/image-bundler/src/types.ts @@ -0,0 +1,30 @@ +import type { ImageDefinition, ImageMetadata } from '@dexto/core'; + +export interface BundleOptions { + /** Path to dexto.image.ts file */ + imagePath: string; + /** Output directory for built image */ + outDir: string; + /** Whether to generate source maps */ + sourcemap?: boolean; + /** Whether to minify output */ + minify?: boolean; +} + +export interface BundleResult { + /** Path to generated entry file */ + entryFile: string; + /** Path to generated types file */ + typesFile: string; + /** Image metadata */ + metadata: ImageMetadata; + /** Warnings encountered during build */ + warnings: string[]; +} + +export interface GeneratedCode { + /** Generated JavaScript code */ + js: string; + /** Generated TypeScript definitions */ + dts: string; +} diff --git a/dexto/packages/image-bundler/tsconfig.json b/dexto/packages/image-bundler/tsconfig.json new file mode 100644 index 00000000..324c4fec --- /dev/null +++ b/dexto/packages/image-bundler/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "allowImportingTsExtensions": true, + "noEmit": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/dexto/packages/image-bundler/tsup.config.ts b/dexto/packages/image-bundler/tsup.config.ts new file mode 100644 index 00000000..1aa8e3ce --- /dev/null +++ b/dexto/packages/image-bundler/tsup.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: ['src/index.ts', 'src/cli.ts'], + format: ['esm'], + dts: true, + clean: true, + sourcemap: true, + splitting: false, + shims: true, + external: ['typescript', '@dexto/core', 'picocolors', 'commander'], + noExternal: [], +}); diff --git a/dexto/packages/image-local/.gitignore b/dexto/packages/image-local/.gitignore new file mode 100644 index 00000000..dd6e803c --- /dev/null +++ b/dexto/packages/image-local/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +*.log +.DS_Store diff --git a/dexto/packages/image-local/CHANGELOG.md b/dexto/packages/image-local/CHANGELOG.md new file mode 100644 index 00000000..bac19b98 --- /dev/null +++ b/dexto/packages/image-local/CHANGELOG.md @@ -0,0 +1,108 @@ +# @dexto/image-local + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +- Updated dependencies [042f4f0] + - @dexto/agent-management@1.5.6 + - @dexto/tools-filesystem@1.5.6 + - @dexto/tools-process@1.5.6 + - @dexto/tools-todo@1.5.6 + - @dexto/core@1.5.6 + +## 1.5.5 + +### Patch Changes + +- 9ab3eac: Added todo tools. +- Updated dependencies [9ab3eac] +- Updated dependencies [63fa083] +- Updated dependencies [6df3ca9] + - @dexto/tools-todo@0.1.1 + - @dexto/core@1.5.5 + - @dexto/tools-filesystem@1.5.5 + - @dexto/tools-process@1.5.5 + - @dexto/agent-management@1.5.5 + +## 1.5.4 + +### Patch Changes + +- Updated dependencies [0016cd3] +- Updated dependencies [499b890] +- Updated dependencies [aa2c9a0] + - @dexto/core@1.5.4 + - @dexto/agent-management@1.5.4 + - @dexto/tools-filesystem@1.5.4 + - @dexto/tools-process@1.5.4 + +## 1.5.3 + +### Patch Changes + +- 4f00295: Added spawn-agent tools and explore agent. +- Updated dependencies [4f00295] +- Updated dependencies [69c944c] + - @dexto/agent-management@1.5.3 + - @dexto/tools-filesystem@1.5.3 + - @dexto/core@1.5.3 + - @dexto/tools-process@1.5.3 + +## 1.5.2 + +### Patch Changes + +- Updated dependencies [8a85ea4] +- Updated dependencies [527f3f9] + - @dexto/core@1.5.2 + - @dexto/tools-filesystem@1.5.2 + - @dexto/tools-process@1.5.2 + +## 1.5.1 + +### Patch Changes + +- Updated dependencies [bfcc7b1] +- Updated dependencies [4aabdb7] + - @dexto/core@1.5.1 + - @dexto/tools-filesystem@1.5.1 + - @dexto/tools-process@1.5.1 + +## 1.5.0 + +### Minor Changes + +- e7722e5: Minor version bump for new release with bundler, custom tool pkgs, etc. + +### Patch Changes + +- 1e7e974: Added image bundler, @dexto/image-local and moved tool services outside core. Added registry providers to select core services. +- 5fa79fa: Renamed compression to compaction, added context-awareness to hono, updated cli tool display formatting and added integration test for image-local. +- Updated dependencies [ee12727] +- Updated dependencies [1e7e974] +- Updated dependencies [4c05310] +- Updated dependencies [5fa79fa] +- Updated dependencies [ef40e60] +- Updated dependencies [e714418] +- Updated dependencies [e7722e5] +- Updated dependencies [7d5ab19] +- Updated dependencies [436a900] + - @dexto/core@1.5.0 + - @dexto/tools-filesystem@1.5.0 + - @dexto/tools-process@1.5.0 diff --git a/dexto/packages/image-local/README.md b/dexto/packages/image-local/README.md new file mode 100644 index 00000000..ceb7957f --- /dev/null +++ b/dexto/packages/image-local/README.md @@ -0,0 +1,200 @@ +# @dexto/image-local + +Local development base image for Dexto agents with filesystem and process tools. + +## Features + +- **SQLite database** - Persistent, local data storage +- **Local filesystem blob storage** - Store blobs on local disk +- **In-memory caching** - Fast temporary storage +- **FileSystem tools** - read, write, edit, glob, grep operations +- **Process tools** - bash exec, output, kill operations +- **Offline-capable** - No external dependencies required +- **Zero configuration** - Sensible defaults for local development + +## Installation + +```bash +pnpm add @dexto/image-local @dexto/core @dexto/agent-management +``` + +## Quick Start + +### 1. Create Agent Config + +```yaml +# agents/my-agent.yml +systemPrompt: + contributors: + - type: static + content: | + You are a helpful AI assistant with filesystem and process capabilities. + +llm: + provider: anthropic + model: claude-sonnet-4-5-20250514 + +# Enable filesystem and process tools +customTools: + - type: filesystem-tools + allowedPaths: ['.'] + blockedPaths: ['.git', 'node_modules'] + - type: process-tools + securityLevel: moderate +``` + +### 2. Create Your App + +```typescript +// index.ts +import { createAgent } from '@dexto/image-local'; +import { loadAgentConfig } from '@dexto/agent-management'; + +const config = await loadAgentConfig('./agents/my-agent.yml'); + +// Providers already registered! Just create and use. +const agent = createAgent(config, './agents/my-agent.yml'); +await agent.start(); + +// Agent now has filesystem and process tools available +const response = await agent.run('List the files in the current directory'); +console.log(response.content); + +await agent.shutdown(); +``` + +**Important**: When using an image, only import from the image package (`@dexto/image-local`). Do not import from `@dexto/core` directly - the image provides everything you need. + +## What's Included + +### Registered Providers + +- **Blob Storage**: `local`, `in-memory` +- **Custom Tools**: `filesystem-tools`, `process-tools` + +### FileSystem Tools + +When `filesystem-tools` is enabled in your config: + +- `read_file` - Read file contents with pagination +- `write_file` - Write or overwrite files +- `edit_file` - Edit files with search/replace operations +- `glob_files` - Find files matching glob patterns +- `grep_content` - Search file contents using regex + +### Process Tools + +When `process-tools` is enabled in your config: + +- `bash_exec` - Execute bash commands (foreground or background) +- `bash_output` - Retrieve output from background processes +- `kill_process` - Terminate background processes + +## Configuration + +### FileSystem Tools Config + +```yaml +customTools: + - type: filesystem-tools + allowedPaths: ['.', '/tmp'] + blockedPaths: ['.git', 'node_modules', '.env'] + blockedExtensions: ['.exe', '.dll'] + maxFileSize: 10485760 # 10MB + workingDirectory: /path/to/project + enableBackups: true + backupPath: ./backups + backupRetentionDays: 7 +``` + +### Process Tools Config + +```yaml +customTools: + - type: process-tools + securityLevel: moderate # strict | moderate | permissive + workingDirectory: /path/to/project + maxTimeout: 30000 # milliseconds + maxConcurrentProcesses: 5 + maxOutputBuffer: 1048576 # 1MB + allowedCommands: ['ls', 'cat', 'grep'] # strict mode only + blockedCommands: ['rm -rf', 'sudo'] + environment: + MY_VAR: value +``` + +## Architecture + +### On-Demand Service Initialization + +Services are initialized only when needed: + +- **FileSystemService** is created when `filesystem-tools` provider is used +- **ProcessService** is created when `process-tools` provider is used +- No overhead if tools aren't configured + +### Provider Registration + +The image uses **side-effect registration** - providers are registered automatically when you import from the package: + +```typescript +import { createAgent } from '@dexto/image-local'; +// Providers registered as side-effect! ✓ +``` + +All exports from the image (`createAgent`, registries, etc.) trigger provider registration on first import. + +## Default Configuration + +The image provides sensible defaults: + +```typescript +{ + storage: { + blob: { type: 'local', storePath: './data/blobs' }, + database: { type: 'sqlite', path: './data/agent.db' }, + cache: { type: 'in-memory' } + }, + logging: { + level: 'info', + fileLogging: true + }, + customTools: [ + { + type: 'filesystem-tools', + allowedPaths: ['.'], + blockedPaths: ['.git', 'node_modules/.bin', '.env'] + }, + { + type: 'process-tools', + securityLevel: 'moderate' + } + ] +} +``` + +## Security + +### FileSystem Tools + +- Path validation prevents directory traversal +- Blocked paths and extensions prevent access to sensitive files +- File size limits prevent memory exhaustion +- Optional backups protect against data loss + +### Process Tools + +- Command validation blocks dangerous patterns +- Injection detection prevents command injection +- Configurable security levels (strict/moderate/permissive) +- Process limits prevent resource exhaustion + +## See Also + +- [@dexto/core](../core) - Core agent framework +- [@dexto/bundler](../bundler) - Image bundler +- [Image Tutorial](../../docs/docs/tutorials/images/) - Learn about images + +## License + +MIT diff --git a/dexto/packages/image-local/dexto.image.ts b/dexto/packages/image-local/dexto.image.ts new file mode 100644 index 00000000..9edb2e71 --- /dev/null +++ b/dexto/packages/image-local/dexto.image.ts @@ -0,0 +1,104 @@ +/** + * Local Development Image + * + * Pre-configured base image for local agent development with: + * - SQLite database (persistent, local) + * - Local filesystem blob storage + * - In-memory caching + * - FileSystem tools (read, write, edit, glob, grep) + * - Process tools (bash exec, output, kill) + * - Offline-capable + * + * Tools are automatically registered when this image is imported. + * Services are initialized on-demand when tools are used. + */ + +import { defineImage } from '@dexto/core'; +import { PLUGIN_PATH as planToolsPluginPath } from '@dexto/tools-plan'; + +export default defineImage({ + name: 'image-local', + version: '1.0.0', + description: 'Local development image with filesystem and process tools', + target: 'local-development', + + // Bundled plugins - automatically discovered alongside user/project plugins + bundledPlugins: [planToolsPluginPath], + + // Provider registration + // These providers are registered as side-effects when the image is imported + providers: { + // Blob storage providers from core + blobStore: { + register: async () => { + const { localBlobStoreProvider, inMemoryBlobStoreProvider } = await import( + '@dexto/core' + ); + const { blobStoreRegistry } = await import('@dexto/core'); + + blobStoreRegistry.register(localBlobStoreProvider); + blobStoreRegistry.register(inMemoryBlobStoreProvider); + }, + }, + + // Custom tool providers from separate packages + customTools: { + register: async () => { + const { fileSystemToolsProvider } = await import('@dexto/tools-filesystem'); + const { processToolsProvider } = await import('@dexto/tools-process'); + const { agentSpawnerToolsProvider } = await import('@dexto/agent-management'); + const { todoToolsProvider } = await import('@dexto/tools-todo'); + const { planToolsProvider } = await import('@dexto/tools-plan'); + const { customToolRegistry } = await import('@dexto/core'); + + customToolRegistry.register(fileSystemToolsProvider); + customToolRegistry.register(processToolsProvider); + customToolRegistry.register(agentSpawnerToolsProvider); + customToolRegistry.register(todoToolsProvider); + customToolRegistry.register(planToolsProvider); + }, + }, + }, + + // Default configuration values + defaults: { + storage: { + blob: { + type: 'local', + storePath: './data/blobs', + }, + database: { + type: 'sqlite', + path: './data/agent.db', + }, + cache: { + type: 'in-memory', + }, + }, + logging: { + level: 'info', + fileLogging: true, + }, + // Default custom tools configuration + // Users can add these to their config to enable filesystem and process tools + customTools: [ + { + type: 'filesystem-tools', + allowedPaths: ['.'], + blockedPaths: ['.git', 'node_modules/.bin', '.env'], + blockedExtensions: ['.exe', '.dll', '.so'], + enableBackups: false, + }, + { + type: 'process-tools', + securityLevel: 'moderate', + }, + { + type: 'todo-tools', + }, + ], + }, + + // Runtime constraints + constraints: ['filesystem-required', 'offline-capable'], +}); diff --git a/dexto/packages/image-local/package.json b/dexto/packages/image-local/package.json new file mode 100644 index 00000000..f21e31d1 --- /dev/null +++ b/dexto/packages/image-local/package.json @@ -0,0 +1,45 @@ +{ + "name": "@dexto/image-local", + "version": "1.5.6", + "description": "Local development base image for Dexto agents with filesystem and process tools", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsx ../image-bundler/dist/cli.js build", + "test": "vitest run", + "test:integ": "vitest run", + "typecheck": "tsc --noEmit", + "clean": "rm -rf dist" + }, + "keywords": [ + "dexto", + "image", + "local-development", + "base-image", + "filesystem", + "process" + ], + "dependencies": { + "@dexto/agent-management": "workspace:*", + "@dexto/core": "workspace:*", + "@dexto/tools-filesystem": "workspace:*", + "@dexto/tools-plan": "workspace:*", + "@dexto/tools-process": "workspace:*", + "@dexto/tools-todo": "workspace:*" + }, + "devDependencies": { + "@dexto/image-bundler": "workspace:*", + "typescript": "^5.3.3" + }, + "files": [ + "dist", + "README.md" + ] +} \ No newline at end of file diff --git a/dexto/packages/image-local/test/import.integration.test.ts b/dexto/packages/image-local/test/import.integration.test.ts new file mode 100644 index 00000000..6a5b2dc1 --- /dev/null +++ b/dexto/packages/image-local/test/import.integration.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect } from 'vitest'; + +/** + * Integration test to ensure image-local can be imported successfully. + * This catches issues where generated code references renamed/missing exports. + */ +describe('Image Local - Import Integration', () => { + it('should import image-local without errors', async () => { + // This will fail if the generated code has incorrect imports + const module = await import('@dexto/image-local'); + + expect(module).toBeDefined(); + expect(module.createAgent).toBeDefined(); + expect(module.imageMetadata).toBeDefined(); + }); + + it('should have correct registry exports', async () => { + const module = await import('@dexto/image-local'); + + // Verify all registries are exported with correct names + expect(module.customToolRegistry).toBeDefined(); + expect(module.pluginRegistry).toBeDefined(); + expect(module.compactionRegistry).toBeDefined(); + expect(module.blobStoreRegistry).toBeDefined(); + }); + + it('should not reference old registry names', async () => { + // Read the generated file to ensure no old names remain + const fs = await import('fs/promises'); + const path = await import('path'); + const { fileURLToPath } = await import('url'); + + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + const distPath = path.resolve(currentDir, '../dist/index.js'); + const content = await fs.readFile(distPath, 'utf-8'); + + // Should not contain old name + expect(content).not.toContain('compressionRegistry'); + + // Should contain new name + expect(content).toContain('compactionRegistry'); + }); +}); diff --git a/dexto/packages/image-local/tsconfig.json b/dexto/packages/image-local/tsconfig.json new file mode 100644 index 00000000..c6f9561f --- /dev/null +++ b/dexto/packages/image-local/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "allowImportingTsExtensions": true, + "noEmit": true + }, + "include": ["dexto.image.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/dexto/packages/image-local/vitest.config.ts b/dexto/packages/image-local/vitest.config.ts new file mode 100644 index 00000000..47c6b053 --- /dev/null +++ b/dexto/packages/image-local/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + include: ['test/**/*.integration.test.ts'], + }, +}); diff --git a/dexto/packages/registry/CHANGELOG.md b/dexto/packages/registry/CHANGELOG.md new file mode 100644 index 00000000..ffb298d1 --- /dev/null +++ b/dexto/packages/registry/CHANGELOG.md @@ -0,0 +1,46 @@ +# @dexto/registry + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +## 1.5.5 + +### Patch Changes + +- 6df3ca9: Updated readme. Removed stale filesystem and process tool from dexto/core. + +## 1.5.4 + +## 1.5.3 + +## 1.5.2 + +## 1.5.1 + +## 1.5.0 + +### Minor Changes + +- e7722e5: Minor version bump for new release with bundler, custom tool pkgs, etc. + +## 1.4.0 + +### Minor Changes + +- f73a519: Revamp CLI. Breaking change to DextoAgent.generate() and stream() apis and hono message APIs, so new minor version. Other fixes for logs, web UI related to message streaming/generating diff --git a/dexto/packages/registry/package.json b/dexto/packages/registry/package.json new file mode 100644 index 00000000..18d74a49 --- /dev/null +++ b/dexto/packages/registry/package.json @@ -0,0 +1,34 @@ +{ + "name": "@dexto/registry", + "version": "1.5.6", + "private": false, + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.cjs" + }, + "./package.json": "./package.json" + }, + "scripts": { + "build": "tsup && tsc -p tsconfig.json --emitDeclarationOnly", + "dev": "tsup --watch", + "typecheck": "tsc -p tsconfig.typecheck.json --noEmit", + "lint": "eslint . --ext .ts" + }, + "files": [ + "dist", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "devDependencies": { + "tsup": "^8.0.0", + "typescript": "^5" + } +} diff --git a/dexto/packages/registry/src/index.ts b/dexto/packages/registry/src/index.ts new file mode 100644 index 00000000..1e113f5f --- /dev/null +++ b/dexto/packages/registry/src/index.ts @@ -0,0 +1,8 @@ +/** + * @dexto/registry + * + * Shared registry data for Dexto CLI and WebUI. + * Contains MCP server presets and future registry types. + */ + +export * from './mcp/index.js'; diff --git a/dexto/packages/registry/src/mcp/index.ts b/dexto/packages/registry/src/mcp/index.ts new file mode 100644 index 00000000..fa4865b1 --- /dev/null +++ b/dexto/packages/registry/src/mcp/index.ts @@ -0,0 +1,6 @@ +/** + * MCP Server Registry exports + */ + +export * from './types.js'; +export { ServerRegistryService, getServerRegistry, serverRegistry } from './server-registry.js'; diff --git a/dexto/packages/registry/src/mcp/server-registry-data.json b/dexto/packages/registry/src/mcp/server-registry-data.json new file mode 100644 index 00000000..bdcc5ef2 --- /dev/null +++ b/dexto/packages/registry/src/mcp/server-registry-data.json @@ -0,0 +1,558 @@ +[ + { + "id": "filesystem", + "name": "Filesystem", + "description": "Secure file operations with configurable access controls for reading and writing files", + "category": "productivity", + "icon": "📁", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "."], + "timeout": 30000 + }, + "tags": ["file", "directory", "filesystem", "io"], + "isOfficial": true, + "isInstalled": false, + "author": "Anthropic", + "homepage": "https://github.com/modelcontextprotocol/servers", + "matchIds": ["filesystem"] + }, + { + "id": "meme-mcp", + "name": "Meme Generator", + "description": "Create memes using Imgflip templates", + "category": "creative", + "icon": "🖼️", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "meme-mcp"], + "env": { + "IMGFLIP_USERNAME": "", + "IMGFLIP_PASSWORD": "" + }, + "timeout": 30000 + }, + "tags": ["meme", "image", "creative"], + "isOfficial": false, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Community", + "homepage": "https://www.npmjs.com/package/meme-mcp", + "matchIds": ["meme-mcp"] + }, + { + "id": "product-name-scout", + "name": "Product Name Scout", + "description": "SERP analysis, autocomplete, dev collisions, and scoring for product names", + "category": "research", + "icon": "🔎", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@truffle-ai/product-name-scout-mcp"], + "timeout": 30000 + }, + "tags": ["research", "naming", "brand"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Truffle AI", + "homepage": "https://github.com/truffle-ai/mcp-servers", + "matchIds": ["product-name-scout"] + }, + { + "id": "duckduckgo", + "name": "DuckDuckGo Search", + "description": "Search the web using DuckDuckGo", + "category": "research", + "icon": "🦆", + "config": { + "type": "stdio", + "command": "uvx", + "args": ["duckduckgo-mcp-server"], + "timeout": 30000 + }, + "tags": ["search", "web", "research"], + "isOfficial": false, + "isInstalled": false, + "requirements": { "platform": "all", "python": ">=3.10", "dependencies": ["uv"] }, + "author": "Community", + "homepage": "https://github.com/duckduckgo/mcp-server", + "matchIds": ["duckduckgo", "ddg"] + }, + { + "id": "domain-checker", + "name": "Domain Checker", + "description": "Check domain availability across TLDs", + "category": "research", + "icon": "🌐", + "config": { + "type": "stdio", + "command": "uvx", + "args": ["truffle-ai-domain-checker-mcp"], + "timeout": 30000 + }, + "tags": ["domains", "availability", "research"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "python": ">=3.10" }, + "author": "Truffle AI", + "homepage": "https://github.com/truffle-ai/mcp-servers", + "matchIds": ["domain-checker"] + }, + { + "id": "linear", + "name": "Linear", + "description": "Manage Linear issues, projects, and workflows", + "category": "productivity", + "icon": "📋", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "mcp-remote", "https://mcp.linear.app/sse"], + "timeout": 30000 + }, + "tags": ["linear", "tasks", "projects"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Linear", + "homepage": "https://mcp.linear.app", + "matchIds": ["linear"] + }, + { + "id": "image-editor", + "name": "Image Editor", + "description": "Comprehensive image processing and manipulation tools", + "category": "creative", + "icon": "🖌️", + "config": { + "type": "stdio", + "command": "uvx", + "args": ["truffle-ai-image-editor-mcp"], + "timeout": 30000 + }, + "tags": ["image", "edit", "opencv", "pillow"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "python": ">=3.10" }, + "author": "Truffle AI", + "homepage": "https://github.com/truffle-ai/mcp-servers", + "matchIds": ["image_editor", "image-editor"] + }, + { + "id": "music-creator", + "name": "Music Creator", + "description": "Create, analyze, and transform music and audio", + "category": "creative", + "icon": "🎵", + "config": { + "type": "stdio", + "command": "uvx", + "args": ["truffle-ai-music-creator-mcp"], + "timeout": 30000 + }, + "tags": ["audio", "music", "effects"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "python": ">=3.10" }, + "author": "Truffle AI", + "homepage": "https://github.com/truffle-ai/mcp-servers", + "matchIds": ["music_creator", "music-creator"] + }, + { + "id": "elevenlabs", + "name": "ElevenLabs", + "description": "Text-to-speech and voice synthesis using ElevenLabs API", + "category": "creative", + "icon": "🎤", + "config": { + "type": "stdio", + "command": "uvx", + "args": ["elevenlabs-mcp"], + "env": { + "ELEVENLABS_API_KEY": "" + }, + "timeout": 30000 + }, + "tags": ["tts", "voice", "audio", "synthesis"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "python": ">=3.10" }, + "author": "ElevenLabs", + "homepage": "https://github.com/elevenlabs/elevenlabs-mcp", + "matchIds": ["elevenlabs"] + }, + { + "id": "hf", + "name": "Hugging Face", + "description": "Access Hugging Face models and datasets", + "category": "development", + "icon": "🤗", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@llmindset/mcp-hfspace"], + "timeout": 30000 + }, + "tags": ["huggingface", "models", "ai", "ml"], + "isOfficial": false, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "LLMindset", + "homepage": "https://github.com/llmindset/mcp-hfspace", + "matchIds": ["hf", "huggingface"] + }, + { + "id": "tavily", + "name": "Tavily Search", + "description": "Web search and research using Tavily AI search engine", + "category": "research", + "icon": "🔍", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "tavily-mcp@0.1.3"], + "env": { + "TAVILY_API_KEY": "" + }, + "timeout": 30000 + }, + "tags": ["search", "web", "research", "ai"], + "isOfficial": false, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Tavily AI", + "homepage": "https://www.npmjs.com/package/tavily-mcp", + "matchIds": ["tavily"] + }, + { + "id": "puppeteer", + "name": "Puppeteer", + "description": "Browser automation and web interaction tools", + "category": "productivity", + "icon": "🌐", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@truffle-ai/puppeteer-server"], + "timeout": 30000 + }, + "tags": ["browser", "automation", "web", "puppeteer"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Truffle AI", + "homepage": "https://github.com/truffle-ai/mcp-servers", + "matchIds": ["puppeteer"] + }, + { + "id": "gemini-tts", + "name": "Gemini TTS", + "description": "Google Gemini Text-to-Speech with 30 prebuilt voices and multi-speaker conversation support", + "category": "creative", + "icon": "🎙️", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@truffle-ai/gemini-tts-server"], + "env": { + "GEMINI_API_KEY": "" + }, + "timeout": 60000 + }, + "tags": ["tts", "speech", "voice", "audio", "gemini", "multi-speaker"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Truffle AI", + "homepage": "https://github.com/truffle-ai/mcp-servers", + "matchIds": ["gemini-tts", "gemini_tts"] + }, + { + "id": "nano-banana", + "name": "Nano Banana", + "description": "Google Gemini 2.5 Flash Image for advanced image generation, editing, and manipulation", + "category": "creative", + "icon": "🍌", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@truffle-ai/nano-banana-server@0.1.2"], + "env": { + "GEMINI_API_KEY": "" + }, + "timeout": 60000 + }, + "tags": ["image", "generation", "editing", "ai", "gemini", "nano-banana"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Truffle AI", + "homepage": "https://github.com/truffle-ai/mcp-servers", + "matchIds": ["nano-banana", "nano_banana"] + }, + { + "id": "openai-image", + "name": "OpenAI Image", + "description": "OpenAI's image generation and editing API with GPT Image models and DALL-E support", + "category": "creative", + "icon": "🎨", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@truffle-ai/openai-image-server"], + "env": { + "OPENAI_API_KEY": "" + }, + "timeout": 60000 + }, + "tags": ["image", "generation", "editing", "ai", "openai", "dall-e", "gpt-image"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Truffle AI", + "homepage": "https://github.com/truffle-ai/mcp-servers/tree/main/src/openai-image", + "matchIds": ["openai-image", "openai_image"] + }, + { + "id": "heygen", + "name": "HeyGen", + "description": "Generate realistic human-like audio using HeyGen", + "category": "creative", + "icon": "🎤", + "config": { + "type": "stdio", + "command": "uvx", + "args": ["heygen-mcp"], + "env": { + "HEYGEN_API_KEY": "" + }, + "timeout": 30000 + }, + "tags": ["audio", "voice", "synthesis", "heygen"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "python": ">=3.10" }, + "author": "HeyGen", + "homepage": "https://github.com/heygen-com/heygen-mcp", + "matchIds": ["heygen"] + }, + { + "id": "runway", + "name": "Runway", + "description": "AI-powered creative suite for video and image generation", + "category": "creative", + "icon": "🎬", + "config": { + "type": "stdio", + "command": "npx", + "args": [ + "mcp-remote", + "https://mcp.runway.team", + "--header", + "Authorization: Bearer ${RUNWAY_API_KEY}" + ], + "env": { + "RUNWAY_API_KEY": "" + }, + "timeout": 60000 + }, + "tags": ["runway", "video", "generation", "ai", "creative"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Runway", + "homepage": "https://docs.runway.team/api/runway-mcp-server", + "matchIds": ["runway"] + }, + { + "id": "perplexity", + "name": "Perplexity", + "description": "AI-powered search engine for real-time web search and research", + "category": "research", + "icon": "🔍", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@perplexity-ai/mcp-server"], + "env": { + "PERPLEXITY_API_KEY": "", + "PERPLEXITY_TIMEOUT_MS": "600000" + }, + "timeout": 600000 + }, + "tags": ["search", "web", "research", "ai"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Perplexity AI", + "homepage": "https://github.com/perplexityai/modelcontextprotocol/tree/main", + "matchIds": ["perplexity"] + }, + { + "id": "sora", + "name": "Sora", + "description": "AI-powered video generation using OpenAI's Sora technology", + "category": "creative", + "icon": "🎬", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@truffle-ai/sora-video-server"], + "env": { + "OPENAI_API_KEY": "" + }, + "timeout": 60000 + }, + "tags": ["video", "generation", "ai", "creative"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Truffle AI", + "homepage": "https://github.com/truffle-ai/mcp-servers", + "matchIds": ["sora", "sora_video"] + }, + { + "id": "chartjs", + "name": "ChartJS", + "description": "Charting and visualization tool using ChartJS", + "category": "data", + "icon": "📊", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@ax-crew/chartjs-mcp-server"], + "timeout": 30000 + }, + "tags": ["chart", "visualization", "data", "chartjs"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "ax-crew", + "homepage": "https://github.com/ax-crew/chartjs-mcp-server", + "matchIds": ["chartjs"] + }, + { + "id": "rag-lite-ts", + "name": "Rag-lite TS", + "description": "A local-first TypeScript retrieval engine for semantic search over static documents.", + "category": "data", + "icon": "🔍", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "raglite-mcp"], + "timeout": 30000 + }, + "tags": ["rag", "data", "ai"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "FrugalX", + "homepage": "https://github.com/raglite/rag-lite-ts", + "matchIds": ["rag-lite-ts"] + }, + { + "id": "exa", + "name": "Exa", + "description": "AI-powered web search and research API with semantic search capabilities.", + "category": "data", + "icon": "🔍", + "config": { + "type": "http", + "url": "https://mcp.exa.ai/mcp", + "headers": {} + }, + "tags": ["rag", "data", "ai"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Exa", + "homepage": "https://docs.exa.ai/reference/exa-mcp", + "matchIds": ["exa"] + }, + { + "id": "firecrawl", + "name": "Firecrawl", + "description": "AI-powered web search and research API that performs semantic search across web sources, aggregates and ranks results for relevance, and returns contextual summaries with citations", + "category": "data", + "icon": "🔍", + "config": { + "type": "stdio", + "command": "npx", + "args": ["-y", "firecrawl-mcp"], + "env": { + "FIRECRAWL_API_KEY": "" + }, + "timeout": 30000 + }, + "tags": ["search", "web", "research", "ai"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Firecrawl", + "homepage": "https://docs.firecrawl.dev/mcp-server", + "matchIds": ["firecrawl"] + }, + { + "id": "shadcn", + "name": "Shadcn", + "description": "Shadcn UI components for MCP servers", + "category": "development", + "icon": "🖌️", + "config": { + "type": "stdio", + "command": "npx", + "args": ["shadcn@latest", "mcp"], + "timeout": 30000 + }, + "tags": ["shadcn", "ui", "components", "mcp"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Shadcn", + "homepage": "https://ui.shadcn.com/docs/mcp#configuration", + "matchIds": ["shadcn"] + }, + { + "id": "kite-trade", + "name": "Kite", + "description": "Zerodha Kite API MCP server", + "category": "data", + "icon": "💰", + "config": { + "type": "http", + "url": "https://mcp.kite.trade/mcp", + "headers": {} + }, + "tags": ["kite", "trade", "data", "ai"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "Zerodha", + "homepage": "https://github.com/zerodha/kite-mcp-server", + "matchIds": ["kite-trade"] + }, + { + "id": "coingecko", + "name": "CoinGecko", + "description": "CoinGecko API MCP server", + "category": "data", + "icon": "💰", + "config": { + "type": "http", + "url": "https://mcp.api.coingecko.com/mcp", + "headers": {} + }, + "tags": ["coingecko", "data", "ai"], + "isOfficial": true, + "isInstalled": false, + "requirements": { "platform": "all", "node": ">=18.0.0" }, + "author": "CoinGecko", + "homepage": "https://docs.coingecko.com/docs/mcp-server", + "matchIds": ["coingecko"] + } +] diff --git a/dexto/packages/registry/src/mcp/server-registry.ts b/dexto/packages/registry/src/mcp/server-registry.ts new file mode 100644 index 00000000..1487a443 --- /dev/null +++ b/dexto/packages/registry/src/mcp/server-registry.ts @@ -0,0 +1,181 @@ +/** + * MCP Server Registry Service + * + * Provides access to the built-in registry of MCP servers. + * This is a shared service used by both CLI and WebUI. + */ + +import type { ServerRegistryEntry, ServerRegistryFilter } from './types.js'; +import builtinRegistryData from './server-registry-data.json' with { type: 'json' }; + +/** + * Normalize an ID for comparison + */ +function normalizeId(s: string): string { + return s + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * MCP Server Registry Service + * + * Manages a registry of available MCP servers that can be quickly added to agents. + * The built-in registry data is loaded from an external JSON file. + */ +export class ServerRegistryService { + private static instance: ServerRegistryService; + private registryEntries: ServerRegistryEntry[] = []; + private isInitialized = false; + + private constructor() { + // Private constructor for singleton + } + + static getInstance(): ServerRegistryService { + if (!ServerRegistryService.instance) { + ServerRegistryService.instance = new ServerRegistryService(); + } + return ServerRegistryService.instance; + } + + /** + * Initialize the registry with default entries + */ + async initialize(): Promise { + if (this.isInitialized) return; + + // Load built-in registry entries from JSON file + this.registryEntries = builtinRegistryData as ServerRegistryEntry[]; + this.isInitialized = true; + } + + /** + * Get all registry entries with optional filtering + */ + async getEntries(filter?: ServerRegistryFilter): Promise { + await this.initialize(); + + let filtered = [...this.registryEntries]; + + if (filter?.category) { + filtered = filtered.filter((entry) => entry.category === filter.category); + } + + if (filter?.tags?.length) { + filtered = filtered.filter((entry) => + filter.tags!.some((tag) => entry.tags.includes(tag)) + ); + } + + if (filter?.search) { + const searchLower = filter.search.toLowerCase(); + filtered = filtered.filter( + (entry) => + entry.name.toLowerCase().includes(searchLower) || + entry.description.toLowerCase().includes(searchLower) || + entry.tags.some((tag) => tag.toLowerCase().includes(searchLower)) + ); + } + + if (filter?.installed !== undefined) { + filtered = filtered.filter((entry) => entry.isInstalled === filter.installed); + } + + if (filter?.official !== undefined) { + filtered = filtered.filter((entry) => entry.isOfficial === filter.official); + } + + return filtered.sort((a, b) => { + // Sort by: installed first, then official, then name + if (a.isInstalled !== b.isInstalled) { + return a.isInstalled ? -1 : 1; + } + if (a.isOfficial !== b.isOfficial) { + return a.isOfficial ? -1 : 1; + } + return a.name.localeCompare(b.name); + }); + } + + /** + * Update an existing registry entry's state + */ + async updateEntry(id: string, updates: Partial): Promise { + await this.initialize(); + const entry = this.registryEntries.find((e) => e.id === id); + if (!entry) return false; + + // Use Object.assign to merge partial updates + Object.assign(entry, updates); + + return true; + } + + /** + * Mark a server as installed/uninstalled + */ + async setInstalled(id: string, installed: boolean): Promise { + return this.updateEntry(id, { isInstalled: installed }); + } + + /** + * Sync registry installed status with a list of connected server IDs + */ + async syncInstalledStatus(connectedServerIds: string[]): Promise { + await this.initialize(); + + const normalizedIds = new Set(connectedServerIds.map(normalizeId)); + + for (const entry of this.registryEntries) { + const aliases = [entry.id, entry.name, ...(entry.matchIds || [])] + .filter(Boolean) + .map((x) => normalizeId(String(x))); + + const isInstalled = aliases.some((alias) => normalizedIds.has(alias)); + + if (entry.isInstalled !== isInstalled) { + entry.isInstalled = isInstalled; + } + } + } + + /** + * Get a single server configuration by ID + */ + async getServerConfig(id: string): Promise { + await this.initialize(); + return this.registryEntries.find((entry) => entry.id === id) || null; + } + + /** + * Get all available categories + */ + async getCategories(): Promise { + await this.initialize(); + const categories = new Set(this.registryEntries.map((entry) => entry.category)); + return Array.from(categories).sort(); + } + + /** + * Get all available tags + */ + async getTags(): Promise { + await this.initialize(); + const tags = new Set(this.registryEntries.flatMap((entry) => entry.tags)); + return Array.from(tags).sort(); + } +} + +/** + * Get the singleton registry instance + */ +export function getServerRegistry(): ServerRegistryService { + return ServerRegistryService.getInstance(); +} + +/** + * Export singleton instance for convenience + */ +export const serverRegistry = ServerRegistryService.getInstance(); diff --git a/dexto/packages/registry/src/mcp/types.ts b/dexto/packages/registry/src/mcp/types.ts new file mode 100644 index 00000000..68ee43b5 --- /dev/null +++ b/dexto/packages/registry/src/mcp/types.ts @@ -0,0 +1,95 @@ +/** + * MCP Server Registry Types + * + * Defines types for the MCP server registry, which provides + * preset server configurations for easy installation. + */ + +/** + * Server category for organization and filtering + */ +export type ServerCategory = + | 'productivity' + | 'development' + | 'research' + | 'creative' + | 'data' + | 'communication' + | 'custom'; + +/** + * Server configuration for different transport types + */ +export interface ServerConfig { + type: 'stdio' | 'sse' | 'http'; + command?: string; + args?: string[]; + url?: string; + baseUrl?: string; + env?: Record; + headers?: Record; + timeout?: number; +} + +/** + * Platform and dependency requirements for a server + */ +export interface ServerRequirements { + platform?: 'win32' | 'darwin' | 'linux' | 'all'; + node?: string; + python?: string; + dependencies?: string[]; +} + +/** + * A single entry in the MCP server registry + */ +export interface ServerRegistryEntry { + /** Unique identifier for the server */ + id: string; + /** Display name */ + name: string; + /** Description of what the server does */ + description: string; + /** Category for organization */ + category: ServerCategory; + /** Emoji icon for display */ + icon?: string; + /** Author or maintainer */ + author?: string; + /** Homepage or documentation URL */ + homepage?: string; + /** Server connection configuration */ + config: ServerConfig; + /** Tags for search and filtering */ + tags: string[]; + /** Whether this is an official/verified server */ + isOfficial: boolean; + /** Whether this server is currently installed (runtime state) */ + isInstalled: boolean; + /** System requirements */ + requirements?: ServerRequirements; + /** Alternative IDs used to match against connected servers */ + matchIds?: string[]; +} + +/** + * Filter options for querying the registry + */ +export interface ServerRegistryFilter { + category?: string; + tags?: string[]; + search?: string; + installed?: boolean; + official?: boolean; +} + +/** + * State container for registry UI + */ +export interface ServerRegistryState { + entries: ServerRegistryEntry[]; + isLoading: boolean; + error?: string; + lastUpdated?: Date; +} diff --git a/dexto/packages/registry/tsconfig.json b/dexto/packages/registry/tsconfig.json new file mode 100644 index 00000000..27225968 --- /dev/null +++ b/dexto/packages/registry/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": false, + "resolveJsonModule": true + }, + "include": ["src/**/*"], + "exclude": ["**/*.test.ts", "**/*.spec.ts", "**/*.integration.test.ts", "dist", "node_modules"] +} diff --git a/dexto/packages/registry/tsconfig.typecheck.json b/dexto/packages/registry/tsconfig.typecheck.json new file mode 100644 index 00000000..90a93a82 --- /dev/null +++ b/dexto/packages/registry/tsconfig.typecheck.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["dist", "node_modules"] +} diff --git a/dexto/packages/registry/tsup.config.ts b/dexto/packages/registry/tsup.config.ts new file mode 100644 index 00000000..68254ced --- /dev/null +++ b/dexto/packages/registry/tsup.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig([ + { + entry: ['src/index.ts'], + format: ['esm', 'cjs'], + outDir: 'dist', + dts: false, // Use tsc for DTS generation (consistent with other packages) + clean: true, + bundle: true, + platform: 'node', + // Include JSON files in the bundle + loader: { + '.json': 'json', + }, + }, +]); diff --git a/dexto/packages/server/CHANGELOG.md b/dexto/packages/server/CHANGELOG.md new file mode 100644 index 00000000..e9441c81 --- /dev/null +++ b/dexto/packages/server/CHANGELOG.md @@ -0,0 +1,288 @@ +# @dexto/server + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +- Updated dependencies [042f4f0] + - @dexto/agent-management@1.5.6 + - @dexto/image-local@1.5.6 + - @dexto/core@1.5.6 + +## 1.5.5 + +### Patch Changes + +- Updated dependencies [9ab3eac] +- Updated dependencies [63fa083] +- Updated dependencies [6df3ca9] + - @dexto/image-local@1.5.5 + - @dexto/core@1.5.5 + - @dexto/agent-management@1.5.5 + +## 1.5.4 + +### Patch Changes + +- Updated dependencies [0016cd3] +- Updated dependencies [499b890] +- Updated dependencies [aa2c9a0] + - @dexto/core@1.5.4 + - @dexto/agent-management@1.5.4 + - @dexto/image-local@1.5.4 + +## 1.5.3 + +### Patch Changes + +- 69c944c: File integrity & performance improvements, approval system fixes, and developer experience enhancements + + ### File System Improvements + - **File integrity protection**: Store file hashes to prevent edits from corrupting files when content changes between operations (resolves #516) + - **Performance optimization**: Disable backups and remove redundant reads, switch to async non-blocking reads for faster file writes + + ### Approval System Fixes + - **Coding agent auto-approve**: Fix auto-approve not working due to incorrect tool names in auto-approve policies + - **Parallel tool calls**: Fix multiple parallel same-tool calls requiring redundant approvals - now checks all waiting approvals and resolves ones affected by newly approved commands + - **Refactored CLI approval handler**: Decoupled approval handler pattern from server for better separation of concerns + + ### Shell & Scripting Fixes + - **Bash mode aliases**: Fix bash mode not honoring zsh aliases + - **Script improvements**: Miscellaneous script improvements for better developer experience + +- Updated dependencies [4f00295] +- Updated dependencies [69c944c] + - @dexto/agent-management@1.5.3 + - @dexto/image-local@1.5.3 + - @dexto/core@1.5.3 + +## 1.5.2 + +### Patch Changes + +- Updated dependencies [91acb03] +- Updated dependencies [8a85ea4] +- Updated dependencies [527f3f9] + - @dexto/agent-management@1.5.2 + - @dexto/core@1.5.2 + - @dexto/image-local@1.5.2 + +## 1.5.1 + +### Patch Changes + +- bfcc7b1: PostgreSQL improvements and privacy mode + + **PostgreSQL enhancements:** + - Add connection resilience for serverless databases (Neon, Supabase, etc.) with automatic retry on connection failures + - Support custom PostgreSQL schemas via `options.schema` config + - Add schema name validation to prevent SQL injection + - Improve connection pool error handling to prevent process crashes + + **Privacy mode:** + - Add `--privacy-mode` CLI flag to hide file paths from output (useful for screen recording/sharing) + - Can also be enabled via `DEXTO_PRIVACY_MODE=true` environment variable + + **Session improvements:** + - Add message deduplication in history provider to handle data corruption gracefully + - Add warning when conversation history hits 10k message limit + - Improve session deletion to ensure messages are always cleaned up + + **Other fixes:** + - Sanitize explicit `agentId` for filesystem safety + - Change verbose flush logs to debug level + - Export `BaseTypedEventEmitter` from events module + +- Updated dependencies [a25d3ee] +- Updated dependencies [bfcc7b1] +- Updated dependencies [4aabdb7] + - @dexto/agent-management@1.5.1 + - @dexto/core@1.5.1 + - @dexto/image-local@1.5.1 + +## 1.5.0 + +### Minor Changes + +- e7722e5: Minor version bump for new release with bundler, custom tool pkgs, etc. + +### Patch Changes + +- ee12727: Added support for node-llama (llama.cpp) for local GGUF models. Added Ollama as first-class provider. Updated onboarding/setup flow. +- 1e7e974: Added image bundler, @dexto/image-local and moved tool services outside core. Added registry providers to select core services. +- 4c05310: Improve local model/GGUF model support, bash permission fixes in TUI, and add local/ollama switching/deleting support in web UI +- 5fa79fa: Renamed compression to compaction, added context-awareness to hono, updated cli tool display formatting and added integration test for image-local. +- 263fcc6: Add disableAuth parameter for custom auth layers +- ef40e60: Upgrades package versions and related changes to MCP SDK. CLI colors improved and token streaming added to status bar. + + Security: Resolve all Dependabot security vulnerabilities. Updated @modelcontextprotocol/sdk to 1.25.2, esbuild to 0.25.0, langchain to 0.3.37, and @langchain/core to 0.3.80. Added pnpm overrides for indirect vulnerabilities (preact@10.27.3, qs@6.14.1, jws@3.2.3, mdast-util-to-hast@13.2.1). Fixed type errors from MCP SDK breaking changes. + +- 7d5ab19: Updated WebUI design, event and state management and forms +- 436a900: Add support for openrouter, bedrock, glama, vertex ai, fix model switching issues and new model experience for each +- Updated dependencies [ee12727] +- Updated dependencies [1e7e974] +- Updated dependencies [4c05310] +- Updated dependencies [5fa79fa] +- Updated dependencies [ef40e60] +- Updated dependencies [e714418] +- Updated dependencies [e7722e5] +- Updated dependencies [7d5ab19] +- Updated dependencies [436a900] + - @dexto/agent-management@1.5.0 + - @dexto/core@1.5.0 + - @dexto/image-local@1.5.0 + +## 1.4.0 + +### Minor Changes + +- f73a519: Revamp CLI. Breaking change to DextoAgent.generate() and stream() apis and hono message APIs, so new minor version. Other fixes for logs, web UI related to message streaming/generating + +### Patch Changes + +- 7a64414: Updated agent-management to use AgentManager instead of AgentOrchestrator. +- 3cdce89: Revamp CLI for coding agent, add new events, improve mcp management, custom models, minor UI changes, prompt management +- d640e40: Remove LLM services, tokenizers, just stick with vercel, remove 'router' from schema and all types and docs +- 6f5627d: - Approval timeouts are now optional, defaulting to no timeout (infinite wait) + - Tool call history now includes success/failure status tracking +- c54760f: Revamp context management layer - add partial stream cancellation, message queueing, context compression with LLM, MCP UI support and gaming agent. New APIs and UI changes for these things +- Updated dependencies [bd5c097] +- Updated dependencies [7a64414] +- Updated dependencies [3cdce89] +- Updated dependencies [d640e40] +- Updated dependencies [6f5627d] +- Updated dependencies [6e6a3e7] +- Updated dependencies [f73a519] +- Updated dependencies [c54760f] +- Updated dependencies [ab47df8] +- Updated dependencies [3b4b919] + - @dexto/core@1.4.0 + - @dexto/agent-management@1.4.0 + +## 1.3.0 + +### Minor Changes + +- eb266af: Migrate WebUI from next-js to vite. Fix any typing in web UI. Improve types in core. minor renames in event schemas + +### Patch Changes + +- Updated dependencies [e2f770b] +- Updated dependencies [f843b62] +- Updated dependencies [eb266af] + - @dexto/core@1.3.0 + - @dexto/agent-management@1.3.0 + +## 1.2.6 + +### Patch Changes + +- 7feb030: Update memory and prompt configs, fix agent install bug +- Updated dependencies [7feb030] + - @dexto/core@1.2.6 + - @dexto/agent-management@1.2.6 + +## 1.2.5 + +### Patch Changes + +- c1e814f: ## Logger v2 & Config Enrichment + + ### New Features + - **Multi-transport logging system**: Configure console, file, and remote logging transports via `logger` field in agent.yml. Supports log levels (error, warn, info, debug, silly) and automatic log rotation for file transports. + - **Per-agent isolation**: CLI automatically creates per-agent log files at `~/.dexto/logs/.log`, database at `~/.dexto/database/.db`, and blob storage at `~/.dexto/blobs//` + - **Agent ID derivation**: Agent ID is now automatically derived from `agentCard.name` (sanitized) or config filename, enabling proper multi-agent isolation without manual configuration + + ### Breaking Changes + - **Storage blob default changed**: Default blob storage type changed from `local` to `in-memory`. Existing configs with explicit `blob: { type: 'local' }` are unaffected. CLI enrichment provides automatic paths for SQLite and local blob storage. + + ### Improvements + - **Config enrichment layer**: New `enrichAgentConfig()` in agent-management package adds per-agent paths before initialization, eliminating path resolution in core services + - **Logger error factory**: Added typed error factory pattern for logger errors following project conventions + - **Removed wildcard exports**: Logger module now uses explicit named exports for better tree-shaking + + ### Documentation + - Added complete logger configuration section to agent.yml documentation + - Documented agentId field and derivation rules + - Updated storage documentation with CLI auto-configuration notes + - Added logger v2 architecture notes to core README + +- f9bca72: Add changeset for dropping defaultSessions from core layers. +- 8f373cc: Migrate server API to Hono framework with feature flag + - Migrated Express server to Hono with OpenAPI schema generation + - Added DEXTO_USE_HONO environment variable flag (default: false for backward compatibility) + - Fixed WebSocket test isolation by adding sessionId filtering + - Fixed logger context to pass structured objects instead of stringified JSON + - Fixed CI workflow for OpenAPI docs synchronization + - Updated documentation links and fixed broken API references + +- f28ad7e: Migrate webUI to use client-sdk, add agents.md file to webui,improve types in apis for consumption +- a35a256: Migrate from WebSocket to Server-Sent Events (SSE) for real-time streaming + - Replace WebSocket with SSE for message streaming via new `/api/message-stream` endpoint + - Refactor approval system from event-based providers to simpler handler pattern + - Add new APIs for session approval + - Move session title generation to a separate API + - Add `ApprovalCoordinator` for multi-client SSE routing with sessionId mapping + - Add stream and generate methods to DextoAgent and integ tests for itq= + +- cc49f06: Added comprehensive support for A2A protocol +- a154ae0: UI refactor with TanStack Query, new agent management package, and Hono as default server + + **Server:** + - Make Hono the default API server (use `DEXTO_USE_EXPRESS=true` env var to use Express) + - Fix agentId propagation to Hono server for correct agent name display + - Fix circular reference crashes in error logging by using structured logger context + + **WebUI:** + - Integrate TanStack Query for server state management with automatic caching and invalidation + - Add centralized query key factory and API client with structured error handling + - Replace manual data fetching with TanStack Query hooks across all components + - Add Zustand for client-side persistent state (recent agents in localStorage) + - Add keyboard shortcuts support with react-hotkeys-hook + - Add optimistic updates for session management via WebSocket events + - Fix Dialog auto-close bug in CreateMemoryModal + - Add defensive null handling in MemoryPanel + - Standardize Prettier formatting (single quotes, 4-space indentation) + + **Agent Management:** + - Add `@dexto/agent-management` package for centralized agent configuration management + - Extract agent registry, preferences, and path utilities into dedicated package + + **Internal:** + - Improve build orchestration and fix dependency imports + - Add `@dexto/agent-management` to global CLI installation + +- 5a26bdf: Update hono server to chain apis to keep type info, update client sdk to be fully typed +- ac649fd: Fix error handling and UI bugs, add gpt-5.1, gemini-3 +- Updated dependencies [c1e814f] +- Updated dependencies [f9bca72] +- Updated dependencies [c0a10cd] +- Updated dependencies [81598b5] +- Updated dependencies [4c90ffe] +- Updated dependencies [1a20506] +- Updated dependencies [8f373cc] +- Updated dependencies [f28ad7e] +- Updated dependencies [4dd4998] +- Updated dependencies [5e27806] +- Updated dependencies [a35a256] +- Updated dependencies [0fa6ef5] +- Updated dependencies [e2fb5f8] +- Updated dependencies [a154ae0] +- Updated dependencies [ac649fd] + - @dexto/agent-management@1.2.5 + - @dexto/core@1.2.5 diff --git a/dexto/packages/server/SECURITY.md b/dexto/packages/server/SECURITY.md new file mode 100644 index 00000000..1f7dd9f0 --- /dev/null +++ b/dexto/packages/server/SECURITY.md @@ -0,0 +1,225 @@ +# Dexto Server Security + +## 🔒 Authentication Overview + +The Dexto server implements **API key authentication** to protect against unauthorized access. + +## Configuration + +### Environment Variables + +```bash +# Required for production security +DEXTO_SERVER_API_KEY=your-secret-api-key-here + +# Optional: Enable production mode (requires API key) +NODE_ENV=production + +# Optional: Explicitly require auth even in development +DEXTO_SERVER_REQUIRE_AUTH=true +``` + +### Security Modes + +| Mode | Environment | Auth Required | Notes | +|------|-------------|---------------|-------| +| **Development (default)** | No env vars | ❌ No | Default mode - safe for local dev | +| **Production** | `NODE_ENV=production` + `DEXTO_SERVER_API_KEY` | ✅ Yes | Requires API key authentication | +| **Explicit Auth** | `DEXTO_SERVER_REQUIRE_AUTH=true` + `DEXTO_SERVER_API_KEY` | ✅ Yes | Force auth in any environment | + +## Usage + +### Client Authentication + +**HTTP Requests:** +```bash +curl -H "Authorization: Bearer your-api-key" \ + http://localhost:3000/api/llm/current +``` + +**JavaScript Fetch:** +```javascript +fetch('http://localhost:3000/api/message', { + method: 'POST', + headers: { + 'Authorization': 'Bearer your-api-key', + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ message: 'Hello' }) +}); +``` + +### Public Routes (No Auth Required) + +These routes are always accessible: +- `GET /health` - Health check +- `GET /.well-known/agent-card.json` - A2A agent discovery +- `GET /openapi.json` - API documentation + +## Security Best Practices + +### ✅ DO: + +1. **Set DEXTO_SERVER_API_KEY in production** + ```bash + export DEXTO_SERVER_API_KEY=$(openssl rand -base64 32) + ``` + +2. **Use HTTPS in production** + - Deploy behind reverse proxy (nginx, Caddy, Cloudflare) + - Never send API keys over unencrypted HTTP + +3. **Rotate API keys regularly** + ```bash + # Generate new key + NEW_KEY=$(openssl rand -base64 32) + # Update environment variable + export DEXTO_SERVER_API_KEY=$NEW_KEY + # Restart server + ``` + +4. **Use environment-specific keys** + - Different keys for dev/staging/production + - Never commit keys to version control + +5. **Monitor unauthorized access attempts** + - Check logs for "Unauthorized API access attempt" warnings + - Set up alerts for repeated failures + +### ❌ DON'T: + +1. **Don't use weak or guessable API keys** + - ❌ `DEXTO_SERVER_API_KEY=password123` + - ❌ `DEXTO_SERVER_API_KEY=dexto` + - ✅ `DEXTO_SERVER_API_KEY=$(openssl rand -base64 32)` + +2. **Don't expose API keys in client-side code** + ```javascript + // ❌ NEVER DO THIS + const apiKey = 'sk-abc123...'; + fetch('/api/message', { headers: { 'Authorization': `Bearer ${apiKey}` }}); + ``` + +3. **Don't set DEXTO_SERVER_REQUIRE_AUTH=false in production** + - Only use for testing on isolated networks + +4. **Don't share API keys across environments** + - Each environment should have its own key + +## Development Workflow + +### Local Development (No Auth) + +```bash +# Start server in development mode +NODE_ENV=development npm start + +# Access from browser without auth +curl http://localhost:3000/api/llm/current +``` + +### Production Deployment + +```bash +# Generate secure API key +export DEXTO_SERVER_API_KEY=$(openssl rand -base64 32) + +# Start server in production mode +NODE_ENV=production npm start + +# All requests now require authentication +curl -H "Authorization: Bearer $DEXTO_SERVER_API_KEY" \ + https://api.example.com/api/llm/current +``` + +## Threat Model + +### Protected Against: + +- ✅ Unauthorized API access +- ✅ Unauthorized message sending +- ✅ Unauthorized configuration changes +- ✅ Unauthorized session/memory access +- ✅ Brute force attacks (when combined with rate limiting) + +### Not Protected Against (Additional Measures Needed): + +- ⚠️ DDoS attacks → Add rate limiting middleware +- ⚠️ API key leakage → Use secrets management (Vault, AWS Secrets Manager) +- ⚠️ Man-in-the-middle → Use HTTPS/TLS +- ⚠️ Insider threats → Implement audit logging + +## Additional Security Layers (Recommended) + +### 1. Rate Limiting + +```typescript +import { rateLimiter } from 'hono-rate-limiter'; + +app.use('*', rateLimiter({ + windowMs: 15 * 60 * 1000, // 15 minutes + max: 100, // limit each IP to 100 requests per windowMs +})); +``` + +### 2. IP Whitelisting + +```bash +# Add to your reverse proxy (nginx example) +location /api { + allow 10.0.0.0/8; + deny all; + proxy_pass http://localhost:3000; +} +``` + +### 3. Network Isolation + +- Deploy API server on private network +- Use VPN or private networking for access +- Firewall rules to restrict incoming connections + +## Logging and Monitoring + +The server logs authentication events: + +```log +# Successful auth (debug level) +Authorization successful for /api/llm/current + +# Failed auth (warning level) +⚠️ Unauthorized API access attempt + path: /api/message + hasKey: false + origin: https://malicious.com + userAgent: curl/7.81.0 +``` + +Set up monitoring for: +- Repeated 401 responses +- Unusual access patterns +- Requests from unexpected IPs/origins + +## FAQ + +**Q: Can I use the API without authentication in development?** +A: Yes, set `NODE_ENV=development` and access from localhost. + +**Q: How do I generate a secure API key?** +A: Use `openssl rand -base64 32` or a password manager. + +**Q: Can I use multiple API keys?** +A: Currently no. For multi-tenant scenarios, implement token-based auth with JWT. + +**Q: What if my API key is compromised?** +A: Generate a new key immediately and update all clients. + +**Q: Does SSE need authentication too?** +A: Yes, pass `Authorization: Bearer ` header when connecting to the event stream. + +**Q: Can I disable auth for specific routes?** +A: Public routes (/health, /.well-known/agent-card.json) are always accessible. To add more, modify `PUBLIC_ROUTES` in `middleware/auth.ts`. + +## Contact + +For security concerns or to report vulnerabilities, contact: security@dexto.dev diff --git a/dexto/packages/server/package.json b/dexto/packages/server/package.json new file mode 100644 index 00000000..5879537f --- /dev/null +++ b/dexto/packages/server/package.json @@ -0,0 +1,60 @@ +{ + "name": "@dexto/server", + "version": "1.5.6", + "private": false, + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.cjs" + }, + "./hono": { + "types": "./dist/hono/index.d.ts", + "import": "./dist/hono/index.js", + "require": "./dist/hono/index.cjs" + }, + "./hono/node": { + "types": "./dist/hono/node/index.d.ts", + "import": "./dist/hono/node/index.js", + "require": "./dist/hono/node/index.cjs" + }, + "./package.json": "./package.json" + }, + "dependencies": { + "@dexto/agent-management": "workspace:*", + "@dexto/core": "workspace:*", + "@dexto/image-local": "workspace:*", + "@hono/node-server": "1.19.5", + "@hono/zod-openapi": "^0.19.1", + "hono": "^4.6.8", + "ws": "^8.18.1", + "yaml": "^2.7.1", + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "scripts": { + "build": "cross-env NODE_OPTIONS='--max-old-space-size=8192' tsup && cross-env NODE_OPTIONS='--max-old-space-size=8192' tsc -p tsconfig.json --emitDeclarationOnly", + "dev": "tsup --watch", + "typecheck": "tsc -p tsconfig.typecheck.json --noEmit", + "lint": "eslint . --ext .ts", + "test": "vitest run" + }, + "files": [ + "dist", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "devDependencies": { + "@types/ws": "^8.5.11", + "zod": "^3.25.0" + }, + "peerDependencies": { + "zod": "^3.25.0" + } +} diff --git a/dexto/packages/server/src/a2a/adapters/index.ts b/dexto/packages/server/src/a2a/adapters/index.ts new file mode 100644 index 00000000..7ace06ec --- /dev/null +++ b/dexto/packages/server/src/a2a/adapters/index.ts @@ -0,0 +1,10 @@ +/** + * A2A Protocol Adapters + * + * Server-layer adapters for converting between A2A protocol format + * and Dexto's internal representation. + */ + +export { TaskView, createTaskView } from './task-view.js'; +export { a2aToInternalMessage, internalToA2AMessage, internalMessagesToA2A } from './message.js'; +export { deriveTaskState, deriveTaskStateFromA2A } from './state.js'; diff --git a/dexto/packages/server/src/a2a/adapters/message.ts b/dexto/packages/server/src/a2a/adapters/message.ts new file mode 100644 index 00000000..90bc7875 --- /dev/null +++ b/dexto/packages/server/src/a2a/adapters/message.ts @@ -0,0 +1,272 @@ +/** + * A2A Message Format Converters + * + * Bidirectional conversion between A2A protocol message format + * and Dexto's internal message format. + * + * These converters live at the server boundary, translating between + * wire format (A2A) and internal format (DextoAgent). + */ + +import type { InternalMessage } from '@dexto/core'; +import type { Message, Part, MessageRole, ConvertedMessage } from '../types.js'; +import { randomUUID } from 'crypto'; + +/** + * Convert A2A message to internal format for agent.run(). + * + * Extracts text, image, and file from A2A parts array. + * agent.run() expects these as separate parameters. + * + * @param a2aMsg A2A protocol message + * @returns Converted message parts for agent.run() + */ +export function a2aToInternalMessage(a2aMsg: Message): ConvertedMessage { + let text = ''; + let image: ConvertedMessage['image'] | undefined; + let file: ConvertedMessage['file'] | undefined; + + for (const part of a2aMsg.parts) { + switch (part.kind) { + case 'text': + text += (text ? ' ' : '') + part.text; + break; + + case 'file': { + // Determine if this is an image or general file + const fileData = part.file; + const mimeType = fileData.mimeType || ''; + const isImage = mimeType.startsWith('image/'); + + if (isImage && !image) { + // Treat as image (agent.run() supports one image) + const data = 'bytes' in fileData ? fileData.bytes : fileData.uri; + image = { + image: data, + mimeType: mimeType, + }; + } else if (!file) { + // Take first file only (agent.run() supports one file) + const data = 'bytes' in fileData ? fileData.bytes : fileData.uri; + const fileObj: { data: string; mimeType: string; filename?: string } = { + data: data, + mimeType: mimeType, + }; + if (fileData.name) { + fileObj.filename = fileData.name; + } + file = fileObj; + } + break; + } + + case 'data': + // Convert structured data to JSON text + text += (text ? '\n' : '') + JSON.stringify(part.data, null, 2); + break; + } + } + + return { text, image, file }; +} + +/** + * Convert internal message to A2A format. + * + * Maps Dexto's internal message structure to A2A protocol format. + * + * Role mapping: + * - 'user' → 'user' + * - 'assistant' → 'agent' + * - 'system' → filtered out (not part of A2A conversation) + * - 'tool' → 'agent' (tool results presented as agent responses) + * + * @param msg Internal message from session history + * @param taskId Optional task ID to associate message with + * @param contextId Optional context ID to associate message with + * @returns A2A protocol message or null if message should be filtered + */ +export function internalToA2AMessage( + msg: InternalMessage, + taskId?: string, + contextId?: string +): Message | null { + // Filter out system messages (internal context, not part of A2A conversation) + if (msg.role === 'system') { + return null; + } + + // Map role + const role: MessageRole = msg.role === 'user' ? 'user' : 'agent'; + + // Convert content to parts + const parts: Part[] = []; + + if (typeof msg.content === 'string') { + // Simple text content + if (msg.content) { + parts.push({ kind: 'text', text: msg.content }); + } + } else if (msg.content === null) { + // Null content (tool-only messages) - skip for A2A + // These are internal details, not part of user-facing conversation + } else if (Array.isArray(msg.content)) { + // Multi-part content + for (const part of msg.content) { + switch (part.type) { + case 'text': + parts.push({ kind: 'text', text: part.text }); + break; + + case 'image': { + const imageData = part.image; + const mimeType = part.mimeType || 'image/png'; + + // Convert different input types to base64 or URL + let fileObj: any; + if ( + imageData instanceof URL || + (typeof imageData === 'string' && imageData.startsWith('http')) + ) { + // URL reference + fileObj = { + uri: imageData.toString(), + mimeType, + }; + } else if (Buffer.isBuffer(imageData)) { + // Buffer -> base64 + fileObj = { + bytes: imageData.toString('base64'), + mimeType, + }; + } else if (imageData instanceof Uint8Array) { + // Uint8Array -> base64 + fileObj = { + bytes: Buffer.from(imageData).toString('base64'), + mimeType, + }; + } else if (imageData instanceof ArrayBuffer) { + // ArrayBuffer -> base64 + fileObj = { + bytes: Buffer.from(imageData).toString('base64'), + mimeType, + }; + } else if (typeof imageData === 'string') { + // Assume already base64 if string but not a URL + fileObj = { + bytes: imageData, + mimeType, + }; + } + + if (fileObj) { + parts.push({ + kind: 'file', + file: fileObj, + }); + } + break; + } + + case 'file': { + const fileData = part.data; + const mimeType = part.mimeType; + + // Convert different input types to base64 or URL + let fileObj: any; + if ( + fileData instanceof URL || + (typeof fileData === 'string' && fileData.startsWith('http')) + ) { + // URL reference + fileObj = { + uri: fileData.toString(), + mimeType, + }; + } else if (Buffer.isBuffer(fileData)) { + // Buffer -> base64 + fileObj = { + bytes: fileData.toString('base64'), + mimeType, + }; + } else if (fileData instanceof Uint8Array) { + // Uint8Array -> base64 + fileObj = { + bytes: Buffer.from(fileData).toString('base64'), + mimeType, + }; + } else if (fileData instanceof ArrayBuffer) { + // ArrayBuffer -> base64 + fileObj = { + bytes: Buffer.from(fileData).toString('base64'), + mimeType, + }; + } else if (typeof fileData === 'string') { + // Assume already base64 if string but not a URL + fileObj = { + bytes: fileData, + mimeType, + }; + } + + if (fileObj) { + // Add filename if present + if (part.filename) { + fileObj.name = part.filename; + } + + parts.push({ + kind: 'file', + file: fileObj, + }); + } + break; + } + } + } + } + + // If no parts, return null (don't include empty messages in A2A) + if (parts.length === 0) { + return null; + } + + const message: Message = { + role, + parts, + messageId: randomUUID(), + kind: 'message', + }; + + if (taskId) message.taskId = taskId; + if (contextId) message.contextId = contextId; + + return message; +} + +/** + * Convert array of internal messages to A2A messages. + * + * Filters out system messages and empty messages. + * + * @param messages Internal messages from session history + * @param taskId Optional task ID to associate messages with + * @param contextId Optional context ID to associate messages with + * @returns Array of A2A protocol messages + */ +export function internalMessagesToA2A( + messages: InternalMessage[], + taskId?: string, + contextId?: string +): Message[] { + const a2aMessages: Message[] = []; + + for (const msg of messages) { + const a2aMsg = internalToA2AMessage(msg, taskId, contextId); + if (a2aMsg !== null) { + a2aMessages.push(a2aMsg); + } + } + + return a2aMessages; +} diff --git a/dexto/packages/server/src/a2a/adapters/state.ts b/dexto/packages/server/src/a2a/adapters/state.ts new file mode 100644 index 00000000..c12b26a3 --- /dev/null +++ b/dexto/packages/server/src/a2a/adapters/state.ts @@ -0,0 +1,77 @@ +/** + * A2A Task State Derivation + * + * Derives A2A task state from Dexto session state. + * Tasks don't have their own state - state is computed from session history. + */ + +import type { InternalMessage } from '@dexto/core'; +import type { TaskState, Message } from '../types.js'; + +/** + * Derive task state from session message history. + * + * Logic per A2A spec: + * - submitted: Task has been submitted (no messages yet or only user message) + * - working: Agent is processing the task + * - completed: Task completed successfully (has complete exchange) + * - failed: Session encountered an error (would need error tracking) + * - canceled: Session was explicitly cancelled (would need cancellation tracking) + * + * Note: We derive from message patterns, not explicit state tracking. + * This keeps tasks as pure views over sessions. + * + * @param messages Session message history + * @returns Derived task state + */ +export function deriveTaskState(messages: InternalMessage[]): TaskState { + // Empty session = submitted task + if (messages.length === 0) { + return 'submitted'; + } + + // Check for user and assistant messages + const hasUserMessage = messages.some((m) => m.role === 'user'); + const hasAssistantMessage = messages.some((m) => m.role === 'assistant'); + + // Complete exchange = completed task + if (hasUserMessage && hasAssistantMessage) { + return 'completed'; + } + + // User message without response = working task + if (hasUserMessage && !hasAssistantMessage) { + return 'working'; + } + + // Edge case: assistant message without user (shouldn't happen normally) + return 'submitted'; +} + +/** + * Derive task state from A2A messages (already converted). + * + * This is a convenience function when you already have A2A messages + * and don't want to go back to internal format. + * + * @param messages A2A protocol messages + * @returns Derived task state + */ +export function deriveTaskStateFromA2A(messages: Message[]): TaskState { + if (messages.length === 0) { + return 'submitted'; + } + + const hasUserMessage = messages.some((m) => m.role === 'user'); + const hasAgentMessage = messages.some((m) => m.role === 'agent'); + + if (hasUserMessage && hasAgentMessage) { + return 'completed'; + } + + if (hasUserMessage && !hasAgentMessage) { + return 'working'; + } + + return 'submitted'; +} diff --git a/dexto/packages/server/src/a2a/adapters/task-view.ts b/dexto/packages/server/src/a2a/adapters/task-view.ts new file mode 100644 index 00000000..ea398f64 --- /dev/null +++ b/dexto/packages/server/src/a2a/adapters/task-view.ts @@ -0,0 +1,103 @@ +/** + * A2A TaskView Adapter + * + * Wraps a Dexto ChatSession to present it as an A2A Task. + * This is a pure adapter - no storage, no persistence, just a view. + * + * Key principle: taskId === sessionId + */ + +import type { ChatSession } from '@dexto/core'; +import type { Task, TaskStatus } from '../types.js'; +import { internalMessagesToA2A } from './message.js'; +import { deriveTaskState } from './state.js'; + +/** + * TaskView wraps a ChatSession to provide A2A-compliant task interface. + * + * This is a lightweight adapter that converts session state to A2A format + * on-demand. No state is cached or stored. + * + * Usage: + * ```typescript + * const session = await agent.createSession(taskId); + * const taskView = new TaskView(session); + * const task = await taskView.toA2ATask(); + * ``` + */ +export class TaskView { + constructor(private session: ChatSession) {} + + /** + * Convert the wrapped session to an A2A Task. + * + * This reads the session history and converts it to A2A format. + * State is derived from message patterns, not stored separately. + * + * @returns A2A protocol task structure + */ + async toA2ATask(): Promise { + // Get session history + const history = await this.session.getHistory(); + + // Convert internal messages to A2A format + const a2aMessages = internalMessagesToA2A(history, this.session.id, this.session.id); + + // Derive task state from session + const state = deriveTaskState(history); + + // Create TaskStatus object per A2A spec + const status: TaskStatus = { + state, + timestamp: new Date().toISOString(), + }; + + // Construct A2A task + const task: Task = { + id: this.session.id, // taskId === sessionId + contextId: this.session.id, // For now, contextId === taskId (could be enhanced for multi-task contexts) + status, + history: a2aMessages, + kind: 'task', + metadata: { + dexto: { + sessionId: this.session.id, + }, + }, + }; + + return task; + } + + /** + * Get the underlying session ID. + * Since taskId === sessionId, this is the same as the task ID. + */ + get sessionId(): string { + return this.session.id; + } + + /** + * Get the underlying session (for advanced use). + */ + get session_(): ChatSession { + return this.session; + } +} + +/** + * Create a TaskView from a session ID and agent. + * + * Convenience factory function. + * + * @param sessionId Session/Task ID + * @param agent DextoAgent instance + * @returns TaskView wrapper + */ +export async function createTaskView( + sessionId: string, + agent: { createSession(id: string): Promise } +): Promise { + const session = await agent.createSession(sessionId); + return new TaskView(session); +} diff --git a/dexto/packages/server/src/a2a/index.ts b/dexto/packages/server/src/a2a/index.ts new file mode 100644 index 00000000..68e39631 --- /dev/null +++ b/dexto/packages/server/src/a2a/index.ts @@ -0,0 +1,62 @@ +/** + * A2A Protocol Implementation + * + * Server-layer implementation of A2A Protocol v0.3.0. + * Exposes DextoAgent capabilities through A2A-compliant interfaces. + * + * Specification: https://a2a-protocol.org/latest/specification + * + * @module a2a + */ + +// Type definitions (A2A Protocol v0.3.0) +export type { + Task, + TaskState, + TaskStatus, + Message, + MessageRole, + Part, + TextPart, + FilePart, + DataPart, + FileWithBytes, + FileWithUri, + Artifact, + TaskStatusUpdateEvent, + TaskArtifactUpdateEvent, + MessageSendParams, + MessageSendConfiguration, + TaskQueryParams, + ListTasksParams, + ListTasksResult, + TaskIdParams, + ConvertedMessage, +} from './types.js'; + +// Protocol adapters +export { + TaskView, + createTaskView, + a2aToInternalMessage, + internalToA2AMessage, + internalMessagesToA2A, + deriveTaskState, + deriveTaskStateFromA2A, +} from './adapters/index.js'; + +// JSON-RPC transport +export { + JsonRpcServer, + A2AMethodHandlers, + JsonRpcErrorCode, + isJsonRpcError, + isJsonRpcSuccess, +} from './jsonrpc/index.js'; +export type { + JsonRpcRequest, + JsonRpcResponse, + JsonRpcError, + JsonRpcMethodHandler, + JsonRpcServerOptions, +} from './jsonrpc/index.js'; diff --git a/dexto/packages/server/src/a2a/jsonrpc/index.ts b/dexto/packages/server/src/a2a/jsonrpc/index.ts new file mode 100644 index 00000000..5d86f16b --- /dev/null +++ b/dexto/packages/server/src/a2a/jsonrpc/index.ts @@ -0,0 +1,19 @@ +/** + * A2A JSON-RPC 2.0 Implementation + * + * JSON-RPC transport layer for A2A Protocol. + */ + +export { JsonRpcServer } from './server.js'; +export type { JsonRpcMethodHandler, JsonRpcServerOptions } from './server.js'; +export { A2AMethodHandlers } from './methods.js'; +export type { + JsonRpcRequest, + JsonRpcResponse, + JsonRpcSuccessResponse, + JsonRpcErrorResponse, + JsonRpcError, + JsonRpcBatchRequest, + JsonRpcBatchResponse, +} from './types.js'; +export { JsonRpcErrorCode, isJsonRpcError, isJsonRpcSuccess } from './types.js'; diff --git a/dexto/packages/server/src/a2a/jsonrpc/methods.ts b/dexto/packages/server/src/a2a/jsonrpc/methods.ts new file mode 100644 index 00000000..a66e74f1 --- /dev/null +++ b/dexto/packages/server/src/a2a/jsonrpc/methods.ts @@ -0,0 +1,245 @@ +/** + * A2A Protocol JSON-RPC Method Handlers + * + * Implements A2A Protocol v0.3.0 RPC methods by calling DextoAgent. + * These are thin wrappers that translate between A2A protocol and DextoAgent API. + * + * Method names per spec: + * - message/send - Send a message to the agent + * - message/stream - Send a message with streaming response + * - tasks/get - Retrieve a specific task + * - tasks/list - List tasks with optional filtering + * - tasks/cancel - Cancel an in-progress task + */ + +import type { DextoAgent } from '@dexto/core'; +import type { + Task, + Message, + MessageSendParams, + TaskQueryParams, + ListTasksParams, + ListTasksResult, + TaskIdParams, +} from '../types.js'; +import { TaskView } from '../adapters/task-view.js'; +import { a2aToInternalMessage } from '../adapters/message.js'; + +/** + * A2A Method Handlers + * + * Implements all A2A Protocol JSON-RPC methods. + * Each method: + * 1. Validates params + * 2. Calls DextoAgent methods + * 3. Converts response to A2A format using TaskView + * + * Usage: + * ```typescript + * const handlers = new A2AMethodHandlers(agent); + * const server = new JsonRpcServer({ + * methods: handlers.getMethods() + * }); + * ``` + */ +export class A2AMethodHandlers { + constructor(private agent: DextoAgent) {} + + /** + * message/send - Send a message to the agent + * + * This is the primary method for interacting with an agent. + * Creates a task if taskId not provided in message, or adds to existing task. + * + * @param params Message send parameters + * @returns Task or Message depending on configuration.blocking + */ + async messageSend(params: MessageSendParams): Promise { + if (!params?.message) { + throw new Error('message is required'); + } + + const { message } = params; + + // Extract taskId from message (or generate new one) + const taskId = message.taskId; + + // Create or get session + const session = await this.agent.createSession(taskId); + + // Convert A2A message to internal format and run + const { text, image, file } = a2aToInternalMessage(message); + await this.agent.run(text, image, file, session.id); + + // Return task view + const taskView = new TaskView(session); + const task = await taskView.toA2ATask(); + + // If blocking=false, return just the message (non-blocking) + // For now, always return task (blocking behavior) + // TODO: Implement non-blocking mode that returns Message + return task; + } + + /** + * tasks/get - Retrieve a task by ID + * + * @param params Parameters containing task ID + * @returns Task details + * @throws Error if task not found + */ + async tasksGet(params: TaskQueryParams): Promise { + if (!params?.id) { + throw new Error('id is required'); + } + + // Check if session exists (don't create if not found) + const session = await this.agent.getSession(params.id); + if (!session) { + throw new Error(`Task not found: ${params.id}`); + } + + // Convert to task view + const taskView = new TaskView(session); + return await taskView.toA2ATask(); + } + + /** + * tasks/list - List all tasks (optional filters) + * + * Note: This implementation loads all sessions, applies filters, then paginates. + * For production with many sessions, consider filtering at the session manager level. + * + * @param params Optional filter parameters + * @returns List of tasks with pagination info + */ + async tasksList(params?: ListTasksParams): Promise { + // Get all session IDs + const sessionIds = await this.agent.listSessions(); + + // Convert each session to task view and apply filters + const allTasks: Task[] = []; + for (const sessionId of sessionIds) { + // Use getSession to only retrieve existing sessions (don't create) + const session = await this.agent.getSession(sessionId); + if (!session) { + continue; // Skip if session no longer exists + } + + const taskView = new TaskView(session); + const task = await taskView.toA2ATask(); + + // Filter by status if provided + if (params?.status && task.status.state !== params.status) { + continue; + } + + // Filter by contextId if provided + if (params?.contextId && task.contextId !== params.contextId) { + continue; + } + + allTasks.push(task); + } + + // Apply pagination after filtering + const pageSize = Math.min(params?.pageSize ?? 50, 100); + const offset = 0; // TODO: Implement proper pagination with pageToken + const paginatedTasks = allTasks.slice(offset, offset + pageSize); + + return { + tasks: paginatedTasks, + totalSize: allTasks.length, // Total matching tasks before pagination + pageSize, + nextPageToken: '', // TODO: Implement pagination tokens + }; + } + + /** + * tasks/cancel - Cancel a running task + * + * @param params Parameters containing task ID + * @returns Updated task (in canceled state) + * @throws Error if task not found + */ + async tasksCancel(params: TaskIdParams): Promise { + if (!params?.id) { + throw new Error('id is required'); + } + + // Check if session exists (don't create if not found) + const session = await this.agent.getSession(params.id); + if (!session) { + throw new Error(`Task not found: ${params.id}`); + } + + // Cancel the session + session.cancel(); + + // Return updated task view + const taskView = new TaskView(session); + return await taskView.toA2ATask(); + } + + /** + * message/stream - Send a message with streaming response + * + * This is a streaming variant of message/send. Instead of returning a complete Task, + * it returns a stream of TaskStatusUpdateEvent and TaskArtifactUpdateEvent as the + * agent processes the message. + * + * **ARCHITECTURE NOTE**: This method is designed as a lightweight handler that returns + * a taskId immediately. The actual message processing happens at the transport layer: + * + * - **JSON-RPC Transport** (packages/server/src/hono/routes/a2a-jsonrpc.ts:72-112): + * The route intercepts 'message/stream' requests BEFORE calling this handler, + * processes the message directly (lines 96-99), and returns an SSE stream. + * This handler is registered but never actually invoked for JSON-RPC streaming. + * + * - **REST Transport** (packages/server/src/hono/routes/a2a-tasks.ts:206-244): + * Similar pattern - route processes message and returns SSE stream directly. + * + * This design separates concerns: + * - Handler provides taskId for API compatibility + * - Transport layer manages SSE streaming and message processing + * - Event bus broadcasts updates to connected SSE clients + * + * @param params Message send parameters (same as message/send) + * @returns Task ID for streaming (transport layer handles actual SSE stream and message processing) + */ + async messageStream(params: MessageSendParams): Promise<{ taskId: string }> { + if (!params?.message) { + throw new Error('message is required'); + } + + const { message } = params; + + // Extract taskId from message (or generate new one) + const taskId = message.taskId; + + // Create or get session + const session = await this.agent.createSession(taskId); + + // Return task ID immediately - the transport layer will handle + // setting up the SSE stream and calling agent.run() with streaming + // See architecture note above for where message processing occurs + return { taskId: session.id }; + } + + /** + * Get all method handlers as a Record for JsonRpcServer + * + * Returns methods with A2A-compliant names (slash notation). + * + * @returns Map of method names to handlers + */ + getMethods(): Record Promise> { + return { + 'message/send': this.messageSend.bind(this), + 'message/stream': this.messageStream.bind(this), + 'tasks/get': this.tasksGet.bind(this), + 'tasks/list': this.tasksList.bind(this), + 'tasks/cancel': this.tasksCancel.bind(this), + }; + } +} diff --git a/dexto/packages/server/src/a2a/jsonrpc/server.ts b/dexto/packages/server/src/a2a/jsonrpc/server.ts new file mode 100644 index 00000000..23f64b99 --- /dev/null +++ b/dexto/packages/server/src/a2a/jsonrpc/server.ts @@ -0,0 +1,271 @@ +/** + * JSON-RPC 2.0 Server + * + * Handles JSON-RPC 2.0 request parsing, method dispatch, and response formatting. + * Implements the full JSON-RPC 2.0 specification including batch requests. + */ + +import type { + JsonRpcRequest, + JsonRpcResponse, + JsonRpcBatchRequest, + JsonRpcBatchResponse, + JsonRpcError, +} from './types.js'; +import { JsonRpcErrorCode } from './types.js'; + +/** + * Method handler function type + */ +export type JsonRpcMethodHandler = (params: any) => Promise; + +/** + * JSON-RPC 2.0 Server Options + */ +export interface JsonRpcServerOptions { + /** Method handlers map */ + methods: Record; + /** Optional error handler */ + onError?: (error: Error, request?: JsonRpcRequest) => void; +} + +/** + * JSON-RPC 2.0 Server + * + * Parses JSON-RPC requests, dispatches to handlers, and formats responses. + * + * Usage: + * ```typescript + * const server = new JsonRpcServer({ + * methods: { + * 'agent.createTask': async (params) => { ... }, + * 'agent.getTask': async (params) => { ... }, + * } + * }); + * + * const response = await server.handle(request); + * ``` + */ +export class JsonRpcServer { + private methods: Record; + private onError: ((error: Error, request?: JsonRpcRequest) => void) | undefined; + + constructor(options: JsonRpcServerOptions) { + this.methods = options.methods; + this.onError = options.onError; + } + + /** + * Handle a JSON-RPC request (single or batch). + * + * @param request Single request or batch array + * @returns Single response, batch array, or undefined for notifications + */ + async handle( + request: JsonRpcRequest | JsonRpcBatchRequest + ): Promise { + // Handle batch requests + if (Array.isArray(request)) { + return await this.handleBatch(request); + } + + // Handle single request + return await this.handleSingle(request); + } + + /** + * Handle a batch of JSON-RPC requests. + * + * Processes all requests in parallel per JSON-RPC 2.0 spec. + * + * @param requests Array of requests + * @returns Array of responses, or undefined if all were notifications + */ + private async handleBatch( + requests: JsonRpcBatchRequest + ): Promise { + // Empty batch is an error + if (requests.length === 0) { + return [ + this.createErrorResponse(null, JsonRpcErrorCode.INVALID_REQUEST, 'Empty batch'), + ]; + } + + // Process all requests in parallel + const responses = await Promise.all(requests.map((req) => this.handleSingle(req))); + + // Filter out notification responses (undefined) + const validResponses = responses.filter((res): res is JsonRpcResponse => res !== undefined); + + // Per JSON-RPC 2.0 spec: if all requests were notifications, return undefined + if (validResponses.length === 0) { + return undefined; + } + + return validResponses; + } + + /** + * Handle a single JSON-RPC request. + * + * @param request JSON-RPC request object + * @returns JSON-RPC response object, or undefined for notifications + */ + private async handleSingle(request: JsonRpcRequest): Promise { + try { + // Validate JSON-RPC version + if (request.jsonrpc !== '2.0') { + // Notifications must not receive any response, even on error + if (request.id === undefined) { + return undefined; + } + return this.createErrorResponse( + request.id ?? null, + JsonRpcErrorCode.INVALID_REQUEST, + 'Invalid JSON-RPC version (must be "2.0")' + ); + } + + // Validate method exists + if (typeof request.method !== 'string') { + // Notifications must not receive any response, even on error + if (request.id === undefined) { + return undefined; + } + return this.createErrorResponse( + request.id ?? null, + JsonRpcErrorCode.INVALID_REQUEST, + 'Method must be a string' + ); + } + + // Check if method exists + const handler = this.methods[request.method]; + if (!handler) { + // Notifications must not receive any response, even on error + if (request.id === undefined) { + return undefined; + } + return this.createErrorResponse( + request.id ?? null, + JsonRpcErrorCode.METHOD_NOT_FOUND, + `Method not found: ${request.method}` + ); + } + + // Execute method handler + try { + const result = await handler(request.params); + + // Notifications (id is undefined) don't get responses + if (request.id === undefined) { + return undefined; + } + + return this.createSuccessResponse(request.id ?? null, result); + } catch (error) { + // Call error handler if provided (always log server-side) + if (this.onError) { + this.onError( + error instanceof Error ? error : new Error(String(error)), + request + ); + } + + // Notifications must not receive any response, even on error + if (request.id === undefined) { + return undefined; + } + + // Method execution error - return error response + const errorMessage = error instanceof Error ? error.message : String(error); + // Don't leak stack traces to clients (already logged via onError) + const errorData = error instanceof Error ? { name: error.name } : undefined; + + return this.createErrorResponse( + request.id ?? null, + JsonRpcErrorCode.INTERNAL_ERROR, + errorMessage, + errorData + ); + } + } catch (error) { + // Request parsing/validation error - if notification, still no response + if (request.id === undefined) { + return undefined; + } + const errorMessage = error instanceof Error ? error.message : String(error); + return this.createErrorResponse(null, JsonRpcErrorCode.INVALID_REQUEST, errorMessage); + } + } + + /** + * Create a success response. + */ + private createSuccessResponse(id: string | number | null, result: any): JsonRpcResponse { + return { + jsonrpc: '2.0', + result, + id, + }; + } + + /** + * Create an error response. + */ + private createErrorResponse( + id: string | number | null, + code: number, + message: string, + data?: any + ): JsonRpcResponse { + const error: JsonRpcError = { code, message }; + if (data !== undefined) { + error.data = data; + } + + return { + jsonrpc: '2.0', + error, + id, + }; + } + + /** + * Register a new method handler. + * + * @param method Method name + * @param handler Handler function + */ + registerMethod(method: string, handler: JsonRpcMethodHandler): void { + this.methods[method] = handler; + } + + /** + * Unregister a method handler. + * + * @param method Method name + */ + unregisterMethod(method: string): void { + delete this.methods[method]; + } + + /** + * Check if a method is registered. + * + * @param method Method name + * @returns True if method exists + */ + hasMethod(method: string): boolean { + return method in this.methods; + } + + /** + * Get list of registered method names. + * + * @returns Array of method names + */ + getMethods(): string[] { + return Object.keys(this.methods); + } +} diff --git a/dexto/packages/server/src/a2a/jsonrpc/types.ts b/dexto/packages/server/src/a2a/jsonrpc/types.ts new file mode 100644 index 00000000..12724b03 --- /dev/null +++ b/dexto/packages/server/src/a2a/jsonrpc/types.ts @@ -0,0 +1,104 @@ +/** + * JSON-RPC 2.0 Type Definitions + * + * Implements JSON-RPC 2.0 specification for A2A Protocol transport. + * @see https://www.jsonrpc.org/specification + */ + +/** + * JSON-RPC 2.0 Request + */ +export interface JsonRpcRequest { + /** JSON-RPC version (must be "2.0") */ + jsonrpc: '2.0'; + /** Method name to invoke */ + method: string; + /** Method parameters (optional) */ + params?: any; + /** Request ID (can be string, number, or null for notifications) */ + id?: string | number | null; +} + +/** + * JSON-RPC 2.0 Response (Success) + */ +export interface JsonRpcSuccessResponse { + /** JSON-RPC version (must be "2.0") */ + jsonrpc: '2.0'; + /** Result of the method invocation */ + result: any; + /** Request ID (matches request) */ + id: string | number | null; +} + +/** + * JSON-RPC 2.0 Response (Error) + */ +export interface JsonRpcErrorResponse { + /** JSON-RPC version (must be "2.0") */ + jsonrpc: '2.0'; + /** Error object */ + error: JsonRpcError; + /** Request ID (matches request, or null if ID couldn't be determined) */ + id: string | number | null; +} + +/** + * JSON-RPC 2.0 Error Object + */ +export interface JsonRpcError { + /** Error code (integer) */ + code: number; + /** Error message (short description) */ + message: string; + /** Optional additional error data */ + data?: any; +} + +/** + * Union type for JSON-RPC responses + */ +export type JsonRpcResponse = JsonRpcSuccessResponse | JsonRpcErrorResponse; + +/** + * JSON-RPC 2.0 Batch Request + */ +export type JsonRpcBatchRequest = JsonRpcRequest[]; + +/** + * JSON-RPC 2.0 Batch Response + */ +export type JsonRpcBatchResponse = JsonRpcResponse[]; + +/** + * Standard JSON-RPC 2.0 Error Codes + */ +export enum JsonRpcErrorCode { + /** Invalid JSON was received by the server */ + PARSE_ERROR = -32700, + /** The JSON sent is not a valid Request object */ + INVALID_REQUEST = -32600, + /** The method does not exist / is not available */ + METHOD_NOT_FOUND = -32601, + /** Invalid method parameter(s) */ + INVALID_PARAMS = -32602, + /** Internal JSON-RPC error */ + INTERNAL_ERROR = -32603, + /** Reserved for implementation-defined server-errors (-32000 to -32099) */ + SERVER_ERROR_START = -32099, + SERVER_ERROR_END = -32000, +} + +/** + * Type guard to check if response is an error + */ +export function isJsonRpcError(response: JsonRpcResponse): response is JsonRpcErrorResponse { + return 'error' in response; +} + +/** + * Type guard to check if response is success + */ +export function isJsonRpcSuccess(response: JsonRpcResponse): response is JsonRpcSuccessResponse { + return 'result' in response; +} diff --git a/dexto/packages/server/src/a2a/types.ts b/dexto/packages/server/src/a2a/types.ts new file mode 100644 index 00000000..a303be93 --- /dev/null +++ b/dexto/packages/server/src/a2a/types.ts @@ -0,0 +1,262 @@ +/** + * TODO: fetch from a2a sdk to avoid drift over time + * A2A Protocol Type Definitions + * + * Type definitions compliant with A2A Protocol v0.3.0 specification. + * Based on: https://a2a-protocol.org/latest/specification + * + * @module a2a/types + */ + +/** + * Task state per A2A Protocol specification. + * + * States: + * - submitted: Task has been submitted + * - working: Task is being processed + * - input-required: Task needs user input + * - completed: Task completed successfully + * - canceled: Task was canceled + * - failed: Task failed with error + * - rejected: Task was rejected + * - auth-required: Authentication required + * - unknown: State is unknown + */ +export type TaskState = + | 'submitted' + | 'working' + | 'input-required' + | 'completed' + | 'canceled' + | 'failed' + | 'rejected' + | 'auth-required' + | 'unknown'; + +/** + * Message role per A2A Protocol specification. + */ +export type MessageRole = 'user' | 'agent'; + +/** + * Base interface for all part types. + */ +export interface PartBase { + metadata?: { [key: string]: any }; +} + +/** + * Text part - contains text content. + */ +export interface TextPart extends PartBase { + readonly kind: 'text'; + text: string; +} + +/** + * File base interface. + */ +export interface FileBase { + name?: string; + mimeType?: string; +} + +/** + * File with base64-encoded bytes. + */ +export interface FileWithBytes extends FileBase { + bytes: string; // Base64 encoded + uri?: never; +} + +/** + * File with URI reference. + */ +export interface FileWithUri extends FileBase { + uri: string; + bytes?: never; +} + +/** + * File part - contains file data. + */ +export interface FilePart extends PartBase { + readonly kind: 'file'; + file: FileWithBytes | FileWithUri; +} + +/** + * Data part - contains structured JSON data. + */ +export interface DataPart extends PartBase { + readonly kind: 'data'; + data: { [key: string]: any }; +} + +/** + * Union of all part types per A2A specification. + */ +export type Part = TextPart | FilePart | DataPart; + +/** + * A2A Protocol message structure. + */ +export interface Message { + readonly role: MessageRole; + parts: Part[]; // Required: Array of message parts + metadata?: { [key: string]: any }; // Optional: Extension metadata + extensions?: string[]; // Optional: Extension identifiers + referenceTaskIds?: string[]; // Optional: Referenced task IDs + messageId: string; // Required: Unique message identifier + taskId?: string; // Optional: Associated task ID + contextId?: string; // Optional: Context identifier + readonly kind: 'message'; // Required: Discriminator +} + +/** + * Task status structure. + */ +export interface TaskStatus { + state: TaskState; // Required: Current state + message?: Message; // Optional: Status message + timestamp?: string; // Optional: ISO 8601 timestamp +} + +/** + * Artifact - generated output from the agent. + */ +export interface Artifact { + artifactId: string; // Required: Unique artifact ID + name?: string; // Optional: Artifact name + description?: string; // Optional: Description + parts: Part[]; // Required: Artifact content + metadata?: { [key: string]: any }; // Optional: Metadata + extensions?: string[]; // Optional: Extension IDs +} + +/** + * A2A Protocol task structure. + */ +export interface Task { + id: string; // Required: Unique task identifier + contextId: string; // Required: Context across related tasks + status: TaskStatus; // Required: Current task status + history?: Message[]; // Optional: Conversation history + artifacts?: Artifact[]; // Optional: Task artifacts + metadata?: { [key: string]: any }; // Optional: Extension metadata + readonly kind: 'task'; // Required: Discriminator +} + +/** + * Task status update event (streaming). + */ +export interface TaskStatusUpdateEvent { + taskId: string; + contextId: string; + readonly kind: 'status-update'; + status: TaskStatus; + final: boolean; // True for final event + metadata?: { [key: string]: any }; +} + +/** + * Task artifact update event (streaming). + */ +export interface TaskArtifactUpdateEvent { + taskId: string; + contextId: string; + readonly kind: 'artifact-update'; + artifact: Artifact; + append?: boolean; // Append to existing artifact + lastChunk?: boolean; // Final chunk + metadata?: { [key: string]: any }; +} + +/** + * Push notification configuration. + */ +export interface PushNotificationConfig { + url: string; + headers?: { [key: string]: string }; +} + +/** + * Message send configuration. + */ +export interface MessageSendConfiguration { + acceptedOutputModes?: string[]; + historyLength?: number; + pushNotificationConfig?: PushNotificationConfig; + blocking?: boolean; // Wait for completion +} + +/** + * Parameters for message/send and message/stream methods. + */ +export interface MessageSendParams { + message: Message; // Required + configuration?: MessageSendConfiguration; // Optional + metadata?: { [key: string]: any }; // Optional +} + +/** + * Parameters for tasks/get method. + */ +export interface TaskQueryParams { + id: string; // Required: Task ID + historyLength?: number; // Optional: Limit history items + metadata?: { [key: string]: any }; +} + +/** + * Parameters for tasks/list method. + */ +export interface ListTasksParams { + contextId?: string; + status?: TaskState; + pageSize?: number; // 1-100, default 50 + pageToken?: string; + historyLength?: number; + lastUpdatedAfter?: number; // Unix timestamp + includeArtifacts?: boolean; + metadata?: { [key: string]: any }; +} + +/** + * Result for tasks/list method. + */ +export interface ListTasksResult { + tasks: Task[]; + totalSize: number; + pageSize: number; + nextPageToken: string; +} + +/** + * Parameters for tasks/cancel and tasks/resubscribe methods. + */ +export interface TaskIdParams { + id: string; // Required: Task ID + metadata?: { [key: string]: any }; +} + +/** + * Converted message parts for internal use (compatibility layer). + * Used by adapters to convert between A2A and Dexto internal format. + */ +export interface ConvertedMessage { + text: string; + image: + | { + image: string; + mimeType: string; + } + | undefined; + file: + | { + data: string; + mimeType: string; + filename?: string; + } + | undefined; +} diff --git a/dexto/packages/server/src/approval/approval-coordinator.ts b/dexto/packages/server/src/approval/approval-coordinator.ts new file mode 100644 index 00000000..49382cf2 --- /dev/null +++ b/dexto/packages/server/src/approval/approval-coordinator.ts @@ -0,0 +1,90 @@ +import { EventEmitter } from 'node:events'; +import type { ApprovalRequest, ApprovalResponse } from '@dexto/core'; + +/** + * Event coordinator for approval request/response flow between handler and server. + * + * Provides explicit separation between agent lifecycle events (on AgentEventBus) + * and server-mode coordination events (on ApprovalCoordinator). + * + * Used by: + * - ManualApprovalHandler: Emits requests, listens for responses + * - Streaming endpoints: Listens for requests, helps emit responses + * - Approval routes: Emits responses from client submissions + */ +export class ApprovalCoordinator extends EventEmitter { + // Track approvalId -> sessionId mapping for multi-client SSE routing + private approvalSessions = new Map(); + + /** + * Emit an approval request. + * Called by ManualApprovalHandler when tool/command needs approval. + */ + public emitRequest(request: ApprovalRequest): void { + // Store sessionId mapping for later lookup when client submits response + this.approvalSessions.set(request.approvalId, request.sessionId); + this.emit('approval:request', request); + } + + /** + * Emit an approval response. + * Called by API routes when user submits decision. + */ + public emitResponse(response: ApprovalResponse): void { + this.emit('approval:response', response); + // Clean up the mapping after response is emitted + this.approvalSessions.delete(response.approvalId); + } + + /** + * Get the sessionId associated with an approval request. + * Used by API routes to attach sessionId to responses for SSE routing. + */ + public getSessionId(approvalId: string): string | undefined { + return this.approvalSessions.get(approvalId); + } + + /** + * Subscribe to approval requests. + * Used by streaming endpoints to forward requests to SSE clients. + * + * @param handler Callback to handle approval requests + * @param options Optional AbortSignal for cleanup + */ + public onRequest( + handler: (request: ApprovalRequest) => void, + options?: { signal?: AbortSignal } + ): void { + const listener = (request: ApprovalRequest) => handler(request); + this.on('approval:request', listener); + + // Cleanup on abort signal + if (options?.signal) { + options.signal.addEventListener('abort', () => { + this.off('approval:request', listener); + }); + } + } + + /** + * Subscribe to approval responses. + * Used by ManualApprovalHandler to resolve pending approval promises. + * + * @param handler Callback to handle approval responses + * @param options Optional AbortSignal for cleanup + */ + public onResponse( + handler: (response: ApprovalResponse) => void, + options?: { signal?: AbortSignal } + ): void { + const listener = (response: ApprovalResponse) => handler(response); + this.on('approval:response', listener); + + // Cleanup on abort signal + if (options?.signal) { + options.signal.addEventListener('abort', () => { + this.off('approval:response', listener); + }); + } + } +} diff --git a/dexto/packages/server/src/approval/manual-approval-handler.test.ts b/dexto/packages/server/src/approval/manual-approval-handler.test.ts new file mode 100644 index 00000000..84a92153 --- /dev/null +++ b/dexto/packages/server/src/approval/manual-approval-handler.test.ts @@ -0,0 +1,311 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import type { ApprovalRequest, ApprovalResponse } from '@dexto/core'; +import { ApprovalType, ApprovalStatus, DenialReason } from '@dexto/core'; +import { createManualApprovalHandler } from './manual-approval-handler.js'; +import type { ApprovalCoordinator } from './approval-coordinator.js'; + +describe('createManualApprovalHandler', () => { + let mockCoordinator: ApprovalCoordinator; + let listeners: Map void)[]>; + + beforeEach(() => { + listeners = new Map(); + + mockCoordinator = { + on: vi.fn((event: string, listener: (response: ApprovalResponse) => void) => { + const eventListeners = listeners.get(event) || []; + eventListeners.push(listener); + listeners.set(event, eventListeners); + }), + off: vi.fn((event: string, listener: (response: ApprovalResponse) => void) => { + const eventListeners = listeners.get(event) || []; + const index = eventListeners.indexOf(listener); + if (index > -1) { + eventListeners.splice(index, 1); + } + }), + emitRequest: vi.fn(), + emitResponse: vi.fn(), + } as unknown as ApprovalCoordinator; + }); + + describe('Timeout Configuration', () => { + it('should not timeout when timeout is undefined (infinite wait)', async () => { + const handler = createManualApprovalHandler(mockCoordinator); + + const request: ApprovalRequest = { + approvalId: 'test-infinite-1', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + // No timeout - should wait indefinitely + metadata: { + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }, + }; + + // Start the approval request (won't resolve until we emit a response) + const approvalPromise = handler(request); + + // Verify the request was emitted + expect(mockCoordinator.emitRequest).toHaveBeenCalledWith(request); + + // Wait a bit to ensure no timeout occurred + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Manually resolve by emitting a response + const eventListeners = listeners.get('approval:response') || []; + eventListeners.forEach((listener) => { + listener({ + approvalId: 'test-infinite-1', + status: ApprovalStatus.APPROVED, + }); + }); + + const response = await approvalPromise; + expect(response.status).toBe(ApprovalStatus.APPROVED); + }); + + it('should timeout when timeout is specified', async () => { + const handler = createManualApprovalHandler(mockCoordinator); + + const request: ApprovalRequest = { + approvalId: 'test-timeout-1', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + timeout: 50, // 50ms timeout + metadata: { + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }, + }; + + const response = await handler(request); + + expect(response.status).toBe(ApprovalStatus.CANCELLED); + expect(response.reason).toBe(DenialReason.TIMEOUT); + expect(response.message).toContain('timed out'); + expect(response.timeoutMs).toBe(50); + }); + + it('should emit timeout response to coordinator when timeout occurs', async () => { + const handler = createManualApprovalHandler(mockCoordinator); + + const request: ApprovalRequest = { + approvalId: 'test-timeout-emit', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + timeout: 50, + metadata: { + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }, + }; + + await handler(request); + + // Verify coordinator received the timeout response + expect(mockCoordinator.emitResponse).toHaveBeenCalledWith( + expect.objectContaining({ + approvalId: 'test-timeout-emit', + status: ApprovalStatus.CANCELLED, + reason: DenialReason.TIMEOUT, + }) + ); + }); + + it('should clear timeout when response is received before timeout', async () => { + vi.useFakeTimers(); + const handler = createManualApprovalHandler(mockCoordinator); + + const request: ApprovalRequest = { + approvalId: 'test-clear-timeout', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + timeout: 5000, // 5 second timeout + metadata: { + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }, + }; + + const approvalPromise = handler(request); + + // Emit response before timeout + const eventListeners = listeners.get('approval:response') || []; + eventListeners.forEach((listener) => { + listener({ + approvalId: 'test-clear-timeout', + status: ApprovalStatus.APPROVED, + }); + }); + + const response = await approvalPromise; + expect(response.status).toBe(ApprovalStatus.APPROVED); + + // Advance time past the timeout - should not cause any issues + vi.advanceTimersByTime(6000); + + vi.useRealTimers(); + }); + + it('should handle elicitation with no timeout (infinite wait)', async () => { + const handler = createManualApprovalHandler(mockCoordinator); + + const request: ApprovalRequest = { + approvalId: 'test-elicitation-infinite', + type: ApprovalType.ELICITATION, + timestamp: new Date(), + // No timeout for elicitation + metadata: { + schema: { type: 'object' as const, properties: {} }, + prompt: 'Enter data', + serverName: 'TestServer', + }, + }; + + const approvalPromise = handler(request); + + // Wait briefly + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Resolve the elicitation + const eventListeners = listeners.get('approval:response') || []; + eventListeners.forEach((listener) => { + listener({ + approvalId: 'test-elicitation-infinite', + status: ApprovalStatus.APPROVED, + data: { formData: { name: 'test' } }, + }); + }); + + const response = await approvalPromise; + expect(response.status).toBe(ApprovalStatus.APPROVED); + }); + }); + + describe('Cancellation Support', () => { + it('should support cancelling pending approvals', async () => { + const handler = createManualApprovalHandler(mockCoordinator); + + const request: ApprovalRequest = { + approvalId: 'test-cancel-1', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + metadata: { + toolName: 'test_tool', + toolCallId: 'test-call-id', + args: {}, + }, + }; + + const approvalPromise = handler(request); + + // Cancel the approval + handler.cancel?.('test-cancel-1'); + + const response = await approvalPromise; + expect(response.status).toBe(ApprovalStatus.CANCELLED); + expect(response.reason).toBe(DenialReason.SYSTEM_CANCELLED); + }); + + it('should track pending approvals', () => { + const handler = createManualApprovalHandler(mockCoordinator); + + const request1: ApprovalRequest = { + approvalId: 'pending-1', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + metadata: { toolName: 'tool1', toolCallId: 'test-call-id-1', args: {} }, + }; + + const request2: ApprovalRequest = { + approvalId: 'pending-2', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + metadata: { toolName: 'tool2', toolCallId: 'test-call-id-2', args: {} }, + }; + + // Start both requests (don't await) + handler(request1); + handler(request2); + + const pending = handler.getPending?.() || []; + expect(pending).toContain('pending-1'); + expect(pending).toContain('pending-2'); + }); + + it('should cancel all pending approvals', async () => { + const handler = createManualApprovalHandler(mockCoordinator); + + const request1: ApprovalRequest = { + approvalId: 'cancel-all-1', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + metadata: { toolName: 'tool1', toolCallId: 'test-call-id-1', args: {} }, + }; + + const request2: ApprovalRequest = { + approvalId: 'cancel-all-2', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + metadata: { toolName: 'tool2', toolCallId: 'test-call-id-2', args: {} }, + }; + + const promise1 = handler(request1); + const promise2 = handler(request2); + + // Cancel all + handler.cancelAll?.(); + + const [response1, response2] = await Promise.all([promise1, promise2]); + + expect(response1.status).toBe(ApprovalStatus.CANCELLED); + expect(response2.status).toBe(ApprovalStatus.CANCELLED); + }); + }); + + describe('Response Handling', () => { + it('should only handle responses for matching approvalId', async () => { + const handler = createManualApprovalHandler(mockCoordinator); + + const request: ApprovalRequest = { + approvalId: 'test-match-1', + type: ApprovalType.TOOL_CONFIRMATION, + timestamp: new Date(), + metadata: { toolName: 'test_tool', toolCallId: 'test-call-id', args: {} }, + }; + + const approvalPromise = handler(request); + + // Emit response for different approvalId - should be ignored + const eventListeners = listeners.get('approval:response') || []; + eventListeners.forEach((listener) => { + listener({ + approvalId: 'different-id', + status: ApprovalStatus.APPROVED, + }); + }); + + // Wait a bit - request should still be pending + await new Promise((resolve) => setTimeout(resolve, 50)); + + // Now emit correct response + eventListeners.forEach((listener) => { + listener({ + approvalId: 'test-match-1', + status: ApprovalStatus.DENIED, + reason: DenialReason.USER_DENIED, + }); + }); + + const response = await approvalPromise; + expect(response.status).toBe(ApprovalStatus.DENIED); + expect(response.reason).toBe(DenialReason.USER_DENIED); + }); + }); +}); diff --git a/dexto/packages/server/src/approval/manual-approval-handler.ts b/dexto/packages/server/src/approval/manual-approval-handler.ts new file mode 100644 index 00000000..30575891 --- /dev/null +++ b/dexto/packages/server/src/approval/manual-approval-handler.ts @@ -0,0 +1,195 @@ +import type { ApprovalHandler, ApprovalRequest, ApprovalResponse } from '@dexto/core'; +import { ApprovalStatus, DenialReason } from '@dexto/core'; +import type { ApprovalCoordinator } from './approval-coordinator.js'; + +/** + * Creates a manual approval handler that uses ApprovalCoordinator for server communication. + * + * This handler emits `approval:request` and waits for `approval:response` via the coordinator, + * enabling SSE-based approval flows where: + * 1. Handler emits approval:request → Coordinator → SSE endpoint forwards to client + * 2. Client sends decision via POST /api/approvals/{approvalId} + * 3. API route emits approval:response → Coordinator → Handler resolves + * + * The returned handler implements the optional cancellation methods (cancel, cancelAll, getPending) + * for managing pending approval requests. + * + * Timeouts are handled per-request using the timeout value from ApprovalRequest, which + * is set by ApprovalManager based on the request type (tool confirmation vs elicitation). + * + * @param coordinator The approval coordinator for request/response communication + * @returns ApprovalHandler with cancellation support + * + * @example + * ```typescript + * const coordinator = new ApprovalCoordinator(); + * const handler = createManualApprovalHandler(coordinator); + * agent.setApprovalHandler(handler); + * + * // Later, cancel a specific approval (if handler supports it) + * handler.cancel?.('approval-id-123'); + * ``` + */ +export function createManualApprovalHandler(coordinator: ApprovalCoordinator): ApprovalHandler { + // Track pending approvals for cancellation support + const pendingApprovals = new Map< + string, + { + cleanup: () => void; + resolve: (response: ApprovalResponse) => void; + request: ApprovalRequest; + } + >(); + + const handleApproval = (request: ApprovalRequest): Promise => { + return new Promise((resolve) => { + // Use per-request timeout (optional - undefined means no timeout) + // - Tool confirmations use config.toolConfirmation.timeout + // - Elicitations use config.elicitation.timeout + const effectiveTimeout = request.timeout; + + // Set timeout timer ONLY if timeout is specified + // If undefined, wait indefinitely for user response + let timer: NodeJS.Timeout | undefined; + if (effectiveTimeout !== undefined) { + timer = setTimeout(() => { + cleanup(); + pendingApprovals.delete(request.approvalId); + + // Emit timeout response so UI/clients can dismiss the prompt + const timeoutResponse: ApprovalResponse = { + approvalId: request.approvalId, + status: ApprovalStatus.CANCELLED, + sessionId: request.sessionId, + reason: DenialReason.TIMEOUT, + message: `Approval request timed out after ${effectiveTimeout}ms`, + timeoutMs: effectiveTimeout, + }; + coordinator.emitResponse(timeoutResponse); + + // Resolve with CANCELLED response (not reject) to match auto-approve/deny behavior + // Callers can uniformly check response.status instead of handling exceptions + resolve(timeoutResponse); + }, effectiveTimeout); + } + + // Cleanup function to remove listener and clear timeout + let cleanupListener: (() => void) | null = null; + const cleanup = () => { + if (timer !== undefined) { + clearTimeout(timer); + } + if (cleanupListener) { + cleanupListener(); + cleanupListener = null; + } + }; + + // Listen for approval:response events + const listener = (res: ApprovalResponse) => { + // Only handle responses for this specific approval + if (res.approvalId === request.approvalId) { + cleanup(); + pendingApprovals.delete(request.approvalId); + resolve(res); + } + }; + + // Register listener + coordinator.on('approval:response', listener); + cleanupListener = () => coordinator.off('approval:response', listener); + + // Store for cancellation support + pendingApprovals.set(request.approvalId, { + cleanup, + resolve, + request, + }); + + // Emit the approval:request event via coordinator + // SSE endpoints will subscribe to coordinator and forward to clients + coordinator.emitRequest(request); + }); + }; + + const handler: ApprovalHandler = Object.assign(handleApproval, { + cancel: (approvalId: string): void => { + const pending = pendingApprovals.get(approvalId); + if (pending) { + pending.cleanup(); + pendingApprovals.delete(approvalId); + + // Create cancellation response + const cancelResponse: ApprovalResponse = { + approvalId, + status: ApprovalStatus.CANCELLED, + sessionId: pending.request.sessionId, + reason: DenialReason.SYSTEM_CANCELLED, + message: 'Approval request was cancelled', + }; + + // Emit cancellation event so UI listeners can dismiss the prompt + coordinator.emitResponse(cancelResponse); + + // Resolve with CANCELLED response (not reject) to match auto-approve/deny behavior + // Callers can uniformly check response.status instead of handling exceptions + pending.resolve(cancelResponse); + } + }, + + cancelAll: (): void => { + for (const [approvalId] of pendingApprovals) { + handler.cancel?.(approvalId); + } + }, + + getPending: (): string[] => { + return Array.from(pendingApprovals.keys()); + }, + + getPendingRequests: (): ApprovalRequest[] => { + return Array.from(pendingApprovals.values()).map((p) => p.request); + }, + + /** + * Auto-approve pending requests that match a predicate. + * Used when a pattern is remembered to auto-approve other parallel requests + * that would now match the same pattern. + */ + autoApprovePending: ( + predicate: (request: ApprovalRequest) => boolean, + responseData?: Record + ): number => { + let count = 0; + + // Find all pending approvals that match the predicate + for (const [approvalId, pending] of pendingApprovals) { + if (predicate(pending.request)) { + // Clean up the pending state + pending.cleanup(); + pendingApprovals.delete(approvalId); + + // Create auto-approval response + const autoApproveResponse: ApprovalResponse = { + approvalId, + status: ApprovalStatus.APPROVED, + sessionId: pending.request.sessionId, + message: 'Auto-approved due to matching remembered pattern', + data: responseData, + }; + + // Emit response so UI can update + coordinator.emitResponse(autoApproveResponse); + + // Resolve the pending promise + pending.resolve(autoApproveResponse); + count++; + } + } + + return count; + }, + }); + + return handler; +} diff --git a/dexto/packages/server/src/events/__tests__/webhook-subscriber.test.ts b/dexto/packages/server/src/events/__tests__/webhook-subscriber.test.ts new file mode 100644 index 00000000..ded726d7 --- /dev/null +++ b/dexto/packages/server/src/events/__tests__/webhook-subscriber.test.ts @@ -0,0 +1,429 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import type { WebhookConfig } from '../webhook-types.js'; +import { AgentEventBus } from '@dexto/core'; +import { WebhookEventSubscriber } from '../webhook-subscriber.js'; + +// Create a mock fetch function +const mockFetch = vi.fn(); + +// We'll use fake timers selectively for specific tests +// TODO: temporarily DUPE OF cli +describe('WebhookEventSubscriber', () => { + let webhookSubscriber: WebhookEventSubscriber; + let agentEventBus: AgentEventBus; + + beforeEach(() => { + // Set test environment before creating subscriber + process.env.NODE_ENV = 'test'; + + // Completely reset the mock + mockFetch.mockReset(); + + // Set default mock implementation (no artificial delay needed with fake timers) + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + statusText: 'OK', + } as any); + + // Create webhook subscriber with mocked fetch + webhookSubscriber = new WebhookEventSubscriber({ fetchFn: mockFetch as any }); + agentEventBus = new AgentEventBus(); + }); + + afterEach(() => { + // Clean up webhook subscriber and abort controllers + webhookSubscriber.cleanup(); + + // Reset all mocks + vi.resetAllMocks(); + + // Clear the test environment + delete process.env.NODE_ENV; + }); + + describe('Webhook Management', () => { + it('should add a webhook', () => { + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + secret: 'secret123', + description: 'Test webhook', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + + const retrievedWebhook = webhookSubscriber.getWebhook('wh_test_123'); + expect(retrievedWebhook).toEqual(webhook); + }); + + it('should remove a webhook', () => { + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + expect(webhookSubscriber.getWebhook('wh_test_123')).toBeDefined(); + + const removed = webhookSubscriber.removeWebhook('wh_test_123'); + expect(removed).toBe(true); + expect(webhookSubscriber.getWebhook('wh_test_123')).toBeUndefined(); + }); + + it('should return false when removing non-existent webhook', () => { + const removed = webhookSubscriber.removeWebhook('non_existent'); + expect(removed).toBe(false); + }); + + it('should list all webhooks', () => { + const webhook1: WebhookConfig = { + id: 'wh_test_1', + url: 'https://example.com/webhook1', + createdAt: new Date(), + }; + + const webhook2: WebhookConfig = { + id: 'wh_test_2', + url: 'https://example.com/webhook2', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook1); + webhookSubscriber.addWebhook(webhook2); + + const webhooks = webhookSubscriber.getWebhooks(); + expect(webhooks).toHaveLength(2); + expect(webhooks).toContainEqual(webhook1); + expect(webhooks).toContainEqual(webhook2); + }); + }); + + describe('Event Subscription', () => { + it('should subscribe to agent events', () => { + const mockOn = vi.spyOn(agentEventBus, 'on'); + + webhookSubscriber.subscribe(agentEventBus); + + // Verify that all expected events are subscribed to + expect(mockOn).toHaveBeenCalledWith('llm:thinking', expect.any(Function), { + signal: expect.any(AbortSignal), + }); + expect(mockOn).toHaveBeenCalledWith('llm:response', expect.any(Function), { + signal: expect.any(AbortSignal), + }); + expect(mockOn).toHaveBeenCalledWith('session:reset', expect.any(Function), { + signal: expect.any(AbortSignal), + }); + }); + + it('should clean up event listeners on cleanup', () => { + // Subscribe first to create abort controller + webhookSubscriber.subscribe(agentEventBus); + + // Spy on the abort method of the actual abort controller + const abortController = (webhookSubscriber as any).abortController; + expect(abortController).toBeDefined(); + const mockAbort = vi.spyOn(abortController, 'abort'); + + // Call cleanup + webhookSubscriber.cleanup(); + + // Verify abort was called + expect(mockAbort).toHaveBeenCalled(); + + // Verify abortController is cleaned up + expect((webhookSubscriber as any).abortController).toBeUndefined(); + }); + }); + + describe('Event Delivery', () => { + // Default mock is already set up in parent beforeEach + + it('should deliver events to registered webhooks', async () => { + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + secret: 'secret123', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + webhookSubscriber.subscribe(agentEventBus); + + // Emit event and wait for async delivery + agentEventBus.emit('session:reset', { sessionId: 'test-session' }); + + // Wait for async delivery to complete (much shorter in test env due to 1ms delays) + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Check if fetch was called + expect(mockFetch).toHaveBeenCalledWith( + 'https://example.com/webhook', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ + 'Content-Type': 'application/json', + 'User-Agent': 'DextoAgent/1.0', + 'X-Dexto-Event-Type': 'session:reset', + 'X-Dexto-Signature-256': expect.stringMatching(/^sha256=[a-f0-9]{64}$/), + }), + body: expect.stringContaining('"type":"session:reset"'), + }) + ); + }); + + it('should not deliver events when no webhooks are registered', async () => { + webhookSubscriber.subscribe(agentEventBus); + + agentEventBus.emit('session:reset', { sessionId: 'test-session' }); + + await new Promise((resolve) => setTimeout(resolve, 5)); + + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('should include proper webhook event structure', async () => { + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + webhookSubscriber.subscribe(agentEventBus); + + agentEventBus.emit('llm:response', { + content: 'Hello world', + sessionId: 'test-session', + tokenUsage: { totalTokens: 2 }, + model: 'test-model', + }); + + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(mockFetch).toHaveBeenCalled(); + expect(mockFetch.mock.calls[0]).toBeDefined(); + const [_url, requestOptions] = mockFetch.mock.calls[0]!; + const requestBody = JSON.parse((requestOptions as any).body); + + expect(requestBody).toMatchObject({ + id: expect.stringMatching(/^evt_/), + type: 'llm:response', + data: { + content: 'Hello world', + sessionId: 'test-session', + tokenUsage: { totalTokens: 2 }, + model: 'test-model', + }, + created: expect.any(String), + apiVersion: '2025-07-03', + }); + }); + }); + + describe('Webhook Testing', () => { + it('should test webhook successfully', async () => { + // Use default mock which includes delay for responseTime + + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + + const result = await webhookSubscriber.testWebhook('wh_test_123'); + + expect(result.success).toBe(true); + expect(result.statusCode).toBe(200); + expect(result.responseTime).toBeGreaterThanOrEqual(0); + expect(mockFetch).toHaveBeenCalledWith( + 'https://example.com/webhook', + expect.objectContaining({ + method: 'POST', + body: expect.stringContaining('"type":"tools:available-updated"'), + }) + ); + }); + + it('should throw error when testing non-existent webhook', async () => { + await expect(webhookSubscriber.testWebhook('non_existent')).rejects.toThrow( + 'Webhook not found: non_existent' + ); + }); + }); + + describe('Retry Logic', () => { + it('should retry failed requests', async () => { + // First two calls fail, third succeeds + mockFetch + .mockRejectedValueOnce(new Error('Network error')) + .mockRejectedValueOnce(new Error('Network error')) + .mockResolvedValueOnce({ + ok: true, + status: 200, + statusText: 'OK', + } as any); + + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + + const result = await webhookSubscriber.testWebhook('wh_test_123'); + + expect(result.success).toBe(true); + expect(mockFetch).toHaveBeenCalledTimes(3); + }); + + it('should fail after max retries', async () => { + mockFetch.mockRejectedValue(new Error('Network error')); + + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + + const result = await webhookSubscriber.testWebhook('wh_test_123'); + + expect(result.success).toBe(false); + expect(result.error).toContain('Network error'); + expect(mockFetch).toHaveBeenCalledTimes(3); // Default max retries + }); + }); + + describe('Security', () => { + it('should generate HMAC signature when secret is provided', async () => { + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + secret: 'test-secret', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + webhookSubscriber.subscribe(agentEventBus); + + agentEventBus.emit('session:reset', { sessionId: 'test-session' }); + + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(mockFetch).toHaveBeenCalled(); + expect(mockFetch.mock.calls[0]).toBeDefined(); + const [_url, requestOptions] = mockFetch.mock.calls[0]!; + expect((requestOptions as any).headers['X-Dexto-Signature-256']).toMatch( + /^sha256=[a-f0-9]{64}$/ + ); + }); + + it('should not include signature when no secret is provided', async () => { + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + webhookSubscriber.subscribe(agentEventBus); + + agentEventBus.emit('session:reset', { sessionId: 'test-session' }); + + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(mockFetch).toHaveBeenCalled(); + expect(mockFetch.mock.calls[0]).toBeDefined(); + const [_url, requestOptions] = mockFetch.mock.calls[0]!; + expect((requestOptions as any).headers['X-Dexto-Signature-256']).toBeUndefined(); + }); + }); + + describe('Error Handling', () => { + it('should handle HTTP error responses', async () => { + mockFetch.mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 1)); + return { + ok: false, + status: 404, + statusText: 'Not Found', + } as any; + }); + + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + + const result = await webhookSubscriber.testWebhook('wh_test_123'); + + expect(result.success).toBe(false); + expect(result.statusCode).toBe(404); + expect(result.error).toBe('HTTP 404: Not Found'); + }); + + it('should handle timeout errors', async () => { + const abortError = new Error('The operation was aborted'); + abortError.name = 'AbortError'; + mockFetch.mockRejectedValue(abortError); + + const webhook: WebhookConfig = { + id: 'wh_test_123', + url: 'https://example.com/webhook', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook); + + const result = await webhookSubscriber.testWebhook('wh_test_123'); + + expect(result.success).toBe(false); + expect(result.error).toContain('aborted'); + }); + }); + + describe('Multiple Webhooks', () => { + it('should deliver events to multiple webhooks', async () => { + const webhook1: WebhookConfig = { + id: 'wh_test_1', + url: 'https://example.com/webhook1', + createdAt: new Date(), + }; + + const webhook2: WebhookConfig = { + id: 'wh_test_2', + url: 'https://example.com/webhook2', + createdAt: new Date(), + }; + + webhookSubscriber.addWebhook(webhook1); + webhookSubscriber.addWebhook(webhook2); + webhookSubscriber.subscribe(agentEventBus); + + agentEventBus.emit('session:reset', { sessionId: 'test-session' }); + + await new Promise((resolve) => setTimeout(resolve, 200)); + + expect(mockFetch).toHaveBeenCalledTimes(2); + expect(mockFetch).toHaveBeenCalledWith( + 'https://example.com/webhook1', + expect.any(Object) + ); + expect(mockFetch).toHaveBeenCalledWith( + 'https://example.com/webhook2', + expect.any(Object) + ); + }); + }); +}); diff --git a/dexto/packages/server/src/events/a2a-sse-subscriber.ts b/dexto/packages/server/src/events/a2a-sse-subscriber.ts new file mode 100644 index 00000000..ebb2c304 --- /dev/null +++ b/dexto/packages/server/src/events/a2a-sse-subscriber.ts @@ -0,0 +1,337 @@ +/** + * A2A SSE (Server-Sent Events) Event Subscriber + * + * Subscribes to agent events and streams them to SSE clients for A2A tasks. + * Uses standard SSE protocol (text/event-stream). + * + * Design: + * - Filters events by taskId/sessionId for targeted streaming + * - Uses standard SSE format: event: name\ndata: json\n\n + * - Supports multiple concurrent SSE connections + */ + +/* eslint-disable no-undef */ +import { setMaxListeners } from 'events'; +import { AgentEventBus } from '@dexto/core'; +import { logger } from '@dexto/core'; + +/** + * SSE connection state + */ +interface SSEConnection { + /** Task/Session ID to filter events */ + taskId: string; + /** Controller to write SSE events */ + controller: ReadableStreamDefaultController; + /** Abort signal for cleanup */ + abortController: AbortController; + /** Connection timestamp */ + connectedAt: number; +} + +/** + * A2A SSE Event Subscriber + * + * Manages Server-Sent Events connections for A2A Protocol task streaming. + * + * Usage: + * ```typescript + * const sseSubscriber = new A2ASseEventSubscriber(); + * sseSubscriber.subscribe(agent.agentEventBus); + * + * // In route handler + * const stream = sseSubscriber.createStream(taskId); + * return new Response(stream, { + * headers: { + * 'Content-Type': 'text/event-stream', + * 'Cache-Control': 'no-cache', + * 'Connection': 'keep-alive' + * } + * }); + * ``` + */ +export class A2ASseEventSubscriber { + private connections: Map = new Map(); + private eventBus?: AgentEventBus; + private globalAbortController?: AbortController; + + /** + * Subscribe to agent event bus. + * Sets up global event listeners that broadcast to all SSE connections. + * + * @param eventBus Agent event bus to subscribe to + */ + subscribe(eventBus: AgentEventBus): void { + // Abort any previous subscription + this.globalAbortController?.abort(); + + // Create new AbortController for this subscription + this.globalAbortController = new AbortController(); + const { signal } = this.globalAbortController; + + // Increase max listeners + const MAX_SHARED_SIGNAL_LISTENERS = 20; + setMaxListeners(MAX_SHARED_SIGNAL_LISTENERS, signal); + + this.eventBus = eventBus; + + // Subscribe to agent events + eventBus.on( + 'llm:thinking', + (payload) => { + this.broadcastToTask(payload.sessionId, 'task.thinking', { + taskId: payload.sessionId, + }); + }, + { signal } + ); + + eventBus.on( + 'llm:chunk', + (payload) => { + this.broadcastToTask(payload.sessionId, 'task.chunk', { + taskId: payload.sessionId, + type: payload.chunkType, + content: payload.content, + isComplete: payload.isComplete, + }); + }, + { signal } + ); + + eventBus.on( + 'llm:tool-call', + (payload) => { + this.broadcastToTask(payload.sessionId, 'task.toolCall', { + taskId: payload.sessionId, + toolName: payload.toolName, + args: payload.args, + callId: payload.callId, + }); + }, + { signal } + ); + + eventBus.on( + 'llm:tool-result', + (payload) => { + const data: Record = { + taskId: payload.sessionId, + toolName: payload.toolName, + callId: payload.callId, + success: payload.success, + sanitized: payload.sanitized, + }; + if (payload.rawResult !== undefined) { + data.rawResult = payload.rawResult; + } + this.broadcastToTask(payload.sessionId, 'task.toolResult', data); + }, + { signal } + ); + + eventBus.on( + 'llm:response', + (payload) => { + this.broadcastToTask(payload.sessionId, 'task.message', { + taskId: payload.sessionId, + message: { + role: 'agent', + content: [{ type: 'text', text: payload.content }], + timestamp: new Date().toISOString(), + }, + tokenUsage: payload.tokenUsage, + provider: payload.provider, + model: payload.model, + }); + }, + { signal } + ); + + eventBus.on( + 'llm:error', + (payload) => { + this.broadcastToTask(payload.sessionId, 'task.error', { + taskId: payload.sessionId, + error: { + message: payload.error.message, + recoverable: payload.recoverable, + }, + }); + }, + { signal } + ); + + eventBus.on( + 'session:reset', + (payload) => { + this.broadcastToTask(payload.sessionId, 'task.reset', { + taskId: payload.sessionId, + }); + }, + { signal } + ); + + logger.debug('A2ASseEventSubscriber subscribed to agent events'); + } + + /** + * Create a new SSE stream for a specific task. + * + * Returns a ReadableStream that emits SSE events for the task. + * + * @param taskId Task/Session ID to stream events for + * @returns ReadableStream for SSE connection + */ + createStream(taskId: string): ReadableStream { + const connectionId = `${taskId}-${Date.now()}`; + + return new ReadableStream({ + start: (controller) => { + // Create connection state + const connection: SSEConnection = { + taskId, + controller, + abortController: new AbortController(), + connectedAt: Date.now(), + }; + + this.connections.set(connectionId, connection); + logger.debug(`SSE connection opened for task ${taskId}`); + + // Send initial connection event + this.sendSSEEvent(controller, 'connected', { + taskId, + timestamp: new Date().toISOString(), + }); + + // Send keepalive every 30 seconds + const keepaliveInterval = setInterval(() => { + try { + this.sendSSEComment(controller, 'keepalive'); + } catch (_error) { + clearInterval(keepaliveInterval); + } + }, 30000); + + // Cleanup on abort + connection.abortController.signal.addEventListener('abort', () => { + clearInterval(keepaliveInterval); + }); + }, + + cancel: () => { + // Client disconnected, cleanup + const connection = this.connections.get(connectionId); + if (connection) { + connection.abortController.abort(); + this.connections.delete(connectionId); + logger.debug(`SSE connection closed for task ${taskId}`); + } + }, + }); + } + + /** + * Broadcast an event to a specific task's SSE connections. + * + * @param taskId Task ID to broadcast to + * @param eventName SSE event name + * @param data Event data + */ + private broadcastToTask( + taskId: string, + eventName: string, + data: Record + ): void { + let sent = 0; + for (const [connectionId, connection] of this.connections.entries()) { + if (connection.taskId === taskId) { + try { + this.sendSSEEvent(connection.controller, eventName, data); + sent++; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.warn(`Failed to send SSE event to ${connectionId}: ${errorMessage}`); + // Clean up failed connection + connection.abortController.abort(); + this.connections.delete(connectionId); + } + } + } + + if (sent > 0) { + logger.debug(`Broadcast ${eventName} to ${sent} SSE connection(s) for task ${taskId}`); + } + } + + /** + * Send an SSE event to a specific controller. + * + * Format: event: name\ndata: json\n\n + * + * @param controller Stream controller + * @param eventName Event name + * @param data Event data + */ + private sendSSEEvent( + controller: ReadableStreamDefaultController, + eventName: string, + data: Record + ): void { + const event = `event: ${eventName}\ndata: ${JSON.stringify(data)}\n\n`; + controller.enqueue(new TextEncoder().encode(event)); + } + + /** + * Send an SSE comment (for keepalive). + * + * Format: : comment\n + * + * @param controller Stream controller + * @param comment Comment text + */ + private sendSSEComment(controller: ReadableStreamDefaultController, comment: string): void { + const line = `: ${comment}\n`; + controller.enqueue(new TextEncoder().encode(line)); + } + + /** + * Close all connections and cleanup. + */ + cleanup(): void { + logger.debug(`Cleaning up ${this.connections.size} SSE connections`); + + for (const [_connectionId, connection] of this.connections.entries()) { + connection.abortController.abort(); + try { + connection.controller.close(); + } catch (_error) { + // Ignore errors on close + } + } + + this.connections.clear(); + this.globalAbortController?.abort(); + } + + /** + * Get active connection count. + */ + getConnectionCount(): number { + return this.connections.size; + } + + /** + * Get connection count for a specific task. + */ + getTaskConnectionCount(taskId: string): number { + let count = 0; + for (const connection of this.connections.values()) { + if (connection.taskId === taskId) { + count++; + } + } + return count; + } +} diff --git a/dexto/packages/server/src/events/types.ts b/dexto/packages/server/src/events/types.ts new file mode 100644 index 00000000..8c6cdb45 --- /dev/null +++ b/dexto/packages/server/src/events/types.ts @@ -0,0 +1,16 @@ +import { AgentEventBus } from '@dexto/core'; + +/** + * Generic interface for subscribing to core events. + */ +export interface EventSubscriber { + /** + * Attach event handlers to the given event bus. + */ + subscribe(eventBus: AgentEventBus): void; + + /** + * Clean up event listeners and resources. + */ + cleanup?(): void; +} diff --git a/dexto/packages/server/src/events/webhook-subscriber.ts b/dexto/packages/server/src/events/webhook-subscriber.ts new file mode 100644 index 00000000..b7871254 --- /dev/null +++ b/dexto/packages/server/src/events/webhook-subscriber.ts @@ -0,0 +1,353 @@ +import crypto from 'crypto'; +import { setMaxListeners } from 'events'; +import { + AgentEventBus, + INTEGRATION_EVENTS, + type AgentEventMap, + type AgentEventName, +} from '@dexto/core'; +import { logger } from '@dexto/core'; +import { EventSubscriber } from './types.js'; +import { + type WebhookConfig, + type DextoWebhookEvent, + type WebhookDeliveryResult, + type WebhookDeliveryOptions, +} from './webhook-types.js'; + +/** + * Default configuration for webhook delivery + */ +const DEFAULT_DELIVERY_OPTIONS: Required = { + maxRetries: 3, + timeout: 10000, // 10 seconds + includeSignature: true, +}; + +/** + * Webhook event subscriber that delivers agent events via HTTP POST + */ +export class WebhookEventSubscriber implements EventSubscriber { + private webhooks: Map = new Map(); + private abortController?: AbortController; + private deliveryOptions: Required; + private fetchFn: typeof globalThis.fetch; + + constructor({ + fetchFn, + ...deliveryOptions + }: WebhookDeliveryOptions & { fetchFn?: typeof globalThis.fetch } = {}) { + this.deliveryOptions = { ...DEFAULT_DELIVERY_OPTIONS, ...deliveryOptions }; + // Use native fetch (Node.js 20+) or injected implementation (tests) + this.fetchFn = fetchFn || fetch; + logger.debug('WebhookEventSubscriber initialized'); + } + + /** + * Subscribe to agent events and deliver them to registered webhooks + */ + subscribe(eventBus: AgentEventBus): void { + // Abort any previous subscription before creating a new one + this.abortController?.abort(); + + // Create new AbortController for this subscription + this.abortController = new AbortController(); + const { signal } = this.abortController; + + // Increase max listeners since we intentionally share this signal across multiple events + // This prevents the MaxListenersExceededWarning + // INTEGRATION_EVENTS currently has 24 events, so we set this higher with buffer + const MAX_SHARED_SIGNAL_LISTENERS = 50; + setMaxListeners(MAX_SHARED_SIGNAL_LISTENERS, signal); + + // Subscribe to all INTEGRATION_EVENTS (tier 2 visibility) + // This includes streaming events + lifecycle/state events + INTEGRATION_EVENTS.forEach((eventName) => { + eventBus.on( + eventName, + (payload) => { + this.deliverEvent(eventName, payload); + }, + { signal } + ); + }); + + logger.info(`Webhook subscriber active with ${this.webhooks.size} registered webhooks`); + } + + /** + * Register a new webhook endpoint + */ + addWebhook(webhook: WebhookConfig): void { + this.webhooks.set(webhook.id, webhook); + logger.info(`Webhook registered: ${webhook.id} -> ${webhook.url}`); + } + + /** + * Remove a webhook endpoint + */ + removeWebhook(webhookId: string): boolean { + const removed = this.webhooks.delete(webhookId); + if (removed) { + logger.info(`Webhook removed: ${webhookId}`); + } else { + logger.warn(`Attempted to remove non-existent webhook: ${webhookId}`); + } + return removed; + } + + /** + * Get all registered webhooks + */ + getWebhooks(): WebhookConfig[] { + return Array.from(this.webhooks.values()); + } + + /** + * Get a specific webhook by ID + */ + getWebhook(webhookId: string): WebhookConfig | undefined { + return this.webhooks.get(webhookId); + } + + /** + * Test a webhook by sending a sample event + */ + async testWebhook(webhookId: string): Promise { + const webhook = this.webhooks.get(webhookId); + if (!webhook) { + throw new Error(`Webhook not found: ${webhookId}`); + } + + const testEvent: DextoWebhookEvent<'tools:available-updated'> = { + id: `evt_test_${Date.now()}`, + type: 'tools:available-updated', + data: { + tools: ['test-tool'], + source: 'mcp', + }, + created: new Date().toISOString(), + apiVersion: '2025-07-03', + }; + + return this.deliverToWebhook(webhook, testEvent); + } + + /** + * Clean up event listeners and resources + */ + cleanup(): void { + if (this.abortController) { + this.abortController.abort(); + delete (this as any).abortController; + } + + this.webhooks.clear(); + logger.debug('Webhook event subscriber cleaned up'); + } + + /** + * Unsubscribe from current event bus without clearing registered webhooks + */ + unsubscribe(): void { + if (this.abortController) { + const controller = this.abortController; + delete this.abortController; + try { + controller.abort(); + } catch (error) { + logger.debug( + `Error aborting controller during unsubscribe: ${ + error instanceof Error ? error.message : String(error) + }`, + { + location: 'WebhookEventSubscriber.unsubscribe', + ...(error instanceof Error + ? { stack: error.stack } + : { value: String(error) }), + } + ); + } + } + } + + /** + * Deliver an event to all registered webhooks + */ + private async deliverEvent( + eventType: T, + eventData: AgentEventMap[T] + ): Promise { + if (this.webhooks.size === 0) { + return; // No webhooks to deliver to + } + + const webhookEvent: DextoWebhookEvent = { + id: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`, + type: eventType, + data: eventData, + created: new Date().toISOString(), + apiVersion: '2025-07-03', + }; + + logger.debug(`Delivering webhook event: ${eventType} to ${this.webhooks.size} webhooks`); + + // Deliver to all webhooks in parallel + const deliveryPromises = Array.from(this.webhooks.values()).map((webhook) => ({ + webhook, + promise: this.deliverToWebhook(webhook, webhookEvent), + })); + + const handleSettled = (results: PromiseSettledResult[]) => { + results.forEach((result, i) => { + if (result.status === 'rejected') { + const webhook = deliveryPromises[i]?.webhook; + if (webhook) { + logger.error( + `Webhook delivery failed for ${webhook.id}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}` + ); + } + } + }); + }; + + // For testing purposes, we can await this if needed + if (process.env.NODE_ENV === 'test') { + const results = await Promise.allSettled(deliveryPromises.map((p) => p.promise)); + handleSettled(results); + } else { + // Fire-and-forget in production + Promise.allSettled(deliveryPromises.map((p) => p.promise)).then(handleSettled); + } + } + + /** + * Deliver an event to a specific webhook with retry logic + */ + private async deliverToWebhook( + webhook: WebhookConfig, + event: DextoWebhookEvent + ): Promise { + const startTime = Date.now(); + let lastError: Error | undefined; + let lastStatusCode: number | undefined; + + for (let attempt = 1; attempt <= this.deliveryOptions.maxRetries; attempt++) { + try { + const result = await this.sendWebhookRequest(webhook, event, attempt); + if (result.success) { + return result; + } + // Don't duplicate "HTTP xxx:" prefix if it's already in the error message + lastError = new Error(result.error || `HTTP ${result.statusCode}`); + lastStatusCode = result.statusCode; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + logger.warn( + `Webhook delivery attempt ${attempt}/${this.deliveryOptions.maxRetries} failed for ${webhook.id}: ${lastError.message}` + ); + } + + // Wait before retry (exponential backoff with jitter) + if (attempt < this.deliveryOptions.maxRetries) { + // Use shorter delays in test environment for faster tests + const baseDelay = process.env.NODE_ENV === 'test' ? 1 : 1000; + const exp = baseDelay * Math.pow(2, attempt - 1); + const jitter = exp * 0.2 * Math.random(); // ±20% + const backoffMs = Math.min(exp + jitter, 10000); + await new Promise((resolve) => setTimeout(resolve, backoffMs)); + } + } + + // All attempts failed + const totalTime = Date.now() - startTime; + const result: WebhookDeliveryResult = { + success: false, + error: lastError?.message || 'Unknown error', + responseTime: totalTime, + attempt: this.deliveryOptions.maxRetries, + ...(lastStatusCode !== undefined && { statusCode: lastStatusCode }), + }; + + logger.error( + `Webhook delivery failed after ${this.deliveryOptions.maxRetries} attempts for ${webhook.id}: ${result.error}` + ); + + return result; + } + + /** + * Send HTTP request to webhook endpoint + */ + private async sendWebhookRequest( + webhook: WebhookConfig, + event: DextoWebhookEvent, + attempt: number + ): Promise { + const startTime = Date.now(); + const payload = JSON.stringify(event); + + // Prepare headers + const headers: Record = { + 'Content-Type': 'application/json', + 'User-Agent': 'DextoAgent/1.0', + 'X-Dexto-Event-Type': event.type, + 'X-Dexto-Event-Id': event.id, + 'X-Dexto-Delivery-Attempt': attempt.toString(), + }; + + // Add signature if secret is provided + if (webhook.secret && this.deliveryOptions.includeSignature) { + const signature = this.generateSignature(payload, webhook.secret); + headers['X-Dexto-Signature-256'] = signature; + } + + try { + const response = await this.fetchFn(webhook.url, { + method: 'POST', + headers, + body: payload, + signal: AbortSignal.timeout(this.deliveryOptions.timeout), + }); + + const responseTime = Date.now() - startTime; + const success = response.ok; + + const result: WebhookDeliveryResult = { + success, + statusCode: response.status, + responseTime, + attempt, + }; + + if (!success) { + result.error = `HTTP ${response.status}: ${response.statusText}`; + } + + logger.debug( + `Webhook delivery ${success ? 'succeeded' : 'failed'} for ${webhook.id}: ${response.status} in ${responseTime}ms` + ); + + return result; + } catch (error) { + const responseTime = Date.now() - startTime; + const errorMessage = error instanceof Error ? error.message : String(error); + + return { + success: false, + error: errorMessage, + responseTime, + attempt, + }; + } + } + + /** + * Generate HMAC signature for webhook verification + */ + private generateSignature(payload: string, secret: string): string { + const hmac = crypto.createHmac('sha256', secret); + hmac.update(payload, 'utf8'); + return `sha256=${hmac.digest('hex')}`; + } +} diff --git a/dexto/packages/server/src/events/webhook-types.ts b/dexto/packages/server/src/events/webhook-types.ts new file mode 100644 index 00000000..4e9f8546 --- /dev/null +++ b/dexto/packages/server/src/events/webhook-types.ts @@ -0,0 +1,100 @@ +import type { AgentEventMap, AgentEventName } from '@dexto/core'; + +/** + * TODO: temporarily DUPE OF cli + * Webhook configuration interface + * Represents a registered webhook endpoint + */ +export interface WebhookConfig { + /** Unique identifier for the webhook */ + id: string; + /** The URL to send webhook events to */ + url: string; + /** Optional secret for signature verification */ + secret?: string; + /** When the webhook was registered */ + createdAt: Date; + /** Optional description for the webhook */ + description?: string; +} + +/** + * Webhook event payload interface + * Mirrors Stripe.Event structure for familiar developer experience + */ +export interface DextoWebhookEvent { + /** Unique identifier for this webhook event */ + id: string; + /** The type of event - provides IDE autocomplete */ + type: T; + /** The event data payload - typed based on event type */ + data: AgentEventMap[T]; + /** When the event was created */ + created: string; + /** API version for future compatibility */ + apiVersion: string; +} + +/** + * Webhook delivery attempt result + */ +export interface WebhookDeliveryResult { + /** Whether the delivery was successful */ + success: boolean; + /** HTTP status code received */ + statusCode?: number; + /** Error message if delivery failed */ + error?: string; + /** Response time in milliseconds */ + responseTime: number; + /** Number of delivery attempts */ + attempt: number; +} + +/** + * Webhook registration request body + */ +export interface WebhookRegistrationRequest { + /** The URL to send webhook events to */ + url: string; + /** Optional secret for signature verification */ + secret?: string; + /** Optional description for the webhook */ + description?: string; +} + +/** + * Webhook test event payload + */ +export interface WebhookTestEvent extends DextoWebhookEvent<'tools:available-updated'> { + /** Indicates this is a test event */ + test: true; +} + +/** + * Type-safe webhook handler function signature + * Provides autocomplete for event types and typed data payloads + */ +export type WebhookHandler = ( + event: DextoWebhookEvent +) => Promise | void; + +/** + * Webhook handler mapping for type-safe event routing + * Provides IDE autocomplete for event names like Stripe webhooks + */ +export type WebhookEventHandlers = { + [K in AgentEventName]?: WebhookHandler; +}; + +/** + * Webhook delivery options + */ +export interface WebhookDeliveryOptions { + /** Maximum number of retry attempts */ + maxRetries?: number; + /** Timeout in milliseconds */ + timeout?: number; + /** Include signature verification header */ + includeSignature?: boolean; +} diff --git a/dexto/packages/server/src/hono/__tests__/agents.integration.test.ts b/dexto/packages/server/src/hono/__tests__/agents.integration.test.ts new file mode 100644 index 00000000..331262e6 --- /dev/null +++ b/dexto/packages/server/src/hono/__tests__/agents.integration.test.ts @@ -0,0 +1,244 @@ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { createTestAgent, startTestServer, httpRequest, type TestServer } from './test-fixtures.js'; +import { DextoAgent } from '@dexto/core'; +import { AgentFactory } from '@dexto/agent-management'; +import type { CreateDextoAppOptions } from '../index.js'; + +describe('Hono API Integration Tests - Agent Routes', () => { + let testServer: TestServer | undefined; + let initialAgent: DextoAgent; + let mockAgents: Array<{ + id: string; + name: string; + description: string; + author: string; + tags: string[]; + type: 'builtin' | 'custom'; + }> = []; + + beforeAll(async () => { + initialAgent = await createTestAgent(); + + // Mock AgentFactory.listAgents to return test agents + mockAgents = [ + { + id: 'test-agent-1', + name: 'Test Agent 1', + description: 'First test agent', + author: 'Test Author', + tags: ['test'], + type: 'builtin' as const, + }, + { + id: 'test-agent-2', + name: 'Test Agent 2', + description: 'Second test agent', + author: 'Test Author', + tags: ['test'], + type: 'builtin' as const, + }, + ]; + + vi.spyOn(AgentFactory, 'listAgents').mockResolvedValue({ + installed: mockAgents, + available: [], + }); + + // Create agentsContext with switching functions + let activeAgent = initialAgent; + let activeAgentId = 'test-agent-1'; + let isSwitching = false; + + const agentsContext: CreateDextoAppOptions['agentsContext'] = { + switchAgentById: async (id: string) => { + if (isSwitching) throw new Error('Agent switch in progress'); + isSwitching = true; + try { + // Create a new test agent instance (no need to use AgentFactory.createAgent in tests) + const newAgent = await createTestAgent(); + await newAgent.start(); + if (activeAgent.isStarted()) { + await activeAgent.stop(); + } + activeAgent = newAgent; + activeAgentId = id; + return { id, name: mockAgents.find((a) => a.id === id)?.name ?? id }; + } finally { + isSwitching = false; + } + }, + switchAgentByPath: async (filePath: string) => { + if (isSwitching) throw new Error('Agent switch in progress'); + isSwitching = true; + try { + const newAgent = await createTestAgent(); + await newAgent.start(); + if (activeAgent.isStarted()) { + await activeAgent.stop(); + } + activeAgent = newAgent; + activeAgentId = `agent-from-${filePath}`; + return { id: activeAgentId, name: 'Agent from Path' }; + } finally { + isSwitching = false; + } + }, + resolveAgentInfo: async (id: string) => { + const agent = mockAgents.find((a) => a.id === id); + return { + id, + name: agent?.name ?? id, + }; + }, + ensureAgentAvailable: () => { + if (isSwitching) throw new Error('Agent switch in progress'); + if (!activeAgent.isStarted()) throw new Error('Agent not started'); + }, + getActiveAgentId: () => activeAgentId, + }; + + testServer = await startTestServer(initialAgent, undefined, agentsContext); + }); + + afterAll(async () => { + vi.restoreAllMocks(); + if (testServer) { + await testServer.cleanup(); + } + }); + + describe('Agent Management Routes', () => { + it('GET /api/agents returns list of agents', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/agents'); + expect(res.status).toBe(200); + expect(Array.isArray((res.body as { installed: unknown[] }).installed)).toBe(true); + expect( + (res.body as { installed: Array<{ id: string }> }).installed.length + ).toBeGreaterThan(0); + }); + + it('GET /api/agents/current returns current agent', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/agents/current'); + expect(res.status).toBe(200); + expect((res.body as { id: string }).id).toBeDefined(); + }); + + it('POST /api/agents/switch validates input', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/agents/switch', {}); + expect(res.status).toBeGreaterThanOrEqual(400); + }); + + it('POST /api/agents/switch switches agent by ID', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Note: Agent switching requires updating getAgent() closure which is complex + // For now, we test the endpoint accepts valid input + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/agents/switch', { + id: 'test-agent-2', + }); + // May return 400 if validation fails or 200 if switch succeeds + // The actual switch logic is complex and requires getAgent() to be dynamic + expect([200, 400]).toContain(res.status); + if (res.status === 200) { + const body = res.body as { switched: boolean; id: string; name: string }; + expect(body.switched).toBe(true); + expect(body.id).toBe('test-agent-2'); + expect(typeof body.name).toBe('string'); + } + }); + + it('POST /api/agents/validate-name validates agent name', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/agents/validate-name', { + id: 'valid-agent-name-that-does-not-exist', + }); + expect(res.status).toBe(200); + const body = res.body as { valid: boolean; message?: string }; + expect(body.valid).toBe(true); + }); + + it('POST /api/agents/validate-name rejects invalid names', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/agents/validate-name', { + id: 'test-agent-1', // This conflicts with our mock + }); + expect(res.status).toBe(200); + const body = res.body as { valid: boolean; conflict?: string; message?: string }; + expect(body.valid).toBe(false); + expect(body.conflict).toBeDefined(); + }); + }); + + describe('Agent Config Routes', () => { + // Note: Agent path/config routes require agent to have configPath set + // These are skipped in test environment as we use in-memory agents + it.skip('GET /api/agent/path returns agent path', async () => { + // Requires agent with configPath - test agents don't have this + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/agent/path'); + expect(res.status).toBe(200); + const body = res.body as { + path: string; + relativePath: string; + name: string; + isDefault: boolean; + }; + expect(typeof body.path).toBe('string'); + expect(typeof body.relativePath).toBe('string'); + expect(typeof body.name).toBe('string'); + expect(typeof body.isDefault).toBe('boolean'); + }); + + it.skip('GET /api/agent/config returns agent config', async () => { + // Requires agent with configPath - test agents don't have this + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/agent/config'); + expect(res.status).toBe(200); + const body = res.body as { config: unknown; path: string; lastModified?: unknown }; + expect(body.config).toBeDefined(); + expect(typeof body.path).toBe('string'); + }); + + it('GET /api/agent/config/export exports config', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/agent/config/export'); + expect(res.status).toBe(200); + // Export returns YAML text, not JSON + expect(res.headers['content-type']).toContain('yaml'); + expect(typeof res.text).toBe('string'); + expect(res.text.length).toBeGreaterThan(0); + }); + + it('POST /api/agent/validate validates config', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/agent/validate', { + yaml: 'systemPrompt: "You are a helpful assistant."\ngreeting: Hello\nllm:\n provider: openai\n model: gpt-5\n apiKey: sk-test-key-for-validation', + }); + expect(res.status).toBe(200); + const body = res.body as { valid: boolean; errors?: unknown[]; warnings?: unknown[] }; + expect(body.valid).toBe(true); + // errors may be undefined or empty array + expect( + body.errors === undefined || + (Array.isArray(body.errors) && body.errors.length === 0) + ).toBe(true); + }); + + it('POST /api/agent/validate rejects invalid config', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/agent/validate', { + yaml: 'invalid: yaml: content: [', + }); + expect(res.status).toBe(200); + const body = res.body as { valid: boolean; errors: unknown[]; warnings: unknown[] }; + expect(body.valid).toBe(false); + expect(Array.isArray(body.errors)).toBe(true); + expect(body.errors.length).toBeGreaterThan(0); + const firstError = body.errors[0] as { code: string; message: string }; + expect(typeof firstError.code).toBe('string'); + expect(typeof firstError.message).toBe('string'); + }); + }); +}); diff --git a/dexto/packages/server/src/hono/__tests__/api.integration.test.ts b/dexto/packages/server/src/hono/__tests__/api.integration.test.ts new file mode 100644 index 00000000..5f832734 --- /dev/null +++ b/dexto/packages/server/src/hono/__tests__/api.integration.test.ts @@ -0,0 +1,706 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { TextDecoder } from 'node:util'; +import type { StreamingEvent } from '@dexto/core'; +import { + createTestAgent, + startTestServer, + httpRequest, + type TestServer, + expectResponseStructure, + validators, +} from './test-fixtures.js'; + +describe('Hono API Integration Tests', () => { + let testServer: TestServer | undefined; + + beforeAll(async () => { + const agent = await createTestAgent(); + testServer = await startTestServer(agent); + }, 30000); // 30 second timeout for server startup + + afterAll(async () => { + if (testServer) { + await testServer.cleanup(); + } + }); + + describe('Health', () => { + it('GET /health returns OK', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/health'); + expect(res.status).toBe(200); + expect(res.text).toBe('OK'); + }); + }); + + describe('LLM Routes', () => { + it('GET /api/llm/current returns current LLM config', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/llm/current'); + expect(res.status).toBe(200); + expectResponseStructure(res.body, { + config: validators.object, + }); + const config = ( + res.body as { + config: { + provider: string; + model: string; + displayName?: string; + }; + } + ).config; + expect(config.provider).toBe('openai'); + expect(config.model).toBe('gpt-5-nano'); + expect(typeof config.displayName === 'string' || config.displayName === undefined).toBe( + true + ); + }); + + it('GET /api/llm/current with sessionId returns session-specific config', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create a session first + const createRes = await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-session-llm', + }); + expect(createRes.status).toBe(201); + + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/llm/current?sessionId=test-session-llm' + ); + expect(res.status).toBe(200); + expect((res.body as { config: unknown }).config).toBeDefined(); + }); + + it('GET /api/llm/catalog returns LLM catalog', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/llm/catalog'); + expect(res.status).toBe(200); + expectResponseStructure(res.body, { + providers: validators.object, + }); + const providers = (res.body as { providers: Record }).providers; + expect(Object.keys(providers).length).toBeGreaterThan(0); + // Validate provider structure + const firstProvider = Object.values(providers)[0] as { + models: unknown; + }; + expect(firstProvider).toBeDefined(); + expect(typeof firstProvider === 'object').toBe(true); + }); + + it('POST /api/llm/switch validates input', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/llm/switch', {}); + expect(res.status).toBeGreaterThanOrEqual(400); + }); + + it('POST /api/llm/switch with model update succeeds', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/llm/switch', { + model: 'gpt-5', + }); + expect(res.status).toBe(200); + }); + }); + + describe('Sessions Routes', () => { + it('GET /api/sessions returns empty list initially', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/sessions'); + expect(res.status).toBe(200); + expectResponseStructure(res.body, { + sessions: validators.array, + }); + const sessions = (res.body as { sessions: unknown[] }).sessions; + // May have sessions from previous tests in integration suite + expect(sessions.length).toBeGreaterThanOrEqual(0); + }); + + it('POST /api/sessions creates a new session', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-session-1', + }); + expect(res.status).toBe(201); + expectResponseStructure(res.body, { + session: validators.object, + }); + const session = ( + res.body as { + session: { + id: string; + createdAt: number | null; + lastActivity: number | null; + messageCount: number; + title: string | null; + }; + } + ).session; + expect(session.id).toBe('test-session-1'); + expect(typeof session.messageCount).toBe('number'); + expect(session.createdAt === null || typeof session.createdAt === 'number').toBe(true); + }); + + it('GET /api/sessions/:id returns session details', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create session first + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-session-details', + }); + + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/sessions/test-session-details' + ); + expect(res.status).toBe(200); + expect((res.body as { session: { id: string } }).session.id).toBe( + 'test-session-details' + ); + }); + + it('GET /api/sessions/:id returns 404 for non-existent session', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/sessions/non-existent-session' + ); + expect(res.status).toBe(404); + }); + + it('GET /api/sessions/:id/load validates and returns session info', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create session first + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-session-load', + }); + + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/sessions/test-session-load/load' + ); + expect(res.status).toBe(200); + expect(res.body).toHaveProperty('session'); + expect((res.body as { session: { id: string } }).session.id).toBe('test-session-load'); + }); + + it('GET /api/sessions/:id/history returns session history', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create session first + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-session-history', + }); + + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/sessions/test-session-history/history' + ); + expect(res.status).toBe(200); + expect(Array.isArray((res.body as { history: unknown[] }).history)).toBe(true); + }); + + it('DELETE /api/sessions/:id deletes session', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create session first + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-session-delete', + }); + + const res = await httpRequest( + testServer.baseUrl, + 'DELETE', + '/api/sessions/test-session-delete' + ); + expect(res.status).toBe(200); + + // Verify deletion + const getRes = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/sessions/test-session-delete' + ); + expect(getRes.status).toBe(404); + }); + }); + + describe('Search Routes', () => { + it('GET /api/search/messages requires query parameter', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/search/messages'); + expect(res.status).toBe(400); + }); + + it('GET /api/search/messages with query returns results', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/search/messages?q=test'); + expect(res.status).toBe(200); + expect((res.body as { results: unknown[] }).results).toBeDefined(); + }); + + it('GET /api/search/sessions requires query parameter', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/search/sessions'); + expect(res.status).toBe(400); + }); + + it('GET /api/search/sessions with query returns results', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/search/sessions?q=test'); + expect(res.status).toBe(200); + expect((res.body as { results: unknown[] }).results).toBeDefined(); + }); + }); + + describe('Memory Routes', () => { + it('GET /api/memory returns empty list initially', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/memory'); + expect(res.status).toBe(200); + expect(Array.isArray((res.body as { memories: unknown[] }).memories)).toBe(true); + }); + + it('POST /api/memory creates a memory', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/memory', { + content: 'Test memory content', + tags: ['test'], + }); + expect(res.status).toBe(201); + expect((res.body as { memory: { id: string } }).memory.id).toBeDefined(); + expect((res.body as { memory: { content: string } }).memory.content).toBe( + 'Test memory content' + ); + }); + + it('POST /api/memory validates required fields', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/memory', {}); + expect(res.status).toBeGreaterThanOrEqual(400); + }); + + it('GET /api/memory/:id returns memory details', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create memory first + const createRes = await httpRequest(testServer.baseUrl, 'POST', '/api/memory', { + content: 'Memory to retrieve', + tags: ['test'], + }); + const memoryId = (createRes.body as { memory: { id: string } }).memory.id; + + const res = await httpRequest(testServer.baseUrl, 'GET', `/api/memory/${memoryId}`); + expect(res.status).toBe(200); + expect((res.body as { memory: { id: string } }).memory.id).toBe(memoryId); + }); + + it('PUT /api/memory/:id updates memory', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create memory first + const createRes = await httpRequest(testServer.baseUrl, 'POST', '/api/memory', { + content: 'Original content', + tags: ['test'], + }); + const memoryId = (createRes.body as { memory: { id: string } }).memory.id; + + const res = await httpRequest(testServer.baseUrl, 'PUT', `/api/memory/${memoryId}`, { + content: 'Updated content', + }); + expect(res.status).toBe(200); + expect((res.body as { memory: { content: string } }).memory.content).toBe( + 'Updated content' + ); + }); + + it('DELETE /api/memory/:id deletes memory', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create memory first + const createRes = await httpRequest(testServer.baseUrl, 'POST', '/api/memory', { + content: 'Memory to delete', + tags: ['test'], + }); + const memoryId = (createRes.body as { memory: { id: string } }).memory.id; + + const res = await httpRequest(testServer.baseUrl, 'DELETE', `/api/memory/${memoryId}`); + expect(res.status).toBe(200); + + // Verify deletion + const getRes = await httpRequest(testServer.baseUrl, 'GET', `/api/memory/${memoryId}`); + expect(getRes.status).toBeGreaterThanOrEqual(400); + }); + }); + + describe('MCP Routes', () => { + it('GET /api/mcp/servers returns server list', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/mcp/servers'); + expect(res.status).toBe(200); + expect(typeof res.body).toBe('object'); + }); + + it('POST /api/mcp/servers validates input', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/mcp/servers', {}); + expect(res.status).toBeGreaterThanOrEqual(400); + }); + }); + + describe('Prompts Routes', () => { + it('GET /api/prompts returns prompt list', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/prompts'); + expect(res.status).toBe(200); + expect(typeof res.body).toBe('object'); + }); + + it('GET /api/prompts/:name returns prompt details', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/prompts/non-existent-prompt' + ); + // May return 404 or empty result depending on implementation + expect([200, 404]).toContain(res.status); + }); + }); + + describe('Resources Routes', () => { + it('GET /api/resources returns resource list', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/resources'); + expect(res.status).toBe(200); + expect(typeof res.body).toBe('object'); + }); + }); + + describe('Webhooks Routes', () => { + it('GET /api/webhooks returns webhook list', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/webhooks'); + expect(res.status).toBe(200); + expect(Array.isArray((res.body as { webhooks: unknown[] }).webhooks)).toBe(true); + }); + + it('POST /api/webhooks validates URL', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/webhooks', { + url: 'not-a-url', + }); + expect(res.status).toBeGreaterThanOrEqual(400); + }); + + it('POST /api/webhooks creates webhook', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/webhooks', { + url: 'https://example.com/webhook', + }); + expect(res.status).toBe(201); + }); + }); + + describe('Greeting Route', () => { + it('GET /api/greeting returns greeting', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/api/greeting'); + expect(res.status).toBe(200); + // greeting might be undefined if not set in config, which is valid + expect(res.body).toBeDefined(); + expect( + typeof (res.body as { greeting?: unknown }).greeting === 'string' || + (res.body as { greeting?: unknown }).greeting === undefined + ).toBe(true); + }); + + it('GET /api/greeting with sessionId returns session-specific greeting', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create session first + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-session-greeting', + }); + + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/greeting?sessionId=test-session-greeting' + ); + expect(res.status).toBe(200); + // greeting might be undefined if not set in config, which is valid + expect(res.body).toBeDefined(); + expect( + typeof (res.body as { greeting?: unknown }).greeting === 'string' || + (res.body as { greeting?: unknown }).greeting === undefined + ).toBe(true); + }); + }); + + describe('A2A Routes', () => { + it('GET /.well-known/agent-card.json returns agent card', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/.well-known/agent-card.json' + ); + expect(res.status).toBe(200); + expect((res.body as { name: unknown }).name).toBeDefined(); + }); + }); + + describe('Message Routes', () => { + it('POST /api/message validates input', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/message', {}); + expect(res.status).toBeGreaterThanOrEqual(400); + }); + + it('POST /api/message-sync validates input', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/message-sync', {}); + expect(res.status).toBeGreaterThanOrEqual(400); + }); + + it('POST /api/reset resets conversation', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create session first + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-session-reset', + }); + const res = await httpRequest(testServer.baseUrl, 'POST', '/api/reset', { + sessionId: 'test-session-reset', + }); + expect(res.status).toBe(200); + }); + + it('POST /api/message-stream returns SSE stream directly', async () => { + if (!testServer) throw new Error('Test server not initialized'); + + const sessionId = 'stream-session'; + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { sessionId }); + + const agent = testServer.agent; + const originalStream = agent.stream; + const fakeEvents: StreamingEvent[] = [ + { + name: 'llm:thinking', + sessionId, + }, + { + name: 'llm:chunk', + content: 'hello', + chunkType: 'text', + isComplete: false, + sessionId, + }, + { + name: 'llm:response', + content: 'hello', + tokenUsage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + sessionId, + provider: 'openai', + model: 'test-model', + }, + ]; + + agent.stream = async function ( + _message: string, + _options + ): Promise> { + async function* generator() { + for (const event of fakeEvents) { + yield event; + } + } + return generator(); + } as typeof agent.stream; + + try { + // POST to /api/message-stream - response IS the SSE stream + const response = await fetch(`${testServer.baseUrl}/api/message-stream`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + sessionId, + content: 'Say hello', + }), + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toBe('text/event-stream'); + + const reader = response.body?.getReader(); + if (!reader) throw new Error('Response does not contain a readable body'); + + const decoder = new TextDecoder(); + let received = ''; + let chunks = 0; + while (chunks < 50) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks++; + received += decoder.decode(value, { stream: true }); + if (received.includes('event: llm:response')) { + break; + } + } + + await reader.cancel(); + + expect(received).toContain('event: llm:thinking'); + expect(received).toContain('event: llm:response'); + } finally { + agent.stream = originalStream; + } + }); + }); + + describe('Queue Routes', () => { + it('GET /api/queue/:sessionId returns empty queue initially', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create session first + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-queue-session', + }); + + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/queue/test-queue-session' + ); + expect(res.status).toBe(200); + expect((res.body as { messages: unknown[]; count: number }).messages).toEqual([]); + expect((res.body as { count: number }).count).toBe(0); + }); + + it('GET /api/queue/:sessionId returns 404 for non-existent session', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/queue/non-existent-queue-session' + ); + expect(res.status).toBe(404); + }); + + it('POST /api/queue/:sessionId queues a message', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create session first + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-queue-post-session', + }); + + const res = await httpRequest( + testServer.baseUrl, + 'POST', + '/api/queue/test-queue-post-session', + { content: 'Hello from queue' } + ); + expect(res.status).toBe(201); + expect((res.body as { queued: boolean }).queued).toBe(true); + expect((res.body as { id: string }).id).toBeDefined(); + expect((res.body as { position: number }).position).toBe(1); + + // Verify message is in queue + const getRes = await httpRequest( + testServer.baseUrl, + 'GET', + '/api/queue/test-queue-post-session' + ); + expect((getRes.body as { count: number }).count).toBe(1); + }); + + it('POST /api/queue/:sessionId validates input', async () => { + if (!testServer) throw new Error('Test server not initialized'); + // Create session first + await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId: 'test-queue-validate-session', + }); + + const res = await httpRequest( + testServer.baseUrl, + 'POST', + '/api/queue/test-queue-validate-session', + {} // Empty body should fail validation + ); + expect(res.status).toBeGreaterThanOrEqual(400); + }); + + it('DELETE /api/queue/:sessionId/:messageId removes a queued message', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const sessionId = `queue-delete-msg-${Date.now()}`; + + // Create session and queue a message + const createRes = await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId, + }); + expect(createRes.status).toBe(201); + + const queueRes = await httpRequest( + testServer.baseUrl, + 'POST', + `/api/queue/${sessionId}`, + { content: 'Message to delete' } + ); + expect(queueRes.status).toBe(201); + const messageId = (queueRes.body as { id: string }).id; + + // Delete the message + const res = await httpRequest( + testServer.baseUrl, + 'DELETE', + `/api/queue/${sessionId}/${messageId}` + ); + expect(res.status).toBe(200); + expect((res.body as { removed: boolean }).removed).toBe(true); + + // Verify queue is empty + const getRes = await httpRequest(testServer.baseUrl, 'GET', `/api/queue/${sessionId}`); + expect((getRes.body as { count: number }).count).toBe(0); + }); + + it('DELETE /api/queue/:sessionId clears all queued messages', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const sessionId = `queue-clear-${Date.now()}`; + + // Create session and queue multiple messages + const createRes = await httpRequest(testServer.baseUrl, 'POST', '/api/sessions', { + sessionId, + }); + expect(createRes.status).toBe(201); + + const q1 = await httpRequest(testServer.baseUrl, 'POST', `/api/queue/${sessionId}`, { + content: 'Message 1', + }); + expect(q1.status).toBe(201); + const q2 = await httpRequest(testServer.baseUrl, 'POST', `/api/queue/${sessionId}`, { + content: 'Message 2', + }); + expect(q2.status).toBe(201); + + // Clear the queue + const res = await httpRequest(testServer.baseUrl, 'DELETE', `/api/queue/${sessionId}`); + expect(res.status).toBe(200); + expect((res.body as { cleared: boolean }).cleared).toBe(true); + expect((res.body as { count: number }).count).toBe(2); + + // Verify queue is empty + const getRes = await httpRequest(testServer.baseUrl, 'GET', `/api/queue/${sessionId}`); + expect((getRes.body as { count: number }).count).toBe(0); + }); + }); + + describe('OpenAPI Schema', () => { + it('GET /openapi.json returns OpenAPI schema', async () => { + if (!testServer) throw new Error('Test server not initialized'); + const res = await httpRequest(testServer.baseUrl, 'GET', '/openapi.json'); + expect(res.status).toBe(200); + expect((res.body as { openapi: string }).openapi).toBe('3.0.0'); + }); + }); +}); diff --git a/dexto/packages/server/src/hono/__tests__/test-fixtures.ts b/dexto/packages/server/src/hono/__tests__/test-fixtures.ts new file mode 100644 index 00000000..c68cef57 --- /dev/null +++ b/dexto/packages/server/src/hono/__tests__/test-fixtures.ts @@ -0,0 +1,294 @@ +import { DextoAgent, createAgentCard } from '@dexto/core'; +import type { AgentConfig, AgentCard } from '@dexto/core'; +import type { Server as HttpServer } from 'node:http'; +import type { Context } from 'hono'; +import { createDextoApp } from '../index.js'; +import type { DextoApp } from '../types.js'; +import { createNodeServer, type NodeBridgeResult } from '../node/index.js'; +import type { CreateDextoAppOptions } from '../index.js'; + +/** + * Test configuration for integration tests + * Uses in-memory storage to avoid side effects + */ +export function createTestAgentConfig(): AgentConfig { + return { + systemPrompt: 'You are a test assistant.', + llm: { + provider: 'openai', + model: 'gpt-5-nano', + apiKey: 'test-key-123', // Mock key for testing + maxIterations: 10, + }, + mcpServers: {}, + storage: { + cache: { type: 'in-memory' }, + database: { type: 'in-memory' }, + blob: { type: 'local', storePath: '/tmp/test-blobs' }, + }, + sessions: { + maxSessions: 50, // Increased to accommodate all integration tests + sessionTTL: 3600, + }, + toolConfirmation: { + mode: 'auto-approve', + timeout: 120000, + }, + elicitation: { + enabled: false, + timeout: 120000, + }, + }; +} + +/** + * Creates a real DextoAgent instance with in-memory storage + * No mocks - uses real implementations + */ +export async function createTestAgent(config?: AgentConfig): Promise { + const agentConfig = config ?? createTestAgentConfig(); + const agent = new DextoAgent(agentConfig); + await agent.start(); + return agent; +} + +/** + * Test server setup result + */ +export interface TestServer { + server: HttpServer; + app: DextoApp; + bridge: NodeBridgeResult; + agent: DextoAgent; + agentCard: AgentCard; + baseUrl: string; + port: number; + cleanup: () => Promise; +} + +/** + * Starts a real HTTP server for testing + * Uses createDextoApp and createNodeServer directly + * @param agent - The agent instance to use + * @param port - Optional port (auto-selected if not provided) + * @param agentsContext - Optional agent switching context (enables /api/agents routes) + */ +export async function startTestServer( + agent: DextoAgent, + port?: number, + agentsContext?: CreateDextoAppOptions['agentsContext'] +): Promise { + // Use provided port or find an available port + const serverPort = port ?? (await findAvailablePort()); + + // Create agent card + const agentCard = createAgentCard({ + defaultName: 'test-agent', + defaultVersion: '1.0.0', + defaultBaseUrl: `http://localhost:${serverPort}`, + }); + + // Create getter functions + // Note: For agent switching tests, getAgent needs to reference activeAgent from agentsContext + // This is handled by the agentsContext implementation itself + const getAgent = (_ctx: Context) => agent; + const getAgentCard = () => agentCard; + + // Create event subscribers and approval coordinator for test + const { WebhookEventSubscriber } = await import('../../events/webhook-subscriber.js'); + const { A2ASseEventSubscriber } = await import('../../events/a2a-sse-subscriber.js'); + const { ApprovalCoordinator } = await import('../../approval/approval-coordinator.js'); + + const webhookSubscriber = new WebhookEventSubscriber(); + const sseSubscriber = new A2ASseEventSubscriber(); + const approvalCoordinator = new ApprovalCoordinator(); + + // Subscribe to agent's event bus + webhookSubscriber.subscribe(agent.agentEventBus); + sseSubscriber.subscribe(agent.agentEventBus); + + // Create Hono app + const app = createDextoApp({ + getAgent, + getAgentCard, + approvalCoordinator, + webhookSubscriber, + sseSubscriber, + ...(agentsContext ? { agentsContext } : {}), // Include agentsContext only if provided + }); + + // Create Node server bridge + const bridge = createNodeServer(app, { + getAgent: () => agent, + port: serverPort, + }); + + // Agent card (no updates needed after bridge creation in SSE migration) + const updatedAgentCard = createAgentCard({ + defaultName: 'test-agent', + defaultVersion: '1.0.0', + defaultBaseUrl: `http://localhost:${serverPort}`, + }); + + // Start the server + await new Promise((resolve, reject) => { + bridge.server.listen(serverPort, '0.0.0.0', () => { + resolve(); + }); + bridge.server.on('error', reject); + }); + + const baseUrl = `http://localhost:${serverPort}`; + + return { + server: bridge.server, + app, + bridge, + agent, + agentCard: updatedAgentCard, + baseUrl, + port: serverPort, + cleanup: async () => { + // Cleanup subscribers to prevent memory leaks + webhookSubscriber.cleanup(); + sseSubscriber.cleanup(); + approvalCoordinator.removeAllListeners(); + + await new Promise((resolve, reject) => { + bridge.server.close((err) => { + if (err) reject(err); + else resolve(); + }); + }); + if (agent.isStarted()) { + await agent.stop(); + } + }, + }; +} + +/** + * Finds an available port starting from a random port in the ephemeral range + * Uses ports 49152-65535 (IANA ephemeral port range) + */ +async function findAvailablePort(): Promise { + const { createServer } = await import('node:http'); + // Start from a random port in the ephemeral range to avoid conflicts + const startPort = 49152 + Math.floor(Math.random() * 1000); + + for (let port = startPort; port < 65535; port++) { + try { + await new Promise((resolve, reject) => { + const server = createServer(); + server.on('error', (err: NodeJS.ErrnoException) => { + if (err.code === 'EADDRINUSE') { + reject(new Error(`Port ${port} is in use`)); + } else { + reject(err); + } + }); + server.listen(port, () => { + server.close(() => resolve()); + }); + }); + return port; + } catch { + // Port is in use, try next + continue; + } + } + throw new Error(`Could not find an available port starting from ${startPort}`); +} + +/** + * Helper to make HTTP requests to the test server + */ +export async function httpRequest( + baseUrl: string, + method: string, + path: string, + body?: unknown, + headers?: Record +): Promise<{ + status: number; + headers: Record; + body: unknown; + text: string; +}> { + const url = `${baseUrl}${path}`; + const options: RequestInit = { + method, + headers: { + 'Content-Type': 'application/json', + ...headers, + }, + }; + + if (body !== undefined) { + options.body = JSON.stringify(body); + } + + const response = await fetch(url, options); + const text = await response.text(); + let parsedBody: unknown; + try { + parsedBody = JSON.parse(text); + } catch { + parsedBody = text; + } + + // Convert Headers to plain object for serialization + const headersObject: Record = {}; + response.headers.forEach((value, key) => { + headersObject[key] = value; + }); + + return { + status: response.status, + headers: headersObject, + body: parsedBody, + text, + }; +} + +/** + * Validates that a response has the expected structure + */ +export function expectResponseStructure( + body: unknown, + schema: Record boolean> +): void { + if (typeof body !== 'object' || body === null) { + throw new Error(`Expected object response, got ${typeof body}`); + } + + const bodyObj = body as Record; + for (const [key, validator] of Object.entries(schema)) { + if (!(key in bodyObj)) { + throw new Error(`Missing required field: ${key}`); + } + if (!validator(bodyObj[key])) { + throw new Error( + `Invalid type for field '${key}': expected validator to return true, got false` + ); + } + } +} + +/** + * Common response validators + */ +export const validators = { + string: (value: unknown): boolean => typeof value === 'string', + number: (value: unknown): boolean => typeof value === 'number', + boolean: (value: unknown): boolean => typeof value === 'boolean', + array: (value: unknown): boolean => Array.isArray(value), + object: (value: unknown): boolean => + typeof value === 'object' && value !== null && !Array.isArray(value), + optionalString: (value: unknown): boolean => value === undefined || typeof value === 'string', + optionalNumber: (value: unknown): boolean => value === undefined || typeof value === 'number', + optionalArray: (value: unknown): boolean => value === undefined || Array.isArray(value), + optionalObject: (value: unknown): boolean => + value === undefined || + (typeof value === 'object' && value !== null && !Array.isArray(value)), +}; diff --git a/dexto/packages/server/src/hono/index.ts b/dexto/packages/server/src/hono/index.ts new file mode 100644 index 00000000..44c6d6d0 --- /dev/null +++ b/dexto/packages/server/src/hono/index.ts @@ -0,0 +1,305 @@ +import { OpenAPIHono } from '@hono/zod-openapi'; +import type { Context } from 'hono'; +import type { DextoAgent, AgentCard } from '@dexto/core'; +import { logger } from '@dexto/core'; +import { createHealthRouter } from './routes/health.js'; +import { createGreetingRouter } from './routes/greeting.js'; +import { createMessagesRouter } from './routes/messages.js'; +import { createLlmRouter } from './routes/llm.js'; +import { createSessionsRouter } from './routes/sessions.js'; +import { createSearchRouter } from './routes/search.js'; +import { createMcpRouter } from './routes/mcp.js'; +import { createA2aRouter } from './routes/a2a.js'; +import { createA2AJsonRpcRouter } from './routes/a2a-jsonrpc.js'; +import { createA2ATasksRouter } from './routes/a2a-tasks.js'; +import { createWebhooksRouter } from './routes/webhooks.js'; +import { createPromptsRouter } from './routes/prompts.js'; +import { createResourcesRouter } from './routes/resources.js'; +import { createMemoryRouter } from './routes/memory.js'; +import { createAgentsRouter, type AgentsRouterContext } from './routes/agents.js'; +import { createApprovalsRouter } from './routes/approvals.js'; +import { createQueueRouter } from './routes/queue.js'; +import { createOpenRouterRouter } from './routes/openrouter.js'; +import { createKeyRouter } from './routes/key.js'; +import { createToolsRouter } from './routes/tools.js'; +import { createDiscoveryRouter } from './routes/discovery.js'; +import { createModelsRouter } from './routes/models.js'; +import { createDextoAuthRouter } from './routes/dexto-auth.js'; +import { + createStaticRouter, + createSpaFallbackHandler, + type WebUIRuntimeConfig, +} from './routes/static.js'; +import { WebhookEventSubscriber } from '../events/webhook-subscriber.js'; +import { A2ASseEventSubscriber } from '../events/a2a-sse-subscriber.js'; +import { handleHonoError } from './middleware/error.js'; +import { prettyJsonMiddleware, redactionMiddleware } from './middleware/redaction.js'; +import { createCorsMiddleware } from './middleware/cors.js'; +import { createAuthMiddleware } from './middleware/auth.js'; +import { ApprovalCoordinator } from '../approval/approval-coordinator.js'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const packageJson = JSON.parse(readFileSync(join(__dirname, '../../package.json'), 'utf-8')) as { + version: string; +}; + +// Dummy context for type inference and runtime fallback +// Used when running in single-agent mode (CLI, Docker, etc.) where multi-agent +// features aren't available. Agents router is always mounted for consistent API +// structure, but will return clear errors if multi-agent endpoints are called. +// This ensures type safety across different deployment modes. +const dummyAgentsContext: AgentsRouterContext = { + switchAgentById: async () => { + throw new Error('Multi-agent features not available in single-agent mode'); + }, + switchAgentByPath: async () => { + throw new Error('Multi-agent features not available in single-agent mode'); + }, + resolveAgentInfo: async () => { + throw new Error('Multi-agent features not available in single-agent mode'); + }, + ensureAgentAvailable: () => {}, + getActiveAgentId: () => undefined, +}; + +// Type for async getAgent with context support +export type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +export type CreateDextoAppOptions = { + /** + * Prefix for API routes. Defaults to '/api'. + */ + apiPrefix?: string; + getAgent: GetAgentFn; + getAgentCard: () => AgentCard; + approvalCoordinator: ApprovalCoordinator; + webhookSubscriber: WebhookEventSubscriber; + sseSubscriber: A2ASseEventSubscriber; + agentsContext?: AgentsRouterContext; + /** Absolute path to WebUI build output. If provided, static files will be served. */ + webRoot?: string; + /** Runtime configuration to inject into WebUI (analytics, etc.) */ + webUIConfig?: WebUIRuntimeConfig; + /** Disable built-in auth middleware. Use when you have your own auth layer. */ + disableAuth?: boolean; +}; + +// Default API prefix as a const literal for type inference +const DEFAULT_API_PREFIX = '/api' as const; + +export function createDextoApp(options: CreateDextoAppOptions) { + const { + apiPrefix, + getAgent, + getAgentCard, + approvalCoordinator, + webhookSubscriber, + sseSubscriber, + agentsContext, + webRoot, + webUIConfig, + disableAuth = false, + } = options; + + // Security check: Warn when auth is disabled + if (disableAuth) { + logger.warn( + `⚠️ Authentication disabled (disableAuth=true). createAuthMiddleware() skipped. Ensure external auth is in place.` + ); + } + + const app = new OpenAPIHono({ strict: false }); + + // Global CORS middleware for cross-origin requests (must be first) + app.use('*', createCorsMiddleware()); + + // Global authentication middleware (after CORS, before routes) + // Can be disabled when using an external auth layer + if (!disableAuth) { + app.use('*', createAuthMiddleware()); + } + + // Global error handling for all routes + app.onError((err, ctx) => handleHonoError(ctx, err)); + + // Normalize prefix: strip trailing slashes, treat '' as '/' + const rawPrefix = apiPrefix ?? DEFAULT_API_PREFIX; + const normalizedPrefix = rawPrefix === '' ? '/' : rawPrefix.replace(/\/+$/, '') || '/'; + const middlewarePattern = normalizedPrefix === '/' ? '/*' : `${normalizedPrefix}/*`; + + app.use(middlewarePattern, prettyJsonMiddleware); + app.use(middlewarePattern, redactionMiddleware); + + // Cast to literal type for RPC client type inference (webui uses default '/api') + const routePrefix = normalizedPrefix as typeof DEFAULT_API_PREFIX; + + // Mount all API routers at the configured prefix for proper type inference + // Each router is mounted individually so Hono can properly track route types + const fullApp = app + // Public health endpoint + .route('/health', createHealthRouter(getAgent)) + // Follows A2A discovery protocol + .route('/', createA2aRouter(getAgentCard)) + .route('/', createA2AJsonRpcRouter(getAgent, sseSubscriber)) + .route('/', createA2ATasksRouter(getAgent, sseSubscriber)) + // Add agent-specific routes + .route(routePrefix, createGreetingRouter(getAgent)) + .route(routePrefix, createMessagesRouter(getAgent, approvalCoordinator)) + .route(routePrefix, createLlmRouter(getAgent)) + .route(routePrefix, createSessionsRouter(getAgent)) + .route(routePrefix, createSearchRouter(getAgent)) + .route(routePrefix, createMcpRouter(getAgent)) + .route(routePrefix, createWebhooksRouter(getAgent, webhookSubscriber)) + .route(routePrefix, createPromptsRouter(getAgent)) + .route(routePrefix, createResourcesRouter(getAgent)) + .route(routePrefix, createMemoryRouter(getAgent)) + .route(routePrefix, createApprovalsRouter(getAgent, approvalCoordinator)) + .route(routePrefix, createAgentsRouter(getAgent, agentsContext || dummyAgentsContext)) + .route(routePrefix, createQueueRouter(getAgent)) + .route(routePrefix, createOpenRouterRouter()) + .route(routePrefix, createKeyRouter()) + .route(routePrefix, createToolsRouter(getAgent)) + .route(routePrefix, createDiscoveryRouter()) + .route(routePrefix, createModelsRouter()) + .route(routePrefix, createDextoAuthRouter(getAgent)); + + // Expose OpenAPI document + // Current approach uses @hono/zod-openapi's .doc() method for OpenAPI spec generation + // Alternative: Use openAPIRouteHandler from hono-openapi (third-party) for auto-generation + // Keeping current approach since: + // 1. @hono/zod-openapi is official Hono package with first-class support + // 2. We already generate spec via scripts/generate-openapi-spec.ts to docs/ + // 3. Switching would require adding hono-openapi dependency and migration effort + // See: https://honohub.dev/docs/openapi/zod#generating-the-openapi-spec + fullApp.doc('/openapi.json', { + openapi: '3.0.0', + info: { + title: 'Dexto API', + version: packageJson.version, + description: 'OpenAPI spec for the Dexto REST API server', + }, + servers: [ + { + url: 'http://localhost:3001', + description: 'Local development server (default port)', + }, + { + url: 'http://localhost:{port}', + description: 'Local development server (custom port)', + variables: { + port: { + default: '3001', + description: 'API server port', + }, + }, + }, + ], + tags: [ + { + name: 'system', + description: 'System health and status endpoints', + }, + { + name: 'config', + description: 'Agent configuration and greeting management', + }, + { + name: 'messages', + description: 'Send messages to the agent and manage conversations', + }, + { + name: 'sessions', + description: 'Create and manage conversation sessions', + }, + { + name: 'llm', + description: 'Configure and switch between LLM providers and models', + }, + { + name: 'mcp', + description: 'Manage Model Context Protocol (MCP) servers and tools', + }, + { + name: 'webhooks', + description: 'Register and manage webhook endpoints for agent events', + }, + { + name: 'search', + description: 'Search through messages and sessions', + }, + { + name: 'memory', + description: 'Store and retrieve agent memories for context', + }, + { + name: 'prompts', + description: 'Manage custom prompts and templates', + }, + { + name: 'resources', + description: 'Access and manage resources from MCP servers and internal providers', + }, + { + name: 'agent', + description: 'Current agent configuration and file operations', + }, + { + name: 'agents', + description: 'Install, switch, and manage agent configurations', + }, + { + name: 'queue', + description: 'Manage message queue for busy sessions', + }, + { + name: 'openrouter', + description: 'OpenRouter model validation and cache management', + }, + { + name: 'discovery', + description: 'Discover available providers and capabilities', + }, + { + name: 'tools', + description: + 'List and inspect available tools from internal, custom, and MCP sources', + }, + { + name: 'models', + description: 'List and manage local GGUF models and Ollama models', + }, + { + name: 'auth', + description: 'Dexto authentication status and management', + }, + ], + }); + + // Mount static file router for WebUI if webRoot is provided + if (webRoot) { + fullApp.route('/', createStaticRouter(webRoot)); + // SPA fallback: serve index.html for unmatched routes without file extensions + // Must be registered as notFound handler so it runs AFTER all routes (including /openapi.json) + // webUIConfig is injected into index.html for runtime configuration (analytics, etc.) + fullApp.notFound(createSpaFallbackHandler(webRoot, webUIConfig)); + } + + // NOTE: Subscribers and approval handler are wired in CLI layer before agent.start() + // This ensures proper initialization order and validation + // We attach webhookSubscriber as a property but don't include it in the return type + // to preserve Hono's route type inference + Object.assign(fullApp, { webhookSubscriber }); + + return fullApp; +} + +// Export inferred AppType +// Routes are now properly typed since they're all mounted directly +export type AppType = ReturnType; + +// Re-export types needed by CLI +export type { WebUIRuntimeConfig } from './routes/static.js'; diff --git a/dexto/packages/server/src/hono/middleware/auth.ts b/dexto/packages/server/src/hono/middleware/auth.ts new file mode 100644 index 00000000..fd76e589 --- /dev/null +++ b/dexto/packages/server/src/hono/middleware/auth.ts @@ -0,0 +1,89 @@ +import type { MiddlewareHandler } from 'hono'; +import { logger } from '@dexto/core'; + +/** + * Authentication middleware for API security + * + * Security model: + * 1. Default (no env): Development mode - no auth required + * 2. NODE_ENV=production: Production mode - auth required + * 3. DEXTO_SERVER_REQUIRE_AUTH=true: Explicit auth enforcement + * 4. Public routes (health check, A2A discovery) are always accessible + * + * Usage: + * Development (default): + * npm start # No auth needed, existing scripts work + * + * Production: + * DEXTO_SERVER_API_KEY=your-key NODE_ENV=production npm start + * Clients must send: Authorization: Bearer + */ + +const PUBLIC_ROUTES = ['/health', '/.well-known/agent-card.json', '/openapi.json']; + +export function createAuthMiddleware(): MiddlewareHandler { + const apiKey = process.env.DEXTO_SERVER_API_KEY; + const isProduction = process.env.NODE_ENV === 'production'; + const requireAuth = process.env.DEXTO_SERVER_REQUIRE_AUTH === 'true'; // Explicit opt-in + + // Log security configuration on startup + if (isProduction && !apiKey) { + logger.warn( + `⚠️ SECURITY WARNING: Running in production mode (NODE_ENV=production) without DEXTO_SERVER_API_KEY. Dexto Server API is UNPROTECTED. Set DEXTO_SERVER_API_KEY environment variable to secure your API.` + ); + } + + return async (ctx, next) => { + const path = ctx.req.path; + + // Always allow public routes + if (PUBLIC_ROUTES.some((route) => path === route || path.startsWith(route))) { + return next(); + } + + // Default behavior: Development mode (no auth required) + // This ensures existing dev scripts don't break + if (!isProduction && !requireAuth) { + return next(); + } + + // Production mode or explicit DEXTO_SERVER_REQUIRE_AUTH=true + // Requires API key to be set - fail closed for security + if (!apiKey) { + return ctx.json( + { + error: 'Configuration Error', + message: requireAuth + ? 'DEXTO_SERVER_REQUIRE_AUTH=true but DEXTO_SERVER_API_KEY not set. Set DEXTO_SERVER_API_KEY environment variable.' + : 'NODE_ENV=production requires DEXTO_SERVER_API_KEY. Set DEXTO_SERVER_API_KEY environment variable to secure your API.', + }, + 500 + ); + } + + // API key is set - validate it + const authHeader = ctx.req.header('Authorization'); + const providedKey = authHeader?.replace(/^Bearer\s+/i, ''); + + if (!providedKey || providedKey !== apiKey) { + logger.warn('Unauthorized API access attempt', { + path, + hasKey: !!providedKey, + origin: ctx.req.header('origin'), + userAgent: ctx.req.header('user-agent'), + }); + + return ctx.json( + { + error: 'Unauthorized', + message: + 'Invalid or missing API key. Provide Authorization: Bearer header.', + }, + 401 + ); + } + + // Valid API key - proceed + await next(); + }; +} diff --git a/dexto/packages/server/src/hono/middleware/cors.ts b/dexto/packages/server/src/hono/middleware/cors.ts new file mode 100644 index 00000000..e18b15fd --- /dev/null +++ b/dexto/packages/server/src/hono/middleware/cors.ts @@ -0,0 +1,49 @@ +import { cors } from 'hono/cors'; +import type { MiddlewareHandler } from 'hono'; + +/** + * CORS middleware that allows: + * 1. All localhost/127.0.0.1 origins on any port (for local development) + * 2. Custom origins specified in DEXTO_ALLOWED_ORIGINS environment variable + * 3. Server-to-server requests with no origin header + */ +export function createCorsMiddleware(): MiddlewareHandler { + return cors({ + origin: (origin) => { + // If no origin header (server-to-server), omit CORS headers + // Returning null allows the request without Access-Control-Allow-Origin header + // This is compatible with credentials: true (unlike '*') + if (!origin) { + return null; + } + + try { + const originUrl = new URL(origin); + const hostname = originUrl.hostname; + + // Always allow localhost/127.0.0.1 on any port for local development + if (hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1') { + return origin; + } + + // Check custom allowed origins from environment variable + const customOrigins = process.env.DEXTO_ALLOWED_ORIGINS; + if (customOrigins) { + const allowedList = customOrigins.split(',').map((o) => o.trim()); + if (allowedList.includes(origin)) { + return origin; + } + } + + // Origin not allowed + return null; + } catch { + // Invalid URL format, reject + return null; + } + }, + allowMethods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS', 'HEAD'], + allowHeaders: ['Content-Type', 'Authorization'], + credentials: true, + }); +} diff --git a/dexto/packages/server/src/hono/middleware/error.ts b/dexto/packages/server/src/hono/middleware/error.ts new file mode 100644 index 00000000..1bf8fcfa --- /dev/null +++ b/dexto/packages/server/src/hono/middleware/error.ts @@ -0,0 +1,129 @@ +import { DextoRuntimeError, DextoValidationError, ErrorType, zodToIssues } from '@dexto/core'; +import { logger } from '@dexto/core'; +import { ZodError } from 'zod'; + +// TODO: Standardize error responses across all server routes. +// Currently, routes use inconsistent error response formats: +// - Some throw typed errors (approvals.ts, prompts.ts) → middleware handles → standard format +// - Others return ad-hoc shapes like { error: '...' } or { ok: false, error: '...' } +// (mcp.ts, webhooks.ts, sessions.ts, queue.ts, a2a-tasks.ts) +// +// Target: All routes should throw DextoRuntimeError/DextoValidationError for errors, +// letting this middleware handle conversion to the standard response format. +// See also: packages/server/src/hono/schemas/responses.ts for OpenAPI schema limitations. + +export const mapErrorTypeToStatus = (type: ErrorType): number => { + switch (type) { + case ErrorType.USER: + return 400; + case ErrorType.PAYMENT_REQUIRED: + return 402; + case ErrorType.FORBIDDEN: + return 403; + case ErrorType.NOT_FOUND: + return 404; + case ErrorType.TIMEOUT: + return 408; + case ErrorType.CONFLICT: + return 409; + case ErrorType.RATE_LIMIT: + return 429; + case ErrorType.SYSTEM: + return 500; + case ErrorType.THIRD_PARTY: + return 502; + case ErrorType.UNKNOWN: + default: + return 500; + } +}; + +export const statusForValidation = (issues: ReturnType): number => { + const firstError = issues.find((i) => i.severity === 'error'); + const type = firstError?.type ?? ErrorType.USER; + return mapErrorTypeToStatus(type); +}; + +export function handleHonoError(ctx: any, err: unknown) { + // Extract endpoint information for better error context + const endpoint = ctx.req.path || 'unknown'; + const method = ctx.req.method || 'unknown'; + + if (err instanceof DextoRuntimeError) { + return ctx.json( + { + ...err.toJSON(), + endpoint, + method, + }, + mapErrorTypeToStatus(err.type) + ); + } + + if (err instanceof DextoValidationError) { + return ctx.json( + { + ...err.toJSON(), + endpoint, + method, + }, + statusForValidation(err.issues) + ); + } + + if (err instanceof ZodError) { + const issues = zodToIssues(err); + const dexErr = new DextoValidationError(issues); + return ctx.json( + { + ...dexErr.toJSON(), + endpoint, + method, + }, + statusForValidation(issues) + ); + } + + // Some hono specific handlers (e.g., ctx.req.json()) may throw SyntaxError for invalid/empty JSON + if (err instanceof SyntaxError) { + return ctx.json( + { + code: 'invalid_json', + message: err.message || 'Invalid JSON body', + scope: 'agent', + type: 'user', + severity: 'error', + endpoint, + method, + }, + 400 + ); + } + + const errorMessage = err instanceof Error ? err.message : String(err); + const errorStack = err instanceof Error ? err.stack : undefined; + logger.error( + `Unhandled error in API middleware: ${errorMessage}, endpoint: ${method} ${endpoint}, stack: ${errorStack}, type: ${typeof err}` + ); + + // Only expose error details in development, use generic message in production + const isDevelopment = process.env.NODE_ENV === 'development'; + const userMessage = isDevelopment + ? `An unexpected error occurred: ${errorMessage}` + : 'An unexpected error occurred. Please try again later.'; + + return ctx.json( + { + code: 'internal_error', + message: userMessage, + scope: 'system', + type: 'system', + severity: 'error', + endpoint, + method, + // Only include stack traces in development to avoid exposing internals + ...(isDevelopment && errorStack ? { stack: errorStack } : {}), + }, + 500 + ); +} diff --git a/dexto/packages/server/src/hono/middleware/redaction.ts b/dexto/packages/server/src/hono/middleware/redaction.ts new file mode 100644 index 00000000..9ed20f9b --- /dev/null +++ b/dexto/packages/server/src/hono/middleware/redaction.ts @@ -0,0 +1,22 @@ +import { prettyJSON } from 'hono/pretty-json'; +import type { MiddlewareHandler } from 'hono'; +import { redactSensitiveData } from '@dexto/core'; + +export const prettyJsonMiddleware = prettyJSON(); + +export const redactionMiddleware: MiddlewareHandler = async (ctx, next) => { + // TODO: tighten types once Hono exposes typed overrides for ctx.json/ctx.body + const originalJson = ctx.json.bind(ctx) as any; + ctx.json = ((data: any, status?: any, headers?: any) => { + const redacted = redactSensitiveData(data); + return originalJson(redacted, status, headers); + }) as typeof ctx.json; + + const originalBody = ctx.body.bind(ctx) as any; + ctx.body = ((data: any, status?: any, headers?: any) => { + const payload = typeof data === 'string' ? redactSensitiveData(data) : data; + return originalBody(payload, status, headers); + }) as typeof ctx.body; + + await next(); +}; diff --git a/dexto/packages/server/src/hono/node/index.ts b/dexto/packages/server/src/hono/node/index.ts new file mode 100644 index 00000000..b5ad5eb5 --- /dev/null +++ b/dexto/packages/server/src/hono/node/index.ts @@ -0,0 +1,154 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { Readable } from 'node:stream'; +import type { ReadableStream as NodeReadableStream } from 'stream/web'; +import type { DextoApp } from '../types.js'; +import type { DextoAgent } from '@dexto/core'; +import { logger } from '@dexto/core'; +import type { WebhookEventSubscriber } from '../../events/webhook-subscriber.js'; + +type FetchRequest = globalThis.Request; +type FetchBodyInit = globalThis.BodyInit; + +export type NodeBridgeOptions = { + getAgent: () => DextoAgent; + port?: number; + hostname?: string; + mcpHandlers?: { + handlePost: ( + req: IncomingMessage, + res: ServerResponse, + body: unknown + ) => Promise | void; + handleGet: (req: IncomingMessage, res: ServerResponse) => Promise | void; + } | null; +}; + +export type NodeBridgeResult = { + server: ReturnType; + webhookSubscriber?: WebhookEventSubscriber; +}; + +export function createNodeServer(app: DextoApp, options: NodeBridgeOptions): NodeBridgeResult { + const { getAgent: _getAgent } = options; + const webhookSubscriber = app.webhookSubscriber; + + const server = createServer(async (req, res) => { + try { + if (options.mcpHandlers && req.url?.startsWith('/mcp')) { + if (req.method === 'GET') { + await options.mcpHandlers.handleGet(req, res); + return; + } + if (req.method === 'POST') { + req.setEncoding('utf8'); + let body = ''; + const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10MB limit + req.on('data', (chunk) => { + body += chunk; + if (body.length > MAX_BODY_SIZE) { + req.destroy(); + res.statusCode = 413; + res.end('Payload too large'); + } + }); + req.on('end', async () => { + try { + const parsed = body.length > 0 ? JSON.parse(body) : undefined; + await options.mcpHandlers!.handlePost(req, res, parsed); + } catch (err) { + logger.error(`Failed to process MCP POST body: ${String(err)}`); + res.statusCode = 400; + res.end('Invalid JSON body'); + } + }); + req.on('error', (err: Error) => { + logger.error(`Error reading MCP POST body: ${String(err)}`); + res.statusCode = 500; + res.end('Failed to read request body'); + }); + return; + } + } + + const request = await toRequest(req); + const response = await app.fetch(request); + await sendNodeResponse(res, response); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.error(`Unhandled error in Node bridge: ${message}`, { error }); + res.statusCode = 500; + res.end('Internal Server Error'); + } + }); + + server.on('close', () => { + webhookSubscriber?.cleanup?.(); + }); + + if (typeof options.port === 'number') { + const hostname = options.hostname ?? '0.0.0.0'; + server.listen(options.port, hostname, () => { + logger.info(`Hono Node bridge listening on http://${hostname}:${options.port}`); + }); + } + + const result: NodeBridgeResult = { + server, + }; + + if (webhookSubscriber) { + result.webhookSubscriber = webhookSubscriber; + } + + return result; +} + +async function toRequest(req: IncomingMessage): Promise { + const protocol = (req.socket as any)?.encrypted ? 'https' : 'http'; + const host = req.headers.host ?? 'localhost'; + const url = new URL(req.url ?? '/', `${protocol}://${host}`); + + const headers = new globalThis.Headers(); + for (const [key, value] of Object.entries(req.headers)) { + if (value === undefined) continue; + if (Array.isArray(value)) { + value.forEach((entry) => headers.append(key, entry)); + } else { + headers.set(key, value); + } + } + + const method = req.method ?? 'GET'; + const body: FetchBodyInit | null = + method === 'GET' || method === 'HEAD' ? null : (req as unknown as FetchBodyInit); + + return new globalThis.Request(url, { + method, + headers, + body: body ?? undefined, + duplex: body ? 'half' : undefined, + } as RequestInit); +} + +async function sendNodeResponse(res: ServerResponse, response: Response) { + res.statusCode = response.status; + response.headers.forEach((value, key) => { + if (key.toLowerCase() === 'content-length') { + return; + } + res.setHeader(key, value); + }); + + if (!response.body) { + res.end(); + return; + } + + const webStream = response.body as unknown as NodeReadableStream; + const readable = Readable.fromWeb(webStream); + await new Promise((resolve, reject) => { + readable.on('error', reject); + res.on('finish', resolve); + readable.pipe(res); + }); +} diff --git a/dexto/packages/server/src/hono/routes/a2a-jsonrpc.ts b/dexto/packages/server/src/hono/routes/a2a-jsonrpc.ts new file mode 100644 index 00000000..88f8044f --- /dev/null +++ b/dexto/packages/server/src/hono/routes/a2a-jsonrpc.ts @@ -0,0 +1,176 @@ +/** + * A2A JSON-RPC HTTP Endpoint + * + * Exposes A2A Protocol JSON-RPC methods via HTTP POST endpoint. + * Implements JSON-RPC 2.0 over HTTP transport. + */ + +import { Hono } from 'hono'; +import type { DextoAgent } from '@dexto/core'; +import { JsonRpcServer } from '../../a2a/jsonrpc/server.js'; +import { A2AMethodHandlers } from '../../a2a/jsonrpc/methods.js'; +import { logger } from '@dexto/core'; +import type { A2ASseEventSubscriber } from '../../events/a2a-sse-subscriber.js'; +import { a2aToInternalMessage } from '../../a2a/adapters/message.js'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +/** + * Create A2A JSON-RPC router + * + * Exposes POST /jsonrpc endpoint for A2A Protocol communication. + * + * Usage: + * ```typescript + * const a2aRouter = createA2AJsonRpcRouter(getAgent, sseSubscriber); + * app.route('/', a2aRouter); + * ``` + * + * Example request: + * ```json + * POST /jsonrpc + * Content-Type: application/json + * + * { + * "jsonrpc": "2.0", + * "method": "message/send", + * "params": { + * "message": { + * "role": "user", + * "parts": [{ "kind": "text", "text": "Hello!" }], + * "messageId": "msg-123", + * "kind": "message" + * } + * }, + * "id": 1 + * } + * ``` + * + * @param getAgent Function to get current DextoAgent instance + * @param sseSubscriber SSE event subscriber for streaming methods + * @returns Hono router with /jsonrpc endpoint + */ +export function createA2AJsonRpcRouter(getAgent: GetAgentFn, sseSubscriber: A2ASseEventSubscriber) { + const app = new Hono(); + + /** + * POST /jsonrpc - JSON-RPC 2.0 endpoint + * + * Accepts JSON-RPC requests (single or batch) and returns JSON-RPC responses. + * For streaming methods (message/stream), returns SSE stream. + */ + app.post('/jsonrpc', async (ctx) => { + try { + const agent = await getAgent(ctx); + const requestBody = await ctx.req.json(); + + // Check if this is a streaming method request + const isStreamingRequest = + !Array.isArray(requestBody) && requestBody.method === 'message/stream'; + + if (isStreamingRequest) { + // Handle streaming request with SSE + logger.info('JSON-RPC streaming request: message/stream'); + + const params = requestBody.params; + if (!params?.message) { + return ctx.json({ + jsonrpc: '2.0', + error: { + code: -32602, + message: 'Invalid params: message is required', + }, + id: requestBody.id, + }); + } + + // Create or get session + const taskId = params.message.taskId; + const session = await agent.createSession(taskId); + + // Create SSE stream + const stream = sseSubscriber.createStream(session.id); + + // Start agent processing in background + const { text, image, file } = a2aToInternalMessage(params.message); + agent.run(text, image, file, session.id).catch((error) => { + logger.error(`Error in streaming task ${session.id}: ${error}`); + }); + + logger.info(`JSON-RPC SSE stream opened for task ${session.id}`); + + // Return stream with SSE headers + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'X-Accel-Buffering': 'no', + }, + }); + } + + // Handle regular (non-streaming) JSON-RPC request + const handlers = new A2AMethodHandlers(agent); + const rpcServer = new JsonRpcServer({ + methods: handlers.getMethods(), + onError: (error, request) => { + logger.error(`JSON-RPC error for method ${request?.method}: ${error.message}`, { + error, + request, + }); + }, + }); + + logger.debug(`A2A JSON-RPC request received`, { + method: Array.isArray(requestBody) + ? `batch(${requestBody.length})` + : requestBody.method, + }); + + const response = await rpcServer.handle(requestBody); + return ctx.json(response); + } catch (error) { + logger.error(`Failed to process JSON-RPC request: ${error}`, { error }); + + return ctx.json({ + jsonrpc: '2.0', + error: { + code: -32700, + message: 'Parse error', + data: error instanceof Error ? error.message : String(error), + }, + id: null, + }); + } + }); + + /** + * GET /jsonrpc - Info endpoint (non-standard, for debugging) + * + * Returns information about available JSON-RPC methods. + */ + app.get('/jsonrpc', async (ctx) => { + const agent = await getAgent(ctx); + const handlers = new A2AMethodHandlers(agent); + + return ctx.json({ + service: 'A2A JSON-RPC 2.0', + version: '0.3.0', + endpoint: '/jsonrpc', + methods: Object.keys(handlers.getMethods()), + usage: { + method: 'POST', + contentType: 'application/json', + example: { + jsonrpc: '2.0', + method: 'agent.getInfo', + params: {}, + id: 1, + }, + }, + }); + }); + + return app; +} diff --git a/dexto/packages/server/src/hono/routes/a2a-tasks.ts b/dexto/packages/server/src/hono/routes/a2a-tasks.ts new file mode 100644 index 00000000..5be43dcc --- /dev/null +++ b/dexto/packages/server/src/hono/routes/a2a-tasks.ts @@ -0,0 +1,423 @@ +/** + * A2A REST Task API (Compliant with A2A Protocol v0.3.0) + * + * RESTful HTTP+JSON endpoints for A2A Protocol task management. + * Follows the /v1/ URL pattern per A2A specification. + * + * Endpoint mappings per spec: + * - POST /v1/message:send → message/send + * - GET /v1/tasks/{id} → tasks/get + * - GET /v1/tasks → tasks/list + * - POST /v1/tasks/{id}:cancel → tasks/cancel + */ + +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { DextoAgent } from '@dexto/core'; +import { A2AMethodHandlers } from '../../a2a/jsonrpc/methods.js'; +import { logger } from '@dexto/core'; +import type { A2ASseEventSubscriber } from '../../events/a2a-sse-subscriber.js'; +import { a2aToInternalMessage } from '../../a2a/adapters/message.js'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +// Request/Response Schemas for OpenAPI (using A2A-compliant schema) + +const PartSchema = z + .discriminatedUnion('kind', [ + z.object({ + kind: z.literal('text').describe('Part type discriminator'), + text: z.string().describe('Text content'), + metadata: z.record(z.any()).optional().describe('Extension metadata'), + }), + z.object({ + kind: z.literal('file').describe('Part type discriminator'), + file: z + .union([ + z.object({ + bytes: z.string().describe('Base64-encoded file data'), + name: z.string().optional().describe('File name'), + mimeType: z.string().optional().describe('MIME type'), + }), + z.object({ + uri: z.string().describe('File URI'), + name: z.string().optional().describe('File name'), + mimeType: z.string().optional().describe('MIME type'), + }), + ]) + .describe('File data (bytes or URI)'), + metadata: z.record(z.any()).optional().describe('Extension metadata'), + }), + z.object({ + kind: z.literal('data').describe('Part type discriminator'), + data: z.record(z.any()).describe('Structured JSON data'), + metadata: z.record(z.any()).optional().describe('Extension metadata'), + }), + ]) + .describe('Message part (text, file, or data)'); + +const MessageSchema = z + .object({ + role: z.enum(['user', 'agent']).describe('Message role'), + parts: z.array(PartSchema).describe('Message parts'), + messageId: z.string().describe('Unique message identifier'), + taskId: z.string().optional().describe('Associated task ID'), + contextId: z.string().optional().describe('Context identifier'), + metadata: z.record(z.any()).optional().describe('Extension metadata'), + extensions: z.array(z.string()).optional().describe('Extension identifiers'), + referenceTaskIds: z.array(z.string()).optional().describe('Referenced task IDs'), + kind: z.literal('message').describe('Object type discriminator'), + }) + .describe('A2A Protocol message'); + +const TaskStatusSchema = z + .object({ + state: z + .enum([ + 'submitted', + 'working', + 'input-required', + 'completed', + 'canceled', + 'failed', + 'rejected', + 'auth-required', + 'unknown', + ]) + .describe('Current task state'), + message: MessageSchema.optional().describe('Status message'), + timestamp: z.string().optional().describe('ISO 8601 timestamp'), + }) + .describe('Task status'); + +const TaskSchema = z + .object({ + id: z.string().describe('Unique task identifier'), + contextId: z.string().describe('Context identifier across related tasks'), + status: TaskStatusSchema.describe('Current task status'), + history: z.array(MessageSchema).optional().describe('Conversation history'), + artifacts: z.array(z.any()).optional().describe('Task artifacts'), + metadata: z.record(z.any()).optional().describe('Extension metadata'), + kind: z.literal('task').describe('Object type discriminator'), + }) + .describe('A2A Protocol task'); + +const MessageSendRequestSchema = z + .object({ + message: MessageSchema.describe('Message to send to the agent'), + configuration: z + .object({ + acceptedOutputModes: z + .array(z.string()) + .optional() + .describe('Accepted output MIME types'), + historyLength: z.number().optional().describe('Limit conversation history length'), + pushNotificationConfig: z + .object({ + url: z.string().describe('Push notification webhook URL'), + headers: z + .record(z.string()) + .optional() + .describe('HTTP headers for webhook'), + }) + .optional() + .describe('Push notification configuration'), + blocking: z.boolean().optional().describe('Wait for task completion'), + }) + .optional() + .describe('Optional configuration'), + metadata: z.record(z.any()).optional().describe('Optional metadata'), + }) + .describe('Request body for message/send'); + +const TaskListQuerySchema = z + .object({ + contextId: z.string().optional().describe('Filter by context ID'), + status: z + .enum([ + 'submitted', + 'working', + 'input-required', + 'completed', + 'canceled', + 'failed', + 'rejected', + 'auth-required', + 'unknown', + ]) + .optional() + .describe('Filter by task state'), + pageSize: z + .string() + .optional() + .transform((v) => { + if (!v) return undefined; + const n = Number.parseInt(v, 10); + // Enforce 1-100 range, return undefined for invalid values + if (Number.isNaN(n) || n < 1 || n > 100) return undefined; + return n; + }) + .describe('Number of results (1-100, default 50)'), + pageToken: z + .string() + .optional() + .describe('Pagination token (not yet implemented - reserved for future use)'), + historyLength: z + .string() + .optional() + .transform((v) => { + if (!v) return undefined; + const n = Number.parseInt(v, 10); + return Number.isNaN(n) ? undefined : n; + }) + .describe('Limit history items (not yet implemented - reserved for future use)'), + lastUpdatedAfter: z + .string() + .optional() + .transform((v) => { + if (!v) return undefined; + const n = Number.parseInt(v, 10); + return Number.isNaN(n) ? undefined : n; + }) + .describe('Unix timestamp filter (not yet implemented - reserved for future use)'), + includeArtifacts: z + .string() + .optional() + .transform((v) => v === 'true') + .describe( + 'Include artifacts in response (not yet implemented - reserved for future use)' + ), + }) + .describe('Query parameters for tasks/list'); + +/** + * Create A2A REST Task router + * + * Exposes RESTful endpoints for A2A task management per v0.3.0 spec. + * + * Endpoints: + * - POST /v1/message:send - Send message to agent + * - POST /v1/message:stream - Send message with SSE streaming + * - GET /v1/tasks/{id} - Get task + * - GET /v1/tasks - List tasks + * - POST /v1/tasks/{id}:cancel - Cancel task + * + * @param getAgent Function to get current DextoAgent instance + * @param sseSubscriber SSE event subscriber for streaming + * @returns OpenAPIHono router with REST task endpoints + */ +export function createA2ATasksRouter(getAgent: GetAgentFn, sseSubscriber: A2ASseEventSubscriber) { + const app = new OpenAPIHono(); + + // POST /v1/message:send - Send message to agent + const messageSendRoute = createRoute({ + method: 'post', + path: '/v1/message:send', + summary: 'Send Message', + description: 'Send a message to the agent (A2A message/send)', + tags: ['a2a'], + request: { + body: { + content: { + 'application/json': { + schema: MessageSendRequestSchema, + }, + }, + }, + }, + responses: { + 200: { + description: 'Task with agent response', + content: { + 'application/json': { + schema: TaskSchema, + }, + }, + }, + }, + }); + + // GET /v1/tasks - List tasks + const listTasksRoute = createRoute({ + method: 'get', + path: '/v1/tasks', + summary: 'List Tasks', + description: 'List all A2A tasks with optional filtering (A2A tasks/list)', + tags: ['a2a'], + request: { + query: TaskListQuerySchema, + }, + responses: { + 200: { + description: 'Task list', + content: { + 'application/json': { + schema: z + .object({ + tasks: z.array(TaskSchema).describe('Array of tasks'), + totalSize: z.number().describe('Total number of tasks'), + pageSize: z.number().describe('Number of tasks in this page'), + nextPageToken: z.string().describe('Token for next page'), + }) + .describe('Response body for tasks/list'), + }, + }, + }, + }, + }); + + // GET /v1/tasks/{id} - Get a specific task + const getTaskRoute = createRoute({ + method: 'get', + path: '/v1/tasks/{id}', + summary: 'Get Task', + description: 'Retrieve a specific task by ID (A2A tasks/get)', + tags: ['a2a'], + request: { + params: z.object({ + id: z.string().describe('Task ID'), + }), + }, + responses: { + 200: { + description: 'Task details', + content: { + 'application/json': { + schema: TaskSchema, + }, + }, + }, + 404: { + description: 'Task not found', + }, + }, + }); + + // POST /v1/tasks/{id}:cancel - Cancel task + const cancelTaskRoute = createRoute({ + method: 'post', + path: '/v1/tasks/{id}:cancel', + summary: 'Cancel Task', + description: 'Cancel a running task (A2A tasks/cancel)', + tags: ['a2a'], + request: { + params: z.object({ + id: z.string().describe('Task ID'), + }), + }, + responses: { + 200: { + description: 'Task cancelled', + content: { + 'application/json': { + schema: TaskSchema, + }, + }, + }, + 404: { + description: 'Task not found', + }, + }, + }); + + // POST /v1/message:stream - Send message with streaming response + app.post('/v1/message:stream', async (ctx) => { + try { + const body = await ctx.req.json(); + + // Validate with Zod schema + const parseResult = MessageSendRequestSchema.safeParse(body); + if (!parseResult.success) { + return ctx.json( + { + error: 'Invalid request body', + details: parseResult.error.issues, + }, + 400 + ); + } + + const validatedBody = parseResult.data; + logger.info('REST: message/stream', { hasMessage: !!validatedBody.message }); + + // Create or get session + const taskId = validatedBody.message.taskId; + const agent = await getAgent(ctx); + const session = await agent.createSession(taskId); + + // Create SSE stream + const stream = sseSubscriber.createStream(session.id); + + // Start agent processing in background + // Note: Errors are automatically broadcast via the event bus (llm:error event) + const { text, image, file } = a2aToInternalMessage(validatedBody.message as any); + agent.run(text, image, file, session.id).catch((error) => { + logger.error(`Error in streaming task ${session.id}: ${error}`); + }); + + logger.info(`REST SSE stream opened for task ${session.id}`); + + // Return stream with SSE headers + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'X-Accel-Buffering': 'no', + }, + }); + } catch (error) { + logger.error(`Failed to handle message:stream: ${error}`); + return ctx.json({ error: 'Failed to initiate streaming' }, 500); + } + }); + + return app + .openapi(messageSendRoute, async (ctx) => { + const handlers = new A2AMethodHandlers(await getAgent(ctx)); + const body = ctx.req.valid('json'); + + logger.info('REST: message/send', { hasMessage: !!body.message }); + + // Type cast required: Zod infers readonly modifiers and exactOptionalPropertyTypes differs + // from mutable handler types. Structurally compatible at runtime. + const result = await handlers.messageSend(body as any); + + return ctx.json(result as any); + }) + .openapi(listTasksRoute, async (ctx) => { + const handlers = new A2AMethodHandlers(await getAgent(ctx)); + const query = ctx.req.valid('query'); + + // Type cast required: Zod infers readonly modifiers and exactOptionalPropertyTypes differs + // from mutable handler types. Structurally compatible at runtime. + const result = await handlers.tasksList(query as any); + + return ctx.json(result); + }) + .openapi(getTaskRoute, async (ctx) => { + const handlers = new A2AMethodHandlers(await getAgent(ctx)); + const { id } = ctx.req.valid('param'); + + try { + const task = await handlers.tasksGet({ id }); + return ctx.json(task); + } catch (error) { + logger.warn(`Task ${id} not found: ${error}`); + return ctx.json({ error: 'Task not found' }, 404); + } + }) + .openapi(cancelTaskRoute, async (ctx) => { + const handlers = new A2AMethodHandlers(await getAgent(ctx)); + const { id } = ctx.req.valid('param'); + + logger.info(`REST: tasks/cancel ${id}`); + + try { + const task = await handlers.tasksCancel({ id }); + return ctx.json(task); + } catch (error) { + logger.error(`Failed to cancel task ${id}: ${error}`); + return ctx.json({ error: 'Task not found' }, 404); + } + }); +} diff --git a/dexto/packages/server/src/hono/routes/a2a.ts b/dexto/packages/server/src/hono/routes/a2a.ts new file mode 100644 index 00000000..19a3b8a2 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/a2a.ts @@ -0,0 +1,11 @@ +import { Hono } from 'hono'; +import type { AgentCard } from '@dexto/core'; + +export function createA2aRouter(getAgentCard: () => AgentCard) { + const app = new Hono(); + app.get('/.well-known/agent-card.json', (ctx) => { + const agentCard = getAgentCard(); + return ctx.json(agentCard, 200); + }); + return app; +} diff --git a/dexto/packages/server/src/hono/routes/agents.ts b/dexto/packages/server/src/hono/routes/agents.ts new file mode 100644 index 00000000..681e49a6 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/agents.ts @@ -0,0 +1,956 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { DextoAgent } from '@dexto/core'; +import { + logger, + safeStringify, + AgentConfigSchema, + type LLMProvider, + zodToIssues, +} from '@dexto/core'; +import { + getPrimaryApiKeyEnvVar, + saveProviderApiKey, + reloadAgentConfigFromFile, + enrichAgentConfig, + deriveDisplayName, + AgentFactory, +} from '@dexto/agent-management'; +import { stringify as yamlStringify, parse as yamlParse } from 'yaml'; +import os from 'os'; +import path from 'path'; +import { promises as fs } from 'fs'; +import { DextoValidationError, AgentErrorCode, ErrorScope, ErrorType } from '@dexto/core'; +import { AgentRegistryEntrySchema } from '../schemas/responses.js'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +/** + * OpenAPI-safe version of AgentConfigSchema + * + * This simplified schema is used ONLY for OpenAPI documentation generation. + * Runtime validation still uses the full AgentConfigSchema with complete validation. + * + * Why: The real AgentConfigSchema uses z.lazy() for CustomToolConfigSchema, + * which cannot be serialized to OpenAPI JSON by @hono/zod-openapi. + * + * See lines 780 and 854 where AgentConfigSchema.safeParse() is used for actual validation. + */ +const AgentConfigSchemaForOpenAPI = z + .record(z.any()) + .describe( + 'Complete agent configuration. See AgentConfig type documentation for full schema details.' + ); + +const AgentIdentifierSchema = z + .object({ + id: z + .string() + .min(1, 'Agent id is required') + .describe('Unique agent identifier (e.g., "database-agent")'), + path: z + .string() + .optional() + .describe( + 'Optional absolute file path for file-based agents (e.g., "/path/to/agent.yml")' + ), + }) + .strict() + .describe('Agent identifier for switching agents by ID or file path'); + +const UninstallAgentSchema = z + .object({ + id: z + .string() + .min(1, 'Agent id is required') + .describe('Unique agent identifier to uninstall'), + force: z + .boolean() + .default(false) + .describe('Force uninstall even if agent is currently active'), + }) + .strict() + .describe('Request body for uninstalling an agent'); + +const CustomAgentInstallSchema = z + .object({ + id: z.string().min(1, 'Agent id is required').describe('Unique agent identifier'), + name: z.string().optional().describe('Display name (defaults to derived from id)'), + sourcePath: z.string().min(1).describe('Path to agent configuration file or directory'), + metadata: z + .object({ + description: z.string().min(1).describe('Human-readable description of the agent'), + author: z.string().min(1).describe('Agent author or organization name'), + tags: z.array(z.string()).describe('Tags for categorizing the agent'), + main: z + .string() + .optional() + .describe('Main configuration file name within source directory'), + }) + .strict() + .describe('Agent metadata including description, author, and tags'), + }) + .strict() + .describe('Request body for installing a custom agent from file system') + .transform((value) => { + const displayName = value.name?.trim() || deriveDisplayName(value.id); + return { + id: value.id, + displayName, + sourcePath: value.sourcePath, + metadata: value.metadata, + }; + }); + +const CustomAgentCreateSchema = z + .object({ + // Registry metadata + id: z + .string() + .min(1, 'Agent ID is required') + .regex( + /^[a-z0-9-]+$/, + 'Agent ID must contain only lowercase letters, numbers, and hyphens' + ) + .describe('Unique agent identifier'), + name: z.string().min(1, 'Agent name is required').describe('Display name for the agent'), + description: z + .string() + .min(1, 'Description is required') + .describe('One-line description of the agent'), + author: z.string().optional().describe('Author or organization'), + tags: z.array(z.string()).default([]).describe('Tags for discovery'), + // Full agent configuration + config: AgentConfigSchemaForOpenAPI.describe('Complete agent configuration'), + }) + .strict() + .describe('Request body for creating a new custom agent with full configuration'); + +const AgentConfigValidateSchema = z + .object({ + yaml: z.string().describe('YAML agent configuration content to validate'), + }) + .describe('Request body for validating agent configuration YAML'); + +const AgentConfigSaveSchema = z + .object({ + yaml: z + .string() + .min(1, 'YAML content is required') + .describe('YAML agent configuration content to save'), + }) + .describe('Request body for saving agent configuration YAML'); + +// Response schemas for agent endpoints + +const AgentInfoNullableSchema = z + .object({ + id: z.string().nullable().describe('Agent identifier (null if no active agent)'), + name: z.string().nullable().describe('Agent display name (null if no active agent)'), + }) + .strict() + .describe('Basic agent information (nullable)'); + +const ListAgentsResponseSchema = z + .object({ + installed: z.array(AgentRegistryEntrySchema).describe('Agents installed locally'), + available: z.array(AgentRegistryEntrySchema).describe('Agents available from registry'), + current: AgentInfoNullableSchema.describe('Currently active agent'), + }) + .strict() + .describe('List of all agents'); + +const InstallAgentResponseSchema = z + .object({ + installed: z.literal(true).describe('Indicates successful installation'), + id: z.string().describe('Installed agent ID'), + name: z.string().describe('Installed agent name'), + type: z.enum(['builtin', 'custom']).describe('Type of agent installed'), + }) + .strict() + .describe('Agent installation response'); + +const SwitchAgentResponseSchema = z + .object({ + switched: z.literal(true).describe('Indicates successful agent switch'), + id: z.string().describe('New active agent ID'), + name: z.string().describe('New active agent name'), + }) + .strict() + .describe('Agent switch response'); + +const ValidateNameResponseSchema = z + .object({ + valid: z.boolean().describe('Whether the agent name is valid'), + conflict: z.string().optional().describe('Type of conflict if name is invalid'), + message: z.string().optional().describe('Validation message'), + }) + .strict() + .describe('Agent name validation result'); + +const UninstallAgentResponseSchema = z + .object({ + uninstalled: z.literal(true).describe('Indicates successful uninstallation'), + id: z.string().describe('Uninstalled agent ID'), + }) + .strict() + .describe('Agent uninstallation response'); + +const AgentPathResponseSchema = z + .object({ + path: z.string().describe('Absolute path to agent configuration file'), + relativePath: z.string().describe('Relative path or basename'), + name: z.string().describe('Agent configuration filename without extension'), + isDefault: z.boolean().describe('Whether this is the default agent'), + }) + .strict() + .describe('Agent file path information'); + +const AgentConfigResponseSchema = z + .object({ + yaml: z.string().describe('Raw YAML configuration content'), + path: z.string().describe('Absolute path to configuration file'), + relativePath: z.string().describe('Relative path or basename'), + lastModified: z.date().describe('Last modification timestamp'), + warnings: z.array(z.string()).describe('Configuration warnings'), + }) + .strict() + .describe('Agent configuration content'); + +const SaveConfigResponseSchema = z + .object({ + ok: z.literal(true).describe('Indicates successful save'), + path: z.string().describe('Path to saved configuration file'), + reloaded: z.boolean().describe('Whether configuration was reloaded'), + restarted: z.boolean().describe('Whether agent was restarted'), + changesApplied: z.array(z.string()).describe('List of changes that were applied'), + message: z.string().describe('Success message'), + }) + .strict() + .describe('Configuration save result'); + +export type AgentsRouterContext = { + switchAgentById: (agentId: string) => Promise<{ id: string; name: string }>; + switchAgentByPath: (filePath: string) => Promise<{ id: string; name: string }>; + resolveAgentInfo: (agentId: string) => Promise<{ id: string; name: string }>; + ensureAgentAvailable: () => void; + getActiveAgentId: () => string | undefined; +}; + +export function createAgentsRouter(getAgent: GetAgentFn, context: AgentsRouterContext) { + const app = new OpenAPIHono(); + const { switchAgentById, switchAgentByPath, resolveAgentInfo, getActiveAgentId } = context; + + const listRoute = createRoute({ + method: 'get', + path: '/agents', + summary: 'List Agents', + description: 'Retrieves all agents (installed, available, and current active agent)', + tags: ['agents'], + responses: { + 200: { + description: 'List all agents', + content: { 'application/json': { schema: ListAgentsResponseSchema } }, + }, + }, + }); + + const currentRoute = createRoute({ + method: 'get', + path: '/agents/current', + summary: 'Get Current Agent', + description: 'Retrieves the currently active agent', + tags: ['agents'], + responses: { + 200: { + description: 'Current agent', + content: { 'application/json': { schema: AgentInfoNullableSchema } }, + }, + }, + }); + + const installRoute = createRoute({ + method: 'post', + path: '/agents/install', + summary: 'Install Agent', + description: 'Installs an agent from the registry or from a custom source', + tags: ['agents'], + request: { + body: { + content: { + 'application/json': { + schema: z.union([CustomAgentInstallSchema, AgentIdentifierSchema]), + }, + }, + }, + }, + responses: { + 201: { + description: 'Agent installed', + content: { 'application/json': { schema: InstallAgentResponseSchema } }, + }, + }, + }); + + const switchRoute = createRoute({ + method: 'post', + path: '/agents/switch', + summary: 'Switch Agent', + description: 'Switches to a different agent by ID or file path', + tags: ['agents'], + request: { + body: { + content: { + 'application/json': { + schema: AgentIdentifierSchema, + }, + }, + }, + }, + responses: { + 200: { + description: 'Agent switched', + content: { 'application/json': { schema: SwitchAgentResponseSchema } }, + }, + }, + }); + + const validateNameRoute = createRoute({ + method: 'post', + path: '/agents/validate-name', + summary: 'Validate Agent Name', + description: 'Checks if an agent ID conflicts with existing agents', + tags: ['agents'], + request: { + body: { + content: { + 'application/json': { + schema: AgentIdentifierSchema, + }, + }, + }, + }, + responses: { + 200: { + description: 'Name validation result', + content: { 'application/json': { schema: ValidateNameResponseSchema } }, + }, + }, + }); + + const uninstallRoute = createRoute({ + method: 'post', + path: '/agents/uninstall', + summary: 'Uninstall Agent', + description: + 'Removes an agent from the system. Custom agents are removed from registry; builtin agents can be reinstalled', + tags: ['agents'], + request: { + body: { + content: { + 'application/json': { + schema: UninstallAgentSchema, + }, + }, + }, + }, + responses: { + 200: { + description: 'Agent uninstalled', + content: { 'application/json': { schema: UninstallAgentResponseSchema } }, + }, + }, + }); + + const customCreateRoute = createRoute({ + method: 'post', + path: '/agents/custom/create', + summary: 'Create Custom Agent', + description: 'Creates a new custom agent from scratch via the UI/API', + tags: ['agents'], + request: { + body: { + content: { + 'application/json': { + schema: CustomAgentCreateSchema, + }, + }, + }, + }, + responses: { + 201: { + description: 'Custom agent created', + content: { + 'application/json': { + schema: z + .object({ + created: z.literal(true).describe('Creation success indicator'), + id: z.string().describe('Agent identifier'), + name: z.string().describe('Agent name'), + }) + .strict(), + }, + }, + }, + }, + }); + + const getPathRoute = createRoute({ + method: 'get', + path: '/agent/path', + summary: 'Get Agent File Path', + description: 'Retrieves the file path of the currently active agent configuration', + tags: ['agent'], + responses: { + 200: { + description: 'Agent file path', + content: { + 'application/json': { + schema: AgentPathResponseSchema, + }, + }, + }, + }, + }); + + const getConfigRoute = createRoute({ + method: 'get', + path: '/agent/config', + summary: 'Get Agent Configuration', + description: 'Retrieves the raw YAML configuration of the currently active agent', + tags: ['agent'], + responses: { + 200: { + description: 'Agent configuration', + content: { + 'application/json': { + schema: AgentConfigResponseSchema, + }, + }, + }, + }, + }); + + const validateConfigRoute = createRoute({ + method: 'post', + path: '/agent/validate', + summary: 'Validate Agent Configuration', + description: 'Validates YAML agent configuration without saving it', + tags: ['agent'], + request: { + body: { + content: { + 'application/json': { + schema: AgentConfigValidateSchema, + }, + }, + }, + }, + responses: { + 200: { + description: 'Validation result', + content: { + 'application/json': { + schema: z + .object({ + valid: z.boolean().describe('Whether configuration is valid'), + errors: z + .array( + z + .object({ + line: z + .number() + .int() + .optional() + .describe('Line number'), + column: z + .number() + .int() + .optional() + .describe('Column number'), + path: z + .string() + .optional() + .describe('Configuration path'), + message: z.string().describe('Error message'), + code: z.string().describe('Error code'), + }) + .passthrough() + ) + .describe('Validation errors'), + warnings: z + .array( + z + .object({ + path: z.string().describe('Configuration path'), + message: z.string().describe('Warning message'), + code: z.string().describe('Warning code'), + }) + .strict() + ) + .describe('Configuration warnings'), + }) + .strict(), + }, + }, + }, + }, + }); + + const saveConfigRoute = createRoute({ + method: 'post', + path: '/agent/config', + summary: 'Save Agent Configuration', + description: 'Saves and applies YAML agent configuration. Creates backup before saving', + tags: ['agent'], + request: { + body: { + content: { + 'application/json': { + schema: AgentConfigSaveSchema, + }, + }, + }, + }, + responses: { + 200: { + description: 'Configuration saved', + content: { + 'application/json': { + schema: SaveConfigResponseSchema, + }, + }, + }, + }, + }); + + const exportConfigRoute = createRoute({ + method: 'get', + path: '/agent/config/export', + summary: 'Export Agent Configuration', + description: 'Exports the effective agent configuration with sensitive values redacted', + tags: ['agent'], + request: { + query: z.object({ + sessionId: z + .string() + .optional() + .describe('Session identifier to export session-specific configuration'), + }), + }, + responses: { + 200: { + description: 'Exported configuration', + content: { 'application/x-yaml': { schema: z.string() } }, + }, + }, + }); + + return app + .openapi(listRoute, async (ctx) => { + const agents = await AgentFactory.listAgents(); + const currentId = getActiveAgentId() ?? null; + return ctx.json({ + installed: agents.installed, + available: agents.available, + current: currentId ? await resolveAgentInfo(currentId) : { id: null, name: null }, + }); + }) + .openapi(currentRoute, async (ctx) => { + const currentId = getActiveAgentId() ?? null; + if (!currentId) { + return ctx.json({ id: null, name: null }); + } + return ctx.json(await resolveAgentInfo(currentId)); + }) + .openapi(installRoute, async (ctx) => { + const body = ctx.req.valid('json'); + + // Check if this is a custom agent installation (has sourcePath and metadata) + if ('sourcePath' in body && 'metadata' in body) { + const { id, displayName, sourcePath, metadata } = body as ReturnType< + typeof CustomAgentInstallSchema.parse + >; + + await AgentFactory.installCustomAgent(id, sourcePath, { + name: displayName, + description: metadata.description, + author: metadata.author, + tags: metadata.tags, + }); + return ctx.json( + { installed: true as const, id, name: displayName, type: 'custom' as const }, + 201 + ); + } else { + // Registry agent installation + const { id } = body as z.output; + await AgentFactory.installAgent(id); + const agentInfo = await resolveAgentInfo(id); + return ctx.json( + { + installed: true as const, + ...agentInfo, + type: 'builtin' as const, + }, + 201 + ); + } + }) + .openapi(switchRoute, async (ctx) => { + const { id, path: filePath } = ctx.req.valid('json'); + + // Route based on presence of path parameter + const result = filePath ? await switchAgentByPath(filePath) : await switchAgentById(id); + + return ctx.json({ switched: true as const, ...result }); + }) + .openapi(validateNameRoute, async (ctx) => { + const { id } = ctx.req.valid('json'); + const agents = await AgentFactory.listAgents(); + + // Check if name exists in installed agents + const installedAgent = agents.installed.find((a) => a.id === id); + if (installedAgent) { + return ctx.json({ + valid: false, + conflict: installedAgent.type, + message: `Agent id '${id}' already exists (${installedAgent.type})`, + }); + } + + // Check if name exists in available agents (registry) + const availableAgent = agents.available.find((a) => a.id === id); + if (availableAgent) { + return ctx.json({ + valid: false, + conflict: availableAgent.type, + message: `Agent id '${id}' conflicts with ${availableAgent.type} agent`, + }); + } + + return ctx.json({ valid: true }); + }) + .openapi(uninstallRoute, async (ctx) => { + const { id, force } = ctx.req.valid('json'); + await AgentFactory.uninstallAgent(id, force); + return ctx.json({ uninstalled: true as const, id }); + }) + .openapi(customCreateRoute, async (ctx) => { + const { id, name, description, author, tags, config } = ctx.req.valid('json'); + + // Handle API key: if it's a raw key, store securely and use env var reference + const provider: LLMProvider = config.llm.provider; + let agentConfig = config; + + if (config.llm.apiKey && !config.llm.apiKey.startsWith('$')) { + // Raw API key provided - store securely and get env var reference + const meta = await saveProviderApiKey(provider, config.llm.apiKey, process.cwd()); + const apiKeyRef = `$${meta.envVar}`; + logger.info( + `Stored API key securely for ${provider}, using env var: ${meta.envVar}` + ); + // Update config with env var reference + agentConfig = { + ...config, + llm: { + ...config.llm, + apiKey: apiKeyRef, + }, + }; + } else if (!config.llm.apiKey) { + // No API key provided, use default env var + agentConfig = { + ...config, + llm: { + ...config.llm, + apiKey: `$${getPrimaryApiKeyEnvVar(provider)}`, + }, + }; + } + + const yamlContent = yamlStringify(agentConfig); + logger.info( + `Creating agent config for ${id}: agentConfig=${safeStringify(agentConfig)}, yamlContent=${yamlContent}` + ); + + // Create temporary file + const tmpDir = os.tmpdir(); + const tmpFile = path.join(tmpDir, `${id}-${Date.now()}.yml`); + await fs.writeFile(tmpFile, yamlContent, 'utf-8'); + + try { + // Install the custom agent + await AgentFactory.installCustomAgent(id, tmpFile, { + name, + description, + author: author || 'Custom', + tags: tags || [], + }); + + // Clean up temp file + await fs.unlink(tmpFile).catch(() => {}); + + return ctx.json({ created: true as const, id, name }, 201); + } catch (installError) { + // Clean up temp file on error + await fs.unlink(tmpFile).catch(() => {}); + throw installError; + } + }) + .openapi(getPathRoute, async (ctx) => { + const agent = await getAgent(ctx); + const agentPath = agent.getAgentFilePath(); + + const relativePath = path.basename(agentPath); + const ext = path.extname(agentPath); + const name = path.basename(agentPath, ext); + + return ctx.json({ + path: agentPath, + relativePath, + name, + isDefault: name === 'coding-agent', + }); + }) + .openapi(getConfigRoute, async (ctx) => { + const agent = await getAgent(ctx); + + // Get the agent file path being used + const agentPath = agent.getAgentFilePath(); + + // Read raw YAML from file (not expanded env vars) + const yamlContent = await fs.readFile(agentPath, 'utf-8'); + + // Get metadata + const stats = await fs.stat(agentPath); + + return ctx.json({ + yaml: yamlContent, + path: agentPath, + relativePath: path.basename(agentPath), + lastModified: stats.mtime, + warnings: [ + 'Environment variables ($VAR) will be resolved at runtime', + 'API keys should use environment variables', + ], + }); + }) + .openapi(validateConfigRoute, async (ctx) => { + const { yaml } = ctx.req.valid('json'); + + // Parse YAML + let parsed; + try { + parsed = yamlParse(yaml); + } catch (parseError: any) { + return ctx.json({ + valid: false, + errors: [ + { + line: parseError.linePos?.[0]?.line || 1, + column: parseError.linePos?.[0]?.col || 1, + message: parseError.message, + code: 'YAML_PARSE_ERROR', + }, + ], + warnings: [], + }); + } + + // Check that parsed content is a valid object (not null, array, or primitive) + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + return ctx.json({ + valid: false, + errors: [ + { + line: 1, + column: 1, + message: 'Configuration must be a valid YAML object', + code: 'INVALID_CONFIG_TYPE', + }, + ], + warnings: [], + }); + } + + // Enrich config with defaults/paths to satisfy schema requirements + // Pass undefined for validation-only (no real file path) + // AgentId will be derived from agentCard.name or fall back to 'coding-agent' + const enriched = enrichAgentConfig(parsed, undefined); + + // Validate against schema + const result = AgentConfigSchema.safeParse(enriched); + + if (!result.success) { + // Use zodToIssues to extract detailed validation errors (handles union errors properly) + const issues = zodToIssues(result.error); + const errors = issues.map((issue) => ({ + path: issue.path?.join('.') ?? 'root', + message: issue.message, + code: 'SCHEMA_VALIDATION_ERROR', + })); + + return ctx.json({ + valid: false, + errors, + warnings: [], + }); + } + + // Check for warnings (e.g., plain text API keys) + const warnings: Array<{ path: string; message: string; code: string }> = []; + if (parsed.llm?.apiKey && !parsed.llm.apiKey.startsWith('$')) { + warnings.push({ + path: 'llm.apiKey', + message: 'Consider using environment variable instead of plain text', + code: 'SECURITY_WARNING', + }); + } + + return ctx.json({ + valid: true, + errors: [], + warnings, + }); + }) + .openapi(saveConfigRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { yaml } = ctx.req.valid('json'); + + // Validate YAML syntax first + let parsed; + try { + parsed = yamlParse(yaml); + } catch (parseError: any) { + throw new DextoValidationError([ + { + code: AgentErrorCode.INVALID_CONFIG, + message: `Invalid YAML syntax: ${parseError.message}`, + scope: ErrorScope.AGENT, + type: ErrorType.USER, + severity: 'error', + }, + ]); + } + + // Check that parsed content is a valid object (not null, array, or primitive) + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new DextoValidationError([ + { + code: AgentErrorCode.INVALID_CONFIG, + message: 'Configuration must be a valid YAML object', + scope: ErrorScope.AGENT, + type: ErrorType.USER, + severity: 'error', + }, + ]); + } + + // Get target file path for enrichment + const agentPath = agent.getAgentFilePath(); + + // Enrich config with defaults/paths before validation (same as validation endpoint) + const enriched = enrichAgentConfig(parsed, agentPath); + + // Validate schema + const validationResult = AgentConfigSchema.safeParse(enriched); + + if (!validationResult.success) { + throw new DextoValidationError( + validationResult.error.errors.map((err) => ({ + code: AgentErrorCode.INVALID_CONFIG, + message: `${err.path.join('.')}: ${err.message}`, + scope: ErrorScope.AGENT, + type: ErrorType.USER, + severity: 'error', + })) + ); + } + + // Create backup + const backupPath = `${agentPath}.backup`; + await fs.copyFile(agentPath, backupPath); + + try { + // Write new config + await fs.writeFile(agentPath, yaml, 'utf-8'); + + // Load from file (agent-management's job) + const newConfig = await reloadAgentConfigFromFile(agentPath); + + // Enrich config before reloading into agent (core expects enriched config with paths) + const enrichedConfig = enrichAgentConfig(newConfig, agentPath); + + // Reload into agent (core's job - handles restart automatically) + const reloadResult = await agent.reload(enrichedConfig); + + if (reloadResult.restarted) { + logger.info( + `Agent restarted to apply changes: ${reloadResult.changesApplied.join(', ')}` + ); + } else if (reloadResult.changesApplied.length === 0) { + logger.info('Configuration saved (no changes detected)'); + } + + // Clean up backup file after successful save + await fs.unlink(backupPath).catch(() => { + // Ignore errors if backup file doesn't exist + }); + + logger.info(`Agent configuration saved and applied: ${agentPath}`); + + return ctx.json({ + ok: true as const, + path: agentPath, + reloaded: true, + restarted: reloadResult.restarted, + changesApplied: reloadResult.changesApplied, + message: reloadResult.restarted + ? 'Configuration saved and applied successfully (agent restarted)' + : 'Configuration saved successfully (no changes detected)', + }); + } catch (error) { + // Restore backup on error + await fs.copyFile(backupPath, agentPath).catch(() => { + // Ignore errors if backup restore fails + }); + throw error; + } + }) + .openapi(exportConfigRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('query'); + const config = agent.getEffectiveConfig(sessionId); + + // Redact sensitive values + const maskedConfig = { + ...config, + llm: { + ...config.llm, + apiKey: config.llm.apiKey ? '[REDACTED]' : undefined, + }, + mcpServers: config.mcpServers + ? Object.fromEntries( + Object.entries(config.mcpServers).map(([name, serverConfig]) => [ + name, + serverConfig.type === 'stdio' && serverConfig.env + ? { + ...serverConfig, + env: Object.fromEntries( + Object.keys(serverConfig.env).map((key) => [ + key, + '[REDACTED]', + ]) + ), + } + : serverConfig, + ]) + ) + : undefined, + }; + + const yamlStr = yamlStringify(maskedConfig); + ctx.header('Content-Type', 'application/x-yaml'); + return ctx.body(yamlStr); + }); +} diff --git a/dexto/packages/server/src/hono/routes/approvals.ts b/dexto/packages/server/src/hono/routes/approvals.ts new file mode 100644 index 00000000..a460d4b5 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/approvals.ts @@ -0,0 +1,213 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import { type DextoAgent, DenialReason, ApprovalStatus, ApprovalError } from '@dexto/core'; +import type { ApprovalCoordinator } from '../../approval/approval-coordinator.js'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +const ApprovalBodySchema = z + .object({ + status: z + .enum([ApprovalStatus.APPROVED, ApprovalStatus.DENIED]) + .describe('The user decision'), + formData: z + .record(z.unknown()) + .optional() + .describe('Optional form data provided by the user (for elicitation)'), + rememberChoice: z + .boolean() + .optional() + .describe('Whether to remember this choice for future requests'), + }) + .describe('Request body for submitting an approval decision'); + +const ApprovalResponseSchema = z + .object({ + ok: z.boolean().describe('Whether the approval was successfully processed'), + approvalId: z.string().describe('The ID of the processed approval'), + status: z + .enum([ApprovalStatus.APPROVED, ApprovalStatus.DENIED]) + .describe('The final status'), + }) + .describe('Response after processing approval'); + +const PendingApprovalSchema = z + .object({ + approvalId: z.string().describe('The unique ID of the approval request'), + type: z.string().describe('The type of approval (tool_confirmation, elicitation, etc.)'), + sessionId: z.string().optional().describe('The session ID if applicable'), + timeout: z.number().optional().describe('Timeout in milliseconds'), + timestamp: z.string().describe('ISO timestamp when the request was created'), + metadata: z.record(z.unknown()).describe('Type-specific metadata'), + }) + .describe('A pending approval request'); + +const PendingApprovalsResponseSchema = z + .object({ + ok: z.literal(true).describe('Success indicator'), + approvals: z.array(PendingApprovalSchema).describe('List of pending approval requests'), + }) + .describe('Response containing pending approval requests'); + +export function createApprovalsRouter( + getAgent: GetAgentFn, + approvalCoordinator?: ApprovalCoordinator +) { + const app = new OpenAPIHono(); + + // GET /approvals - Fetch pending approval requests + // Useful for restoring UI state after page refresh + const getPendingApprovalsRoute = createRoute({ + method: 'get', + path: '/approvals', + summary: 'Get Pending Approvals', + description: + 'Fetch all pending approval requests for a session. Use this to restore UI state after page refresh.', + tags: ['approvals'], + request: { + query: z.object({ + sessionId: z.string().describe('The session ID to fetch pending approvals for'), + }), + }, + responses: { + 200: { + description: 'List of pending approval requests', + content: { + 'application/json': { + schema: PendingApprovalsResponseSchema, + }, + }, + }, + }, + }); + + // TODO: Consider adding auth & idempotency for production deployments + // See: https://github.com/truffle-ai/dexto/pull/450#discussion_r2545039760 + // - Auth: Open-source framework should allow flexible auth (reverse proxy, API gateway, etc.) + // - Idempotency: Already documented in schema; platform can add tracking separately + const submitApprovalRoute = createRoute({ + method: 'post', + path: '/approvals/{approvalId}', + summary: 'Submit Approval Decision', + description: 'Submit a user decision for a pending approval request', + tags: ['approvals'], + request: { + params: z.object({ + approvalId: z.string().describe('The ID of the approval request'), + }), + body: { + content: { 'application/json': { schema: ApprovalBodySchema } }, + }, + headers: z.object({ + 'Idempotency-Key': z + .string() + .optional() + .describe('Optional key to ensure idempotent processing'), + }), + }, + responses: { + 200: { + description: 'Approval processed successfully', + content: { + 'application/json': { + schema: ApprovalResponseSchema, + }, + }, + }, + 404: { + description: 'Approval request not found or expired', + }, + 400: { + description: 'Validation error', + }, + 503: { + description: + 'Approval coordinator unavailable (server not initialized for approvals)', + }, + }, + }); + + return app + .openapi(getPendingApprovalsRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('query'); + + agent.logger.debug(`Fetching pending approvals for session ${sessionId}`); + + // Get all pending approval IDs from the approval manager + const pendingIds = agent.services.approvalManager.getPendingApprovals(); + + // For now, return basic approval info + // Full metadata would require storing approval requests in the coordinator + const approvals = pendingIds.map((approvalId) => ({ + approvalId, + type: 'tool_confirmation', // Default type + sessionId, + timestamp: new Date().toISOString(), + metadata: {}, + })); + + return ctx.json({ + ok: true as const, + approvals, + }); + }) + .openapi(submitApprovalRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { approvalId } = ctx.req.valid('param'); + const { status, formData, rememberChoice } = ctx.req.valid('json'); + + agent.logger.info(`Received approval decision for ${approvalId}: ${status}`); + + if (!approvalCoordinator) { + agent.logger.error('ApprovalCoordinator not available'); + return ctx.json({ ok: false as const, approvalId, status }, 503); + } + + // Validate that the approval exists + const pendingApprovals = agent.services.approvalManager.getPendingApprovals(); + if (!pendingApprovals.includes(approvalId)) { + throw ApprovalError.notFound(approvalId); + } + + try { + // Build data object for approved requests + const data: Record = {}; + if (status === ApprovalStatus.APPROVED) { + if (formData !== undefined) { + data.formData = formData; + } + if (rememberChoice !== undefined) { + data.rememberChoice = rememberChoice; + } + } + + // Construct response payload + // Get sessionId from coordinator's mapping (stored when request was emitted) + const sessionId = approvalCoordinator.getSessionId(approvalId); + const responsePayload = { + approvalId, + status, + sessionId, // Attach sessionId for SSE routing to correct client streams + ...(status === ApprovalStatus.DENIED + ? { + reason: DenialReason.USER_DENIED, + message: 'User denied the request via API', + } + : {}), + ...(Object.keys(data).length > 0 ? { data } : {}), + }; + + // Emit via approval coordinator which ManualApprovalHandler listens to + approvalCoordinator.emitResponse(responsePayload); + + return ctx.json({ + ok: true, + approvalId, + status, + }); + } catch (error) { + agent.logger.error('Error processing approval', { approvalId, error }); + return ctx.json({ ok: false as const, approvalId, status }, 500); + } + }); +} diff --git a/dexto/packages/server/src/hono/routes/dexto-auth.ts b/dexto/packages/server/src/hono/routes/dexto-auth.ts new file mode 100644 index 00000000..cd4d5ed1 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/dexto-auth.ts @@ -0,0 +1,65 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { GetAgentFn } from '../index.js'; +import { + isDextoAuthEnabled, + isDextoAuthenticated, + canUseDextoProvider, +} from '@dexto/agent-management'; + +/** + * Dexto authentication status routes. + * Provides endpoints to check dexto auth status for Web UI. + */ +export function createDextoAuthRouter(_getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const statusRoute = createRoute({ + method: 'get', + path: '/dexto-auth/status', + summary: 'Dexto Auth Status', + description: + 'Returns dexto authentication status. Used by Web UI to check if user can use dexto features.', + tags: ['auth'], + responses: { + 200: { + description: 'Dexto auth status', + content: { + 'application/json': { + schema: z.object({ + enabled: z.boolean().describe('Whether dexto auth feature is enabled'), + authenticated: z + .boolean() + .describe('Whether user is authenticated with dexto'), + canUse: z + .boolean() + .describe( + 'Whether user can use dexto (authenticated AND has API key)' + ), + }), + }, + }, + }, + }, + }); + + return app.openapi(statusRoute, async (c) => { + const enabled = isDextoAuthEnabled(); + + if (!enabled) { + return c.json({ + enabled: false, + authenticated: false, + canUse: false, + }); + } + + const authenticated = await isDextoAuthenticated(); + const canUse = await canUseDextoProvider(); + + return c.json({ + enabled, + authenticated, + canUse, + }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/discovery.ts b/dexto/packages/server/src/hono/routes/discovery.ts new file mode 100644 index 00000000..370c3458 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/discovery.ts @@ -0,0 +1,64 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import { listAllProviders } from '@dexto/core'; + +const DiscoveredProviderSchema = z + .object({ + type: z.string().describe('Provider type identifier'), + category: z + .enum(['blob', 'database', 'compaction', 'customTools']) + .describe('Provider category'), + metadata: z + .object({ + displayName: z.string().optional().describe('Human-readable display name'), + description: z.string().optional().describe('Provider description'), + }) + .passthrough() + .optional() + .describe('Optional metadata about the provider'), + }) + .describe('Information about a registered provider'); + +const InternalToolSchema = z + .object({ + name: z + .string() + .describe('Internal tool name identifier (e.g., "search_history", "ask_user")'), + description: z.string().describe('Human-readable description of what the tool does'), + }) + .describe('Information about an internal tool'); + +const DiscoveryResponseSchema = z + .object({ + blob: z.array(DiscoveredProviderSchema).describe('Blob storage providers'), + database: z.array(DiscoveredProviderSchema).describe('Database providers'), + compaction: z.array(DiscoveredProviderSchema).describe('Compaction strategy providers'), + customTools: z.array(DiscoveredProviderSchema).describe('Custom tool providers'), + internalTools: z + .array(InternalToolSchema) + .describe('Internal tools available for configuration'), + }) + .describe('Discovery response with providers grouped by category'); + +export function createDiscoveryRouter() { + const app = new OpenAPIHono(); + + const discoveryRoute = createRoute({ + method: 'get', + path: '/discovery', + summary: 'Discover Available Providers and Tools', + description: + 'Returns all registered providers (blob storage, database, compaction, custom tools) and available internal tools. Useful for building UIs that need to display configurable options.', + tags: ['discovery'], + responses: { + 200: { + description: 'Available providers grouped by category', + content: { 'application/json': { schema: DiscoveryResponseSchema } }, + }, + }, + }); + + return app.openapi(discoveryRoute, async (ctx) => { + const providers = listAllProviders(); + return ctx.json(providers); + }); +} diff --git a/dexto/packages/server/src/hono/routes/greeting.ts b/dexto/packages/server/src/hono/routes/greeting.ts new file mode 100644 index 00000000..b4330d37 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/greeting.ts @@ -0,0 +1,48 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { GetAgentFn } from '../index.js'; + +const querySchema = z + .object({ + sessionId: z + .string() + .optional() + .describe('Session identifier to retrieve session-specific greeting'), + }) + .describe('Query parameters for greeting endpoint'); + +export function createGreetingRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const greetingRoute = createRoute({ + method: 'get', + path: '/greeting', + summary: 'Get Greeting Message', + description: 'Retrieves the greeting message from the agent configuration', + tags: ['config'], + request: { query: querySchema.pick({ sessionId: true }) }, + responses: { + 200: { + description: 'Greeting', + content: { + 'application/json': { + schema: z + .object({ + greeting: z + .string() + .optional() + .describe('Greeting message from agent configuration'), + }) + .strict(), + }, + }, + }, + }, + }); + + return app.openapi(greetingRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('query'); + const cfg = agent.getEffectiveConfig(sessionId); + return ctx.json({ greeting: cfg.greeting }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/health.ts b/dexto/packages/server/src/hono/routes/health.ts new file mode 100644 index 00000000..99737593 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/health.ts @@ -0,0 +1,25 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { GetAgentFn } from '../index.js'; + +/** + * NOTE: If we introduce a transport-agnostic handler layer later, the logic in this module can move + * into that layer. For now we keep the implementation inline for simplicity. + */ +export function createHealthRouter(_getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const route = createRoute({ + method: 'get', + path: '/', + summary: 'Health Check', + description: 'Returns server health status', + tags: ['system'], + responses: { + 200: { + description: 'Server health', + content: { 'text/plain': { schema: z.string().openapi({ example: 'OK' }) } }, + }, + }, + }); + return app.openapi(route, (c) => c.text('OK')); +} diff --git a/dexto/packages/server/src/hono/routes/key.ts b/dexto/packages/server/src/hono/routes/key.ts new file mode 100644 index 00000000..252f7dc8 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/key.ts @@ -0,0 +1,136 @@ +/** + * API Key Management Routes + * + * Endpoints for managing LLM provider API keys. + * + * TODO: For hosted deployments, these endpoints should integrate with a secure + * key management service (e.g., AWS Secrets Manager, HashiCorp Vault) rather + * than storing keys in local .env files. + */ + +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import { LLM_PROVIDERS } from '@dexto/core'; +import { + getProviderKeyStatus, + saveProviderApiKey, + resolveApiKeyForProvider, +} from '@dexto/agent-management'; + +/** + * Masks an API key for safe display, showing only prefix and suffix. + * @example maskApiKey('sk-proj-abc123xyz789') → 'sk-proj...z789' + */ +function maskApiKey(key: string): string { + if (!key) return ''; + if (key.length < 12) { + return key.slice(0, 4) + '...' + key.slice(-4); + } + return key.slice(0, 7) + '...' + key.slice(-4); +} + +const GetKeyParamsSchema = z + .object({ + provider: z.enum(LLM_PROVIDERS).describe('LLM provider identifier'), + }) + .describe('Path parameters for API key operations'); + +const SaveKeySchema = z + .object({ + provider: z + .enum(LLM_PROVIDERS) + .describe('LLM provider identifier (e.g., openai, anthropic)'), + apiKey: z + .string() + .min(1, 'API key is required') + .describe('API key for the provider (writeOnly - never returned in responses)') + .openapi({ writeOnly: true }), + }) + .describe('Request body for saving a provider API key'); + +export function createKeyRouter() { + const app = new OpenAPIHono(); + + const getKeyRoute = createRoute({ + method: 'get', + path: '/llm/key/{provider}', + summary: 'Get Provider API Key Status', + description: + 'Retrieves the API key status for a provider. Returns a masked key value (e.g., sk-proj...xyz4) for UI display purposes.', + tags: ['llm'], + request: { params: GetKeyParamsSchema }, + responses: { + 200: { + description: 'API key status and value', + content: { + 'application/json': { + schema: z + .object({ + provider: z.enum(LLM_PROVIDERS).describe('Provider identifier'), + envVar: z.string().describe('Environment variable name'), + hasKey: z.boolean().describe('Whether API key is configured'), + keyValue: z + .string() + .optional() + .describe( + 'Masked API key value if configured (e.g., sk-proj...xyz4)' + ), + }) + .strict() + .describe('API key status response'), + }, + }, + }, + }, + }); + + const saveKeyRoute = createRoute({ + method: 'post', + path: '/llm/key', + summary: 'Save Provider API Key', + description: 'Stores an API key for a provider in .env and makes it available immediately', + tags: ['llm'], + request: { body: { content: { 'application/json': { schema: SaveKeySchema } } } }, + responses: { + 200: { + description: 'API key saved', + content: { + 'application/json': { + schema: z + .object({ + ok: z.literal(true).describe('Operation success indicator'), + provider: z + .enum(LLM_PROVIDERS) + .describe('Provider for which the key was saved'), + envVar: z + .string() + .describe('Environment variable name where key was stored'), + }) + .strict() + .describe('API key save response'), + }, + }, + }, + }, + }); + + return app + .openapi(getKeyRoute, (ctx) => { + const { provider } = ctx.req.valid('param'); + const keyStatus = getProviderKeyStatus(provider); + const apiKey = resolveApiKeyForProvider(provider); + const maskedKey = apiKey ? maskApiKey(apiKey) : undefined; + + return ctx.json({ + provider, + envVar: keyStatus.envVar, + hasKey: keyStatus.hasApiKey, + ...(maskedKey && { keyValue: maskedKey }), + }); + }) + .openapi(saveKeyRoute, async (ctx) => { + const { provider, apiKey } = ctx.req.valid('json'); + // saveProviderApiKey uses getDextoEnvPath internally for context-aware .env resolution + const meta = await saveProviderApiKey(provider, apiKey); + return ctx.json({ ok: true as const, provider, envVar: meta.envVar }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/llm.ts b/dexto/packages/server/src/hono/routes/llm.ts new file mode 100644 index 00000000..2e4f5427 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/llm.ts @@ -0,0 +1,555 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { DextoAgent } from '@dexto/core'; +import { DextoRuntimeError, ErrorScope, ErrorType } from '@dexto/core'; +import { + LLM_REGISTRY, + LLM_PROVIDERS, + SUPPORTED_FILE_TYPES, + supportsBaseURL, + getAllModelsForProvider, + getSupportedFileTypesForModel, + type ProviderInfo, + type LLMProvider, + type SupportedFileType, + LLMUpdatesSchema, +} from '@dexto/core'; +import { + getProviderKeyStatus, + loadCustomModels, + saveCustomModel, + deleteCustomModel, + CustomModelSchema, + isDextoAuthEnabled, +} from '@dexto/agent-management'; +import type { Context } from 'hono'; +import { + ProviderCatalogSchema, + ModelFlatSchema, + LLMConfigResponseSchema, +} from '../schemas/responses.js'; + +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +const CurrentQuerySchema = z + .object({ + sessionId: z + .string() + .optional() + .describe('Session identifier to retrieve session-specific LLM configuration'), + }) + .describe('Query parameters for getting current LLM configuration'); + +const CatalogQuerySchema = z + .object({ + provider: z + .union([z.string(), z.array(z.string())]) + .optional() + .transform((value): string[] | undefined => + Array.isArray(value) ? value : value ? value.split(',') : undefined + ) + .describe('Comma-separated list of LLM providers to filter by'), + hasKey: z + .union([z.literal('true'), z.literal('false'), z.literal('1'), z.literal('0')]) + .optional() + .transform((raw): boolean | undefined => + raw === 'true' || raw === '1' + ? true + : raw === 'false' || raw === '0' + ? false + : undefined + ) + .describe('Filter by API key presence (true or false)'), + fileType: z + .enum(SUPPORTED_FILE_TYPES) + .optional() + .describe('Filter by supported file type (audio, pdf, or image)'), + defaultOnly: z + .union([z.literal('true'), z.literal('false'), z.literal('1'), z.literal('0')]) + .optional() + .transform((raw): boolean | undefined => + raw === 'true' || raw === '1' + ? true + : raw === 'false' || raw === '0' + ? false + : undefined + ) + .describe('Include only default models (true or false)'), + mode: z + .enum(['grouped', 'flat']) + .default('grouped') + .describe('Response format mode (grouped by provider or flat list)'), + }) + .describe('Query parameters for filtering and formatting the LLM catalog'); + +// Combine LLM updates schema with sessionId for API requests +// LLMUpdatesSchema is no longer strict, so it accepts extra fields like sessionId +const SwitchLLMBodySchema = LLMUpdatesSchema.and( + z.object({ + sessionId: z + .string() + .optional() + .describe('Session identifier for session-specific LLM configuration'), + }) +).describe('LLM switch request body with optional session ID and LLM fields'); + +export function createLlmRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const currentRoute = createRoute({ + method: 'get', + path: '/llm/current', + summary: 'Get Current LLM Config', + description: 'Retrieves the current LLM configuration for the agent or a specific session', + tags: ['llm'], + request: { query: CurrentQuerySchema }, + responses: { + 200: { + description: 'Current LLM config', + content: { + 'application/json': { + schema: z + .object({ + config: LLMConfigResponseSchema.partial({ + maxIterations: true, + }).extend({ + displayName: z + .string() + .optional() + .describe('Human-readable model display name'), + }), + routing: z + .object({ + viaDexto: z + .boolean() + .describe( + 'Whether requests route through Dexto gateway' + ), + }) + .describe( + 'Routing information for the current LLM configuration' + ), + }) + .describe('Response containing current LLM configuration'), + }, + }, + }, + }, + }); + const catalogRoute = createRoute({ + method: 'get', + path: '/llm/catalog', + summary: 'LLM Catalog', + description: 'Providers, models, capabilities, and API key status', + tags: ['llm'], + request: { query: CatalogQuerySchema }, + responses: { + 200: { + description: 'LLM catalog', + content: { + 'application/json': { + schema: z + .union([ + z + .object({ + providers: z + .record(z.enum(LLM_PROVIDERS), ProviderCatalogSchema) + .describe( + 'Providers grouped by ID with their models and capabilities' + ), + }) + .strict() + .describe('Grouped catalog response (mode=grouped)'), + z + .object({ + models: z + .array(ModelFlatSchema) + .describe( + 'Flat list of all models with provider information' + ), + }) + .strict() + .describe('Flat catalog response (mode=flat)'), + ]) + .describe( + 'LLM catalog in grouped or flat format based on mode query parameter' + ), + }, + }, + }, + }, + }); + + const switchRoute = createRoute({ + method: 'post', + path: '/llm/switch', + summary: 'Switch LLM', + description: 'Switches the LLM configuration for the agent or a specific session', + tags: ['llm'], + request: { + body: { + content: { + 'application/json': { + schema: SwitchLLMBodySchema, + }, + }, + }, + }, + responses: { + 200: { + description: 'LLM switch result', + content: { + 'application/json': { + schema: z + .object({ + config: LLMConfigResponseSchema.describe( + 'New LLM configuration with all defaults applied (apiKey omitted)' + ), + sessionId: z + .string() + .optional() + .describe('Session ID if session-specific switch'), + }) + .describe('LLM switch result'), + }, + }, + }, + }, + }); + + // Custom models routes + const listCustomModelsRoute = createRoute({ + method: 'get', + path: '/llm/custom-models', + summary: 'List Custom Models', + description: 'Returns all saved custom openai-compatible model configurations', + tags: ['llm'], + responses: { + 200: { + description: 'List of custom models', + content: { + 'application/json': { + schema: z.object({ + models: z.array(CustomModelSchema).describe('List of custom models'), + }), + }, + }, + }, + }, + }); + + const createCustomModelRoute = createRoute({ + method: 'post', + path: '/llm/custom-models', + summary: 'Create Custom Model', + description: 'Saves a new custom openai-compatible model configuration', + tags: ['llm'], + request: { + body: { content: { 'application/json': { schema: CustomModelSchema } } }, + }, + responses: { + 200: { + description: 'Custom model saved', + content: { + 'application/json': { + schema: z.object({ + ok: z.literal(true).describe('Success indicator'), + model: CustomModelSchema, + }), + }, + }, + }, + }, + }); + + const deleteCustomModelRoute = createRoute({ + method: 'delete', + path: '/llm/custom-models/{name}', + summary: 'Delete Custom Model', + description: 'Deletes a custom model by name', + tags: ['llm'], + request: { + params: z.object({ + name: z.string().min(1).describe('Model name to delete'), + }), + }, + responses: { + 200: { + description: 'Custom model deleted', + content: { + 'application/json': { + schema: z.object({ + ok: z.literal(true).describe('Success indicator'), + deleted: z.string().describe('Name of the deleted model'), + }), + }, + }, + }, + 404: { + description: 'Custom model not found', + content: { + 'application/json': { + schema: z.object({ + ok: z.literal(false).describe('Failure indicator'), + error: z.string().describe('Error message'), + }), + }, + }, + }, + }, + }); + + // Model capabilities endpoint - resolves gateway providers to underlying model capabilities + const capabilitiesRoute = createRoute({ + method: 'get', + path: '/llm/capabilities', + summary: 'Get Model Capabilities', + description: + 'Returns the capabilities (supported file types) for a specific provider/model combination. ' + + 'Handles gateway providers (dexto, openrouter) by resolving to the underlying model capabilities.', + tags: ['llm'], + request: { + query: z.object({ + provider: z.enum(LLM_PROVIDERS).describe('LLM provider name'), + model: z + .string() + .min(1) + .describe('Model name (supports both native and OpenRouter format)'), + }), + }, + responses: { + 200: { + description: 'Model capabilities', + content: { + 'application/json': { + schema: z.object({ + provider: z.enum(LLM_PROVIDERS).describe('Provider name'), + model: z.string().describe('Model name as provided'), + supportedFileTypes: z + .array(z.enum(SUPPORTED_FILE_TYPES)) + .describe('File types supported by this model'), + }), + }, + }, + }, + }, + }); + + return app + .openapi(currentRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('query'); + + const currentConfig = sessionId + ? agent.getEffectiveConfig(sessionId).llm + : agent.getCurrentLLMConfig(); + + let displayName: string | undefined; + try { + // First check registry for built-in models + const model = LLM_REGISTRY[currentConfig.provider]?.models.find( + (m) => m.name.toLowerCase() === String(currentConfig.model).toLowerCase() + ); + displayName = model?.displayName || undefined; + + // If not found in registry, check custom models + if (!displayName) { + const customModels = await loadCustomModels(); + const customModel = customModels.find( + (cm) => cm.name.toLowerCase() === String(currentConfig.model).toLowerCase() + ); + displayName = customModel?.displayName || undefined; + } + } catch { + // ignore lookup errors + } + + // Omit apiKey from response for security + const { apiKey, ...configWithoutKey } = currentConfig; + + // With explicit providers, viaDexto is simply whether the provider is 'dexto' + // Only report viaDexto when the feature is enabled + const viaDexto = isDextoAuthEnabled() && currentConfig.provider === 'dexto'; + + return ctx.json({ + config: { + ...configWithoutKey, + hasApiKey: !!apiKey, + ...(displayName && { displayName }), + }, + routing: { + viaDexto, + }, + }); + }) + .openapi(catalogRoute, (ctx) => { + type ProviderCatalog = Pick & { + name: string; + hasApiKey: boolean; + primaryEnvVar: string; + supportsBaseURL: boolean; + }; + + type ModelFlat = ProviderCatalog['models'][number] & { provider: LLMProvider }; + + const queryParams = ctx.req.valid('query'); + + const providers: Record = {}; + + for (const provider of LLM_PROVIDERS) { + // Skip dexto provider when feature is not enabled + if (provider === 'dexto' && !isDextoAuthEnabled()) { + continue; + } + + const info = LLM_REGISTRY[provider]; + const displayName = provider.charAt(0).toUpperCase() + provider.slice(1); + const keyStatus = getProviderKeyStatus(provider); + + // Use getAllModelsForProvider to get inherited models for gateway providers + // like 'dexto' that have supportsAllRegistryModels: true + const models = getAllModelsForProvider(provider); + + providers[provider] = { + name: displayName, + hasApiKey: keyStatus.hasApiKey, + primaryEnvVar: keyStatus.envVar, + supportsBaseURL: supportsBaseURL(provider), + models, + supportedFileTypes: info.supportedFileTypes, + }; + } + + let filtered: Record = { ...providers }; + + if (queryParams.provider && queryParams.provider.length > 0) { + const allowed = new Set( + queryParams.provider.filter((p) => + (LLM_PROVIDERS as readonly string[]).includes(p) + ) + ); + const filteredByProvider: Record = {}; + for (const [id, catalog] of Object.entries(filtered)) { + if (allowed.has(id)) { + filteredByProvider[id] = catalog; + } + } + filtered = filteredByProvider; + } + + if (typeof queryParams.hasKey === 'boolean') { + const byKey: Record = {}; + for (const [id, catalog] of Object.entries(filtered)) { + if (catalog.hasApiKey === queryParams.hasKey) { + byKey[id] = catalog; + } + } + filtered = byKey; + } + + if (queryParams.fileType) { + const byFileType: Record = {}; + for (const [id, catalog] of Object.entries(filtered)) { + const models = catalog.models.filter((model) => { + const modelTypes = + Array.isArray(model.supportedFileTypes) && + model.supportedFileTypes.length > 0 + ? model.supportedFileTypes + : catalog.supportedFileTypes || []; + return modelTypes.includes(queryParams.fileType!); + }); + if (models.length > 0) { + byFileType[id] = { ...catalog, models }; + } + } + filtered = byFileType; + } + + if (queryParams.defaultOnly) { + const byDefault: Record = {}; + for (const [id, catalog] of Object.entries(filtered)) { + const models = catalog.models.filter((model) => model.default === true); + if (models.length > 0) { + byDefault[id] = { ...catalog, models }; + } + } + filtered = byDefault; + } + + if (queryParams.mode === 'flat') { + const flat: ModelFlat[] = []; + for (const [id, catalog] of Object.entries(filtered)) { + for (const model of catalog.models) { + flat.push({ provider: id as LLMProvider, ...model }); + } + } + return ctx.json({ models: flat }); + } + + return ctx.json({ providers: filtered }); + }) + .openapi(switchRoute, async (ctx) => { + const agent = await getAgent(ctx); + const raw = ctx.req.valid('json'); + const { sessionId, ...llmUpdates } = raw; + + const config = await agent.switchLLM(llmUpdates, sessionId); + + // Omit apiKey from response for security + const { apiKey, ...configWithoutKey } = config; + return ctx.json({ + config: { + ...configWithoutKey, + hasApiKey: !!apiKey, + }, + sessionId, + }); + }) + .openapi(listCustomModelsRoute, async (ctx) => { + const models = await loadCustomModels(); + return ctx.json({ models }); + }) + .openapi(createCustomModelRoute, async (ctx) => { + const model = ctx.req.valid('json'); + await saveCustomModel(model); + return ctx.json({ ok: true as const, model }); + }) + .openapi(deleteCustomModelRoute, async (ctx) => { + const { name: encodedName } = ctx.req.valid('param'); + // Decode URL-encoded name to handle OpenRouter model IDs with slashes + const name = decodeURIComponent(encodedName); + const deleted = await deleteCustomModel(name); + if (!deleted) { + throw new DextoRuntimeError( + 'custom_model_not_found', + ErrorScope.LLM, + ErrorType.NOT_FOUND, + `Custom model '${name}' not found`, + { modelName: name } + ); + } + return ctx.json({ ok: true as const, deleted: name } as const, 200); + }) + .openapi(capabilitiesRoute, (ctx) => { + const { provider, model } = ctx.req.valid('query'); + + // getSupportedFileTypesForModel handles: + // 1. Gateway providers (dexto, openrouter) - resolves via resolveModelOrigin to underlying model + // 2. Native providers - direct lookup in registry + // 3. Custom model providers (openai-compatible) - returns provider-level capabilities + // Falls back to provider-level supportedFileTypes if model not found + let supportedFileTypes: SupportedFileType[]; + try { + supportedFileTypes = getSupportedFileTypesForModel(provider, model); + } catch { + // If model lookup fails, fall back to provider-level capabilities + const providerInfo = LLM_REGISTRY[provider]; + supportedFileTypes = providerInfo?.supportedFileTypes ?? []; + } + + return ctx.json({ + provider, + model, + supportedFileTypes, + }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/mcp.ts b/dexto/packages/server/src/hono/routes/mcp.ts new file mode 100644 index 00000000..36d515f4 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/mcp.ts @@ -0,0 +1,446 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import { logger, McpServerConfigSchema, MCP_CONNECTION_STATUSES } from '@dexto/core'; +import { updateAgentConfigFile } from '@dexto/agent-management'; +import { ResourceSchema } from '../schemas/responses.js'; +import type { GetAgentFn } from '../index.js'; + +const McpServerRequestSchema = z + .object({ + name: z.string().min(1, 'Server name is required').describe('A unique name for the server'), + config: McpServerConfigSchema.describe('The server configuration object'), + persistToAgent: z + .boolean() + .optional() + .describe('If true, saves the server to agent configuration file'), + }) + .describe('Request body for adding or updating an MCP server'); + +const ExecuteToolBodySchema = z + .record(z.unknown()) + .describe( + "Tool execution parameters as JSON object. The specific fields depend on the tool being executed and are defined by the tool's inputSchema." + ); + +// Response schemas +const ServerStatusResponseSchema = z + .object({ + status: z.string().describe('Connection status'), + name: z.string().describe('Server name'), + }) + .strict() + .describe('Server status response'); + +const ServerInfoSchema = z + .object({ + id: z.string().describe('Server identifier'), + name: z.string().describe('Server name'), + status: z.enum(MCP_CONNECTION_STATUSES).describe('Server status'), + }) + .strict() + .describe('MCP server information'); + +const ServersListResponseSchema = z + .object({ + servers: z.array(ServerInfoSchema).describe('Array of server information'), + }) + .strict() + .describe('List of MCP servers'); + +// JSON Schema definition for tool input parameters (based on MCP SDK Tool type) +const JsonSchemaProperty = z + .object({ + type: z + .enum(['string', 'number', 'integer', 'boolean', 'object', 'array']) + .optional() + .describe('Property type'), + description: z.string().optional().describe('Property description'), + enum: z + .array(z.union([z.string(), z.number(), z.boolean()])) + .optional() + .describe('Enum values'), + default: z.any().optional().describe('Default value'), + }) + .passthrough() + .describe('JSON Schema property definition'); + +const ToolInputSchema = z + .object({ + type: z.literal('object').optional().describe('Schema type, always "object" when present'), + properties: z.record(JsonSchemaProperty).optional().describe('Property definitions'), + required: z.array(z.string()).optional().describe('Required property names'), + }) + .passthrough() + .describe('JSON Schema for tool input parameters'); + +const ToolInfoSchema = z + .object({ + id: z.string().describe('Tool identifier'), + name: z.string().describe('Tool name'), + description: z.string().describe('Tool description'), + inputSchema: ToolInputSchema.optional().describe('JSON Schema for tool input parameters'), + }) + .strict() + .describe('Tool information'); + +const ToolsListResponseSchema = z + .object({ + tools: z.array(ToolInfoSchema).describe('Array of available tools'), + }) + .strict() + .describe('List of tools from MCP server'); + +const DisconnectResponseSchema = z + .object({ + status: z.literal('disconnected').describe('Disconnection status'), + id: z.string().describe('Server identifier'), + }) + .strict() + .describe('Server disconnection response'); + +const RestartResponseSchema = z + .object({ + status: z.literal('restarted').describe('Restart status'), + id: z.string().describe('Server identifier'), + }) + .strict() + .describe('Server restart response'); + +const ToolExecutionResponseSchema = z + .object({ + success: z.boolean().describe('Whether tool execution succeeded'), + data: z.any().optional().describe('Tool execution result data'), + error: z.string().optional().describe('Error message if execution failed'), + }) + .strict() + .describe('Tool execution response'); + +const ResourcesListResponseSchema = z + .object({ + success: z.boolean().describe('Success indicator'), + resources: z.array(ResourceSchema).describe('Array of available resources'), + }) + .strict() + .describe('List of resources from MCP server'); + +const ResourceContentSchema = z + .object({ + content: z.any().describe('Resource content data'), + }) + .strict() + .describe('Resource content wrapper'); + +const ResourceContentResponseSchema = z + .object({ + success: z.boolean().describe('Success indicator'), + data: ResourceContentSchema.describe('Resource content'), + }) + .strict() + .describe('Resource content response'); + +export function createMcpRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const addServerRoute = createRoute({ + method: 'post', + path: '/mcp/servers', + summary: 'Add MCP Server', + description: 'Connects a new MCP server dynamically', + tags: ['mcp'], + request: { body: { content: { 'application/json': { schema: McpServerRequestSchema } } } }, + responses: { + 200: { + description: 'Server connected', + content: { 'application/json': { schema: ServerStatusResponseSchema } }, + }, + }, + }); + const listServersRoute = createRoute({ + method: 'get', + path: '/mcp/servers', + summary: 'List MCP Servers', + description: 'Gets a list of all connected and failed MCP servers', + tags: ['mcp'], + responses: { + 200: { + description: 'Servers list', + content: { 'application/json': { schema: ServersListResponseSchema } }, + }, + }, + }); + + const toolsRoute = createRoute({ + method: 'get', + path: '/mcp/servers/{serverId}/tools', + summary: 'List Server Tools', + description: 'Retrieves the list of tools available on a specific MCP server', + tags: ['mcp'], + request: { + params: z.object({ serverId: z.string().describe('The ID of the MCP server') }), + }, + responses: { + 200: { + description: 'Tools list', + content: { 'application/json': { schema: ToolsListResponseSchema } }, + }, + 404: { description: 'Not found' }, + }, + }); + + const deleteServerRoute = createRoute({ + method: 'delete', + path: '/mcp/servers/{serverId}', + summary: 'Remove MCP Server', + description: 'Disconnects and removes an MCP server', + tags: ['mcp'], + request: { + params: z.object({ serverId: z.string().describe('The ID of the MCP server') }), + }, + responses: { + 200: { + description: 'Disconnected', + content: { 'application/json': { schema: DisconnectResponseSchema } }, + }, + 404: { description: 'Not found' }, + }, + }); + + const restartServerRoute = createRoute({ + method: 'post', + path: '/mcp/servers/{serverId}/restart', + summary: 'Restart MCP Server', + description: 'Restarts a connected MCP server', + tags: ['mcp'], + request: { + params: z.object({ serverId: z.string().describe('The ID of the MCP server') }), + }, + responses: { + 200: { + description: 'Server restarted', + content: { 'application/json': { schema: RestartResponseSchema } }, + }, + 404: { description: 'Not found' }, + }, + }); + + const execToolRoute = createRoute({ + method: 'post', + path: '/mcp/servers/{serverId}/tools/{toolName}/execute', + summary: 'Execute MCP Tool', + description: 'Executes a tool on an MCP server directly', + tags: ['mcp'], + request: { + params: z.object({ + serverId: z.string().describe('The ID of the MCP server'), + toolName: z.string().describe('The name of the tool to execute'), + }), + body: { content: { 'application/json': { schema: ExecuteToolBodySchema } } }, + }, + responses: { + 200: { + description: 'Tool executed', + content: { 'application/json': { schema: ToolExecutionResponseSchema } }, + }, + 404: { description: 'Not found' }, + }, + }); + + const listResourcesRoute = createRoute({ + method: 'get', + path: '/mcp/servers/{serverId}/resources', + summary: 'List Server Resources', + description: 'Retrieves all resources available from a specific MCP server', + tags: ['mcp'], + request: { + params: z.object({ serverId: z.string().describe('The ID of the MCP server') }), + }, + responses: { + 200: { + description: 'Server resources', + content: { 'application/json': { schema: ResourcesListResponseSchema } }, + }, + 404: { description: 'Not found' }, + }, + }); + + const getResourceContentRoute = createRoute({ + method: 'get', + path: '/mcp/servers/{serverId}/resources/{resourceId}/content', + summary: 'Read Server Resource Content', + description: + 'Reads content from a specific resource on an MCP server. This endpoint automatically constructs the qualified URI format (mcp:serverId:resourceId)', + tags: ['mcp'], + request: { + params: z.object({ + serverId: z.string().describe('The ID of the MCP server'), + resourceId: z + .string() + .min(1, 'Resource ID is required') + .transform((encoded) => decodeURIComponent(encoded)) + .describe('The URI-encoded resource identifier on that server'), + }), + }, + responses: { + 200: { + description: 'Resource content', + content: { 'application/json': { schema: ResourceContentResponseSchema } }, + }, + 404: { description: 'Not found' }, + }, + }); + + return app + .openapi(addServerRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { name, config, persistToAgent } = ctx.req.valid('json'); + + // Add the server (connects if enabled, otherwise just registers) + await agent.addMcpServer(name, config); + const isConnected = config.enabled !== false; + logger.info( + isConnected + ? `Successfully connected to new server '${name}' via API request.` + : `Registered server '${name}' (disabled) via API request.` + ); + + // If persistToAgent is true, save to agent config file + if (persistToAgent === true) { + try { + // Get the current effective config to read existing mcpServers + const currentConfig = agent.getEffectiveConfig(); + + // Create update with new server added to mcpServers + const updates = { + mcpServers: { + ...(currentConfig.mcpServers || {}), + [name]: config, + }, + }; + + // Write to file (agent-management's job) + const newConfig = await updateAgentConfigFile( + agent.getAgentFilePath(), + updates + ); + + // Reload into agent (core's job - handles restart automatically) + const reloadResult = await agent.reload(newConfig); + if (reloadResult.restarted) { + logger.info( + `Agent restarted to apply changes: ${reloadResult.changesApplied.join(', ')}` + ); + } + logger.info(`Saved server '${name}' to agent configuration file`); + } catch (saveError) { + const errorMessage = + saveError instanceof Error ? saveError.message : String(saveError); + logger.warn( + `Failed to save server '${name}' to agent config: ${errorMessage}`, + { + error: saveError, + } + ); + // Don't fail the request if saving fails - server is still connected + } + } + + const status = isConnected ? 'connected' : 'registered'; + return ctx.json({ status, name }, 200); + }) + .openapi(listServersRoute, async (ctx) => { + const agent = await getAgent(ctx); + const clientsMap = agent.getMcpClients(); + const failedConnections = agent.getMcpFailedConnections(); + const servers: z.output[] = []; + for (const name of clientsMap.keys()) { + servers.push({ id: name, name, status: 'connected' }); + } + for (const name of Object.keys(failedConnections)) { + servers.push({ id: name, name, status: 'error' }); + } + return ctx.json({ servers }); + }) + .openapi(toolsRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { serverId } = ctx.req.valid('param'); + const client = agent.getMcpClients().get(serverId); + if (!client) { + return ctx.json({ error: `Server '${serverId}' not found` }, 404); + } + const toolsMap = await client.getTools(); + const tools = Object.entries(toolsMap).map(([toolName, toolDef]) => ({ + id: toolName, + name: toolName, + description: toolDef.description || '', + inputSchema: toolDef.parameters, + })); + return ctx.json({ tools }); + }) + .openapi(deleteServerRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { serverId } = ctx.req.valid('param'); + const clientExists = + agent.getMcpClients().has(serverId) || agent.getMcpFailedConnections()[serverId]; + if (!clientExists) { + return ctx.json({ error: `Server '${serverId}' not found.` }, 404); + } + + await agent.removeMcpServer(serverId); + return ctx.json({ status: 'disconnected', id: serverId }); + }) + .openapi(restartServerRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { serverId } = ctx.req.valid('param'); + logger.info(`Received request to POST /api/mcp/servers/${serverId}/restart`); + + const clientExists = agent.getMcpClients().has(serverId); + if (!clientExists) { + logger.warn(`Attempted to restart non-existent server: ${serverId}`); + return ctx.json({ error: `Server '${serverId}' not found.` }, 404); + } + + await agent.restartMcpServer(serverId); + return ctx.json({ status: 'restarted', id: serverId }); + }) + .openapi(execToolRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { serverId, toolName } = ctx.req.valid('param'); + const body = ctx.req.valid('json'); + const client = agent.getMcpClients().get(serverId); + if (!client) { + return ctx.json({ success: false, error: `Server '${serverId}' not found` }, 404); + } + // Execute tool directly on the specified server (matches Express implementation) + try { + const rawResult = await client.callTool(toolName, body); + return ctx.json({ success: true, data: rawResult }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error( + `Tool execution failed for '${toolName}' on server '${serverId}': ${errorMessage}`, + { error } + ); + return ctx.json({ success: false, error: errorMessage }, 200); + } + }) + .openapi(listResourcesRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { serverId } = ctx.req.valid('param'); + const client = agent.getMcpClients().get(serverId); + if (!client) { + return ctx.json({ error: `Server '${serverId}' not found` }, 404); + } + const resources = await agent.listResourcesForServer(serverId); + return ctx.json({ success: true, resources }); + }) + .openapi(getResourceContentRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { serverId, resourceId } = ctx.req.valid('param'); + const client = agent.getMcpClients().get(serverId); + if (!client) { + return ctx.json({ error: `Server '${serverId}' not found` }, 404); + } + const qualifiedUri = `mcp:${serverId}:${resourceId}`; + const content = await agent.readResource(qualifiedUri); + return ctx.json({ success: true, data: { content } }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/memory.ts b/dexto/packages/server/src/hono/routes/memory.ts new file mode 100644 index 00000000..b75cc5e1 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/memory.ts @@ -0,0 +1,233 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { DextoAgent } from '@dexto/core'; +import { CreateMemoryInputSchema, UpdateMemoryInputSchema } from '@dexto/core'; +import { MemorySchema } from '../schemas/responses.js'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +const MemoryIdParamSchema = z + .object({ + id: z.string().min(1, 'Memory ID is required').describe('Memory unique identifier'), + }) + .describe('Path parameters for memory endpoints'); + +const ListMemoriesQuerySchema = z + .object({ + tags: z + .string() + .optional() + .transform((val) => (val ? val.split(',').map((t) => t.trim()) : undefined)) + .describe('Comma-separated list of tags to filter by'), + source: z.enum(['user', 'system']).optional().describe('Filter by source (user or system)'), + pinned: z + .string() + .optional() + .transform((val) => (val === 'true' ? true : val === 'false' ? false : undefined)) + .describe('Filter by pinned status (true or false)'), + limit: z + .string() + .optional() + .transform((val) => (val ? parseInt(val, 10) : undefined)) + .describe('Maximum number of memories to return'), + offset: z + .string() + .optional() + .transform((val) => (val ? parseInt(val, 10) : undefined)) + .describe('Number of memories to skip'), + }) + .describe('Query parameters for listing and filtering memories'); + +// Response schemas +const MemoryResponseSchema = z + .object({ + ok: z.literal(true).describe('Indicates successful response'), + memory: MemorySchema.describe('The created or retrieved memory'), + }) + .strict() + .describe('Single memory response'); + +const MemoriesListResponseSchema = z + .object({ + ok: z.literal(true).describe('Indicates successful response'), + memories: z.array(MemorySchema).describe('List of memories'), + }) + .strict() + .describe('Multiple memories response'); + +const MemoryDeleteResponseSchema = z + .object({ + ok: z.literal(true).describe('Indicates successful response'), + message: z.string().describe('Deletion confirmation message'), + }) + .strict() + .describe('Memory deletion response'); + +export function createMemoryRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const createMemoryRoute = createRoute({ + method: 'post', + path: '/memory', + summary: 'Create Memory', + description: 'Creates a new memory', + tags: ['memory'], + request: { + body: { + content: { + 'application/json': { + schema: CreateMemoryInputSchema, + }, + }, + }, + }, + responses: { + 201: { + description: 'Memory created', + content: { 'application/json': { schema: MemoryResponseSchema } }, + }, + }, + }); + + const listRoute = createRoute({ + method: 'get', + path: '/memory', + summary: 'List Memories', + description: 'Retrieves a list of all memories with optional filtering', + tags: ['memory'], + request: { query: ListMemoriesQuerySchema }, + responses: { + 200: { + description: 'List memories', + content: { 'application/json': { schema: MemoriesListResponseSchema } }, + }, + }, + }); + + const getRoute = createRoute({ + method: 'get', + path: '/memory/{id}', + summary: 'Get Memory by ID', + description: 'Retrieves a specific memory by its unique identifier', + tags: ['memory'], + request: { + params: MemoryIdParamSchema, + }, + responses: { + 200: { + description: 'Memory details', + content: { 'application/json': { schema: MemoryResponseSchema } }, + }, + }, + }); + + const updateRoute = createRoute({ + method: 'put', + path: '/memory/{id}', + summary: 'Update Memory', + description: 'Updates an existing memory. Only provided fields will be updated', + tags: ['memory'], + request: { + params: MemoryIdParamSchema, + body: { + content: { + 'application/json': { + schema: UpdateMemoryInputSchema, + }, + }, + }, + }, + responses: { + 200: { + description: 'Memory updated', + content: { 'application/json': { schema: MemoryResponseSchema } }, + }, + }, + }); + + const deleteRoute = createRoute({ + method: 'delete', + path: '/memory/{id}', + summary: 'Delete Memory', + description: 'Permanently deletes a memory. This action cannot be undone', + tags: ['memory'], + request: { + params: MemoryIdParamSchema, + }, + responses: { + 200: { + description: 'Memory deleted', + content: { 'application/json': { schema: MemoryDeleteResponseSchema } }, + }, + }, + }); + + return app + .openapi(createMemoryRoute, async (ctx) => { + const input = ctx.req.valid('json'); + + // Filter out undefined values for exactOptionalPropertyTypes compatibility + const createInput: { + content: string; + tags?: string[]; + metadata?: Record; + } = { + content: input.content, + }; + if (input.tags !== undefined && Array.isArray(input.tags)) { + createInput.tags = input.tags; + } + if (input.metadata !== undefined) { + createInput.metadata = input.metadata; + } + const agent = await getAgent(ctx); + const memory = await agent.memoryManager.create(createInput); + return ctx.json({ ok: true as const, memory }, 201); + }) + .openapi(listRoute, async (ctx) => { + const query = ctx.req.valid('query'); + const options: { + tags?: string[]; + source?: 'user' | 'system'; + pinned?: boolean; + limit?: number; + offset?: number; + } = {}; + if (query.tags !== undefined) options.tags = query.tags; + if (query.source !== undefined) options.source = query.source; + if (query.pinned !== undefined) options.pinned = query.pinned; + if (query.limit !== undefined) options.limit = query.limit; + if (query.offset !== undefined) options.offset = query.offset; + + const agent = await getAgent(ctx); + const memories = await agent.memoryManager.list(options); + return ctx.json({ ok: true as const, memories }); + }) + .openapi(getRoute, async (ctx) => { + const { id } = ctx.req.valid('param'); + const agent = await getAgent(ctx); + const memory = await agent.memoryManager.get(id); + return ctx.json({ ok: true as const, memory }); + }) + .openapi(updateRoute, async (ctx) => { + const { id } = ctx.req.valid('param'); + const updatesRaw = ctx.req.valid('json'); + // Build updates object only with defined properties for exactOptionalPropertyTypes + const updates: { + content?: string; + metadata?: Record; + tags?: string[]; + } = {}; + if (updatesRaw.content !== undefined) updates.content = updatesRaw.content; + if (updatesRaw.metadata !== undefined) updates.metadata = updatesRaw.metadata; + if (updatesRaw.tags !== undefined) updates.tags = updatesRaw.tags; + const agent = await getAgent(ctx); + const memory = await agent.memoryManager.update(id, updates); + return ctx.json({ ok: true as const, memory }); + }) + .openapi(deleteRoute, async (ctx) => { + const { id } = ctx.req.valid('param'); + const agent = await getAgent(ctx); + await agent.memoryManager.delete(id); + return ctx.json({ ok: true as const, message: 'Memory deleted successfully' }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/messages.ts b/dexto/packages/server/src/hono/routes/messages.ts new file mode 100644 index 00000000..1a8d0055 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/messages.ts @@ -0,0 +1,397 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import { streamSSE } from 'hono/streaming'; +import type { ContentInput } from '@dexto/core'; +import { LLM_PROVIDERS } from '@dexto/core'; +import type { ApprovalCoordinator } from '../../approval/approval-coordinator.js'; +import { TokenUsageSchema } from '../schemas/responses.js'; +import type { GetAgentFn } from '../index.js'; + +// ContentPart schemas matching @dexto/core types +// TODO: The Zod-inferred types don't exactly match core's ContentInput due to +// exactOptionalPropertyTypes (Zod infers `mimeType?: string | undefined` vs core's `mimeType?: string`). +// We cast to ContentInput after validation. Fix by either: +// 1. Export Zod schemas from @dexto/core and reuse here +// 2. Use .transform() to convert to exact types +// 3. Relax exactOptionalPropertyTypes in tsconfig +const TextPartSchema = z + .object({ + type: z.literal('text').describe('Content type identifier'), + text: z.string().describe('Text content'), + }) + .describe('Text content part'); + +const ImagePartSchema = z + .object({ + type: z.literal('image').describe('Content type identifier'), + image: z.string().describe('Base64-encoded image data or URL'), + mimeType: z.string().optional().describe('MIME type (e.g., image/png)'), + }) + .describe('Image content part'); + +const FilePartSchema = z + .object({ + type: z.literal('file').describe('Content type identifier'), + data: z.string().describe('Base64-encoded file data or URL'), + mimeType: z.string().describe('MIME type (e.g., application/pdf)'), + filename: z.string().optional().describe('Optional filename'), + }) + .describe('File content part'); + +const ContentPartSchema = z + .discriminatedUnion('type', [TextPartSchema, ImagePartSchema, FilePartSchema]) + .describe('Content part - text, image, or file'); + +const MessageBodySchema = z + .object({ + content: z + .union([z.string(), z.array(ContentPartSchema)]) + .describe('Message content - string for text, or ContentPart[] for multimodal'), + sessionId: z + .string() + .min(1, 'Session ID is required') + .describe('The session to use for this message'), + }) + .describe('Request body for sending a message to the agent'); + +const ResetBodySchema = z + .object({ + sessionId: z + .string() + .min(1, 'Session ID is required') + .describe('The ID of the session to reset'), + }) + .describe('Request body for resetting a conversation'); + +export function createMessagesRouter( + getAgent: GetAgentFn, + approvalCoordinator?: ApprovalCoordinator +) { + const app = new OpenAPIHono(); + + // TODO: Deprecate this endpoint - this async pattern is problematic and should be replaced + // with a proper job queue or streaming-only approach. Consider removing in next major version. + // Users should use /message/sync for synchronous responses or SSE for streaming. + const messageRoute = createRoute({ + method: 'post', + path: '/message', + summary: 'Send Message (async)', + description: + 'Sends a message and returns immediately. The full response will be sent over SSE', + tags: ['messages'], + request: { + body: { + content: { 'application/json': { schema: MessageBodySchema } }, + }, + }, + responses: { + 202: { + description: 'Message accepted for async processing; subscribe to SSE for results', + content: { + 'application/json': { + schema: z + .object({ + accepted: z + .literal(true) + .describe('Indicates request was accepted'), + sessionId: z.string().describe('Session ID used for this message'), + }) + .strict(), + }, + }, + }, + 400: { description: 'Validation error' }, + }, + }); + const messageSyncRoute = createRoute({ + method: 'post', + path: '/message-sync', + summary: 'Send Message (sync)', + description: 'Sends a message and waits for the full response', + tags: ['messages'], + request: { + body: { content: { 'application/json': { schema: MessageBodySchema } } }, + }, + responses: { + 200: { + description: 'Synchronous response', + content: { + 'application/json': { + schema: z + .object({ + response: z.string().describe('Agent response text'), + sessionId: z.string().describe('Session ID used for this message'), + tokenUsage: + TokenUsageSchema.optional().describe('Token usage statistics'), + reasoning: z + .string() + .optional() + .describe('Extended thinking content from reasoning models'), + model: z + .string() + .optional() + .describe('Model used for this response'), + provider: z.enum(LLM_PROVIDERS).optional().describe('LLM provider'), + }) + .strict(), + }, + }, + }, + 400: { description: 'Validation error' }, + }, + }); + + const resetRoute = createRoute({ + method: 'post', + path: '/reset', + summary: 'Reset Conversation', + description: 'Resets the conversation history for a given session', + tags: ['messages'], + request: { + body: { content: { 'application/json': { schema: ResetBodySchema } } }, + }, + responses: { + 200: { + description: 'Reset initiated', + content: { + 'application/json': { + schema: z + .object({ + status: z + .string() + .describe('Status message indicating reset was initiated'), + sessionId: z.string().describe('Session ID that was reset'), + }) + .strict(), + }, + }, + }, + }, + }); + + const messageStreamRoute = createRoute({ + method: 'post', + path: '/message-stream', + summary: 'Stream message response', + description: + 'Sends a message and streams the response via Server-Sent Events (SSE). Returns SSE stream directly in response. Events include llm:thinking, llm:chunk, llm:tool-call, llm:tool-result, llm:response, and llm:error. If the session is busy processing another message, returns 202 with queue information.', + tags: ['messages'], + request: { + body: { + content: { 'application/json': { schema: MessageBodySchema } }, + }, + }, + responses: { + 200: { + description: + 'SSE stream of agent events. Standard SSE format with event type and JSON data.', + headers: { + 'Content-Type': { + description: 'SSE content type', + schema: { type: 'string', example: 'text/event-stream' }, + }, + 'Cache-Control': { + description: 'Disable caching for stream', + schema: { type: 'string', example: 'no-cache' }, + }, + Connection: { + description: 'Keep connection alive for streaming', + schema: { type: 'string', example: 'keep-alive' }, + }, + 'X-Accel-Buffering': { + description: 'Disable nginx buffering', + schema: { type: 'string', example: 'no' }, + }, + }, + content: { + 'text/event-stream': { + schema: z + .string() + .describe( + 'Server-Sent Events stream. Events: llm:thinking (start), llm:chunk (text fragments), llm:tool-call (tool execution), llm:tool-result (tool output), llm:response (final), llm:error (errors)' + ), + }, + }, + }, + 202: { + description: + 'Session is busy processing another message. Use the queue endpoints to manage pending messages.', + content: { + 'application/json': { + schema: z + .object({ + busy: z.literal(true).describe('Indicates session is busy'), + sessionId: z.string().describe('The session ID'), + queueLength: z + .number() + .describe('Current number of messages in queue'), + hint: z.string().describe('Instructions for the client'), + }) + .strict(), + }, + }, + }, + 400: { description: 'Validation error' }, + }, + }); + + return app + .openapi(messageRoute, async (ctx) => { + const agent = await getAgent(ctx); + agent.logger.info('Received message via POST /api/message'); + const { content, sessionId } = ctx.req.valid('json'); + + agent.logger.info(`Message for session: ${sessionId}`); + + // Fire and forget - start processing asynchronously + // Results will be delivered via SSE + agent.generate(content as ContentInput, sessionId).catch((error) => { + agent.logger.error( + `Error in async message processing: ${error instanceof Error ? error.message : String(error)}` + ); + }); + + return ctx.json({ accepted: true, sessionId }, 202); + }) + .openapi(messageSyncRoute, async (ctx) => { + const agent = await getAgent(ctx); + agent.logger.info('Received message via POST /api/message-sync'); + const { content, sessionId } = ctx.req.valid('json'); + + agent.logger.info(`Message for session: ${sessionId}`); + + const result = await agent.generate(content as ContentInput, sessionId); + + // Get the session's current LLM config to include model/provider info + const llmConfig = agent.stateManager.getLLMConfig(sessionId); + + return ctx.json({ + response: result.content, + sessionId: result.sessionId, + tokenUsage: result.usage, + reasoning: result.reasoning, + model: llmConfig.model, + provider: llmConfig.provider, + }); + }) + .openapi(resetRoute, async (ctx) => { + const agent = await getAgent(ctx); + agent.logger.info('Received request via POST /api/reset'); + const { sessionId } = ctx.req.valid('json'); + await agent.resetConversation(sessionId); + return ctx.json({ status: 'reset initiated', sessionId }); + }) + .openapi(messageStreamRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { content, sessionId } = ctx.req.valid('json'); + + // Check if session is busy before starting stream + const isBusy = await agent.isSessionBusy(sessionId); + if (isBusy) { + const queuedMessages = await agent.getQueuedMessages(sessionId); + return ctx.json( + { + busy: true as const, + sessionId, + queueLength: queuedMessages.length, + hint: 'Use POST /api/queue/{sessionId} to queue this message, or wait for the current request to complete.', + }, + 202 + ); + } + + // Create abort controller for cleanup + const abortController = new AbortController(); + const { signal } = abortController; + + // Start agent streaming + const iterator = await agent.stream(content as ContentInput, sessionId, { signal }); + + // Use Hono's streamSSE helper which handles backpressure correctly + return streamSSE(ctx, async (stream) => { + // Store pending approval events to be written to stream (only if coordinator available) + const pendingApprovalEvents: Array<{ event: string; data: unknown }> = []; + + // Subscribe to approval events from coordinator (if available) + if (approvalCoordinator) { + approvalCoordinator.onRequest( + (request) => { + if (request.sessionId === sessionId) { + // No transformation needed - SSE uses 'name' discriminant, payload keeps 'type' + pendingApprovalEvents.push({ + event: 'approval:request', + data: request, + }); + } + }, + { signal } + ); + + approvalCoordinator.onResponse( + (response) => { + if (response.sessionId === sessionId) { + pendingApprovalEvents.push({ + event: 'approval:response', + data: response, + }); + } + }, + { signal } + ); + } + + try { + // Stream LLM/tool events from iterator + for await (const event of iterator) { + // First, write any pending approval events + while (pendingApprovalEvents.length > 0) { + const approvalEvent = pendingApprovalEvents.shift()!; + await stream.writeSSE({ + event: approvalEvent.event, + data: JSON.stringify(approvalEvent.data), + }); + } + + // Then write the LLM/tool event + // Serialize errors properly since Error objects don't JSON.stringify well + const eventData = + event.name === 'llm:error' && event.error instanceof Error + ? { + ...event, + error: { + message: event.error.message, + name: event.error.name, + stack: event.error.stack, + }, + } + : event; + await stream.writeSSE({ + event: event.name, + data: JSON.stringify(eventData), + }); + } + + // Write any remaining approval events + while (pendingApprovalEvents.length > 0) { + const approvalEvent = pendingApprovalEvents.shift()!; + await stream.writeSSE({ + event: approvalEvent.event, + data: JSON.stringify(approvalEvent.data), + }); + } + } catch (error) { + await stream.writeSSE({ + event: 'llm:error', + data: JSON.stringify({ + error: { + message: error instanceof Error ? error.message : String(error), + }, + recoverable: false, + sessionId, + }), + }); + } finally { + abortController.abort(); // Cleanup subscriptions + } + }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/models.ts b/dexto/packages/server/src/hono/routes/models.ts new file mode 100644 index 00000000..c2431bc6 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/models.ts @@ -0,0 +1,413 @@ +/** + * Models Routes + * + * API endpoints for listing and managing local/ollama models. + * These endpoints expose model discovery that CLI does directly via function calls. + */ + +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import { promises as fs } from 'fs'; +import { + getLocalModelById, + listOllamaModels, + DEFAULT_OLLAMA_URL, + checkOllamaStatus, + logger, +} from '@dexto/core'; +import { + getAllInstalledModels, + getInstalledModel, + removeInstalledModel, +} from '@dexto/agent-management'; + +// ============================================================================ +// Schemas +// ============================================================================ + +const LocalModelSchema = z + .object({ + id: z.string().describe('Model identifier'), + displayName: z.string().describe('Human-readable model name'), + filePath: z.string().describe('Absolute path to the GGUF file'), + sizeBytes: z.number().describe('File size in bytes'), + contextLength: z.number().optional().describe('Maximum context length in tokens'), + source: z + .enum(['huggingface', 'manual']) + .optional() + .describe('Where the model was downloaded from'), + }) + .describe('An installed local GGUF model'); + +const OllamaModelSchema = z + .object({ + name: z.string().describe('Ollama model name (e.g., llama3.2:latest)'), + size: z.number().optional().describe('Model size in bytes'), + digest: z.string().optional().describe('Model digest/hash'), + modifiedAt: z.string().optional().describe('Last modified timestamp'), + }) + .describe('An Ollama model'); + +const ValidateFileRequestSchema = z + .object({ + filePath: z.string().min(1).describe('Absolute path to the GGUF file to validate'), + }) + .describe('File validation request'); + +const ValidateFileResponseSchema = z + .object({ + valid: z.boolean().describe('Whether the file exists and is readable'), + sizeBytes: z.number().optional().describe('File size in bytes if valid'), + error: z.string().optional().describe('Error message if invalid'), + }) + .describe('File validation response'); + +// ============================================================================ +// Route Definitions +// ============================================================================ + +const listLocalModelsRoute = createRoute({ + method: 'get', + path: '/models/local', + summary: 'List Local Models', + description: + 'Returns all installed local GGUF models from ~/.dexto/models/state.json. ' + + 'These are models downloaded from HuggingFace or manually registered.', + tags: ['models'], + responses: { + 200: { + description: 'List of installed local models', + content: { + 'application/json': { + schema: z.object({ + models: z + .array(LocalModelSchema) + .describe('List of installed local models'), + }), + }, + }, + }, + }, +}); + +const listOllamaModelsRoute = createRoute({ + method: 'get', + path: '/models/ollama', + summary: 'List Ollama Models', + description: + 'Returns available models from the local Ollama server. ' + + 'Returns empty list with available=false if Ollama is not running.', + tags: ['models'], + request: { + query: z.object({ + baseURL: z + .string() + .url() + .optional() + .describe(`Ollama server URL (default: ${DEFAULT_OLLAMA_URL})`), + }), + }, + responses: { + 200: { + description: 'List of Ollama models', + content: { + 'application/json': { + schema: z.object({ + available: z.boolean().describe('Whether Ollama server is running'), + version: z.string().optional().describe('Ollama server version'), + models: z + .array(OllamaModelSchema) + .describe('List of available Ollama models'), + error: z + .string() + .optional() + .describe('Error message if Ollama not available'), + }), + }, + }, + }, + }, +}); + +const validateLocalFileRoute = createRoute({ + method: 'post', + path: '/models/local/validate', + summary: 'Validate GGUF File', + description: + 'Validates that a GGUF file exists and is readable. ' + + 'Used by Web UI to validate custom file paths before saving.', + tags: ['models'], + request: { + body: { + content: { + 'application/json': { + schema: ValidateFileRequestSchema, + }, + }, + }, + }, + responses: { + 200: { + description: 'Validation result', + content: { + 'application/json': { + schema: ValidateFileResponseSchema, + }, + }, + }, + }, +}); + +const DeleteModelRequestSchema = z + .object({ + deleteFile: z + .boolean() + .default(true) + .describe('Whether to also delete the GGUF file from disk'), + }) + .describe('Delete model request options'); + +const DeleteModelResponseSchema = z + .object({ + success: z.boolean().describe('Whether the deletion was successful'), + modelId: z.string().describe('The deleted model ID'), + fileDeleted: z.boolean().describe('Whether the GGUF file was deleted'), + error: z.string().optional().describe('Error message if deletion failed'), + }) + .describe('Delete model response'); + +const deleteLocalModelRoute = createRoute({ + method: 'delete', + path: '/models/local/{modelId}', + summary: 'Delete Installed Model', + description: + 'Removes an installed local model from state.json. ' + + 'Optionally deletes the GGUF file from disk (default: true).', + tags: ['models'], + request: { + params: z.object({ + modelId: z.string().describe('The model ID to delete'), + }), + body: { + content: { + 'application/json': { + schema: DeleteModelRequestSchema, + }, + }, + required: false, + }, + }, + responses: { + 200: { + description: 'Model deleted successfully', + content: { + 'application/json': { + schema: DeleteModelResponseSchema, + }, + }, + }, + 404: { + description: 'Model not found', + content: { + 'application/json': { + schema: DeleteModelResponseSchema, + }, + }, + }, + }, +}); + +// ============================================================================ +// Router +// ============================================================================ + +export function createModelsRouter() { + const app = new OpenAPIHono(); + + return app + .openapi(listLocalModelsRoute, async (ctx) => { + const installedModels = await getAllInstalledModels(); + + const models = installedModels.map((model) => { + // Get display name from registry if available + const registryInfo = getLocalModelById(model.id); + + return { + id: model.id, + displayName: registryInfo?.name || model.id, + filePath: model.filePath, + sizeBytes: model.sizeBytes, + contextLength: registryInfo?.contextLength, + source: model.source, + }; + }); + + return ctx.json({ models }); + }) + .openapi(listOllamaModelsRoute, async (ctx) => { + const { baseURL } = ctx.req.valid('query'); + const ollamaURL = baseURL || DEFAULT_OLLAMA_URL; + + try { + // Check if Ollama is running + const status = await checkOllamaStatus(ollamaURL); + + if (!status.running) { + return ctx.json({ + available: false, + models: [], + error: 'Ollama server is not running', + }); + } + + // List available models + const ollamaModels = await listOllamaModels(ollamaURL); + + return ctx.json({ + available: true, + version: status.version, + models: ollamaModels.map((m) => ({ + name: m.name, + size: m.size, + digest: m.digest, + modifiedAt: m.modifiedAt, + })), + }); + } catch (error) { + return ctx.json({ + available: false, + models: [], + error: + error instanceof Error + ? error.message + : 'Failed to connect to Ollama server', + }); + } + }) + .openapi(validateLocalFileRoute, async (ctx) => { + const { filePath } = ctx.req.valid('json'); + + // Security: Basic path validation + // Prevent path traversal attacks by ensuring absolute path + if (!filePath.startsWith('/')) { + return ctx.json({ + valid: false, + error: 'File path must be absolute (start with /)', + }); + } + + // Validate file extension + if (!filePath.endsWith('.gguf')) { + return ctx.json({ + valid: false, + error: 'File must have .gguf extension', + }); + } + + try { + const stats = await fs.stat(filePath); + + if (!stats.isFile()) { + return ctx.json({ + valid: false, + error: 'Path is not a file', + }); + } + + // Check file is readable + await fs.access(filePath, fs.constants.R_OK); + + return ctx.json({ + valid: true, + sizeBytes: stats.size, + }); + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + + if (nodeError.code === 'ENOENT') { + return ctx.json({ + valid: false, + error: 'File not found', + }); + } + + if (nodeError.code === 'EACCES') { + return ctx.json({ + valid: false, + error: 'File is not readable (permission denied)', + }); + } + + return ctx.json({ + valid: false, + error: error instanceof Error ? error.message : 'Failed to access file', + }); + } + }) + .openapi(deleteLocalModelRoute, async (ctx) => { + const { modelId } = ctx.req.valid('param'); + + // Get body if provided, default to deleteFile: true + let deleteFile = true; + try { + const body = await ctx.req.json(); + if (body && typeof body.deleteFile === 'boolean') { + deleteFile = body.deleteFile; + } + } catch { + // No body or invalid JSON - use default (deleteFile: true) + } + + // Get the model info first (need filePath for deletion) + const model = await getInstalledModel(modelId); + if (!model) { + return ctx.json( + { + success: false, + modelId, + fileDeleted: false, + error: `Model '${modelId}' not found`, + }, + 404 + ); + } + + const filePath = model.filePath; + let fileDeleted = false; + + // Delete the GGUF file if requested + if (deleteFile && filePath) { + try { + await fs.unlink(filePath); + fileDeleted = true; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + // File already deleted or doesn't exist - that's fine + if (nodeError.code === 'ENOENT') { + fileDeleted = true; // Consider it deleted + } else { + // Permission error or other issue - report but continue + logger.warn( + `Failed to delete GGUF file ${filePath}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + } + + // Remove from state.json + const removed = await removeInstalledModel(modelId); + if (!removed) { + return ctx.json({ + success: false, + modelId, + fileDeleted, + error: 'Failed to remove model from state', + }); + } + + return ctx.json({ + success: true, + modelId, + fileDeleted, + }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/openrouter.ts b/dexto/packages/server/src/hono/routes/openrouter.ts new file mode 100644 index 00000000..6515368f --- /dev/null +++ b/dexto/packages/server/src/hono/routes/openrouter.ts @@ -0,0 +1,172 @@ +/** + * OpenRouter Validation Routes + * + * Standalone routes for validating OpenRouter model IDs against the registry. + * Decoupled from agent runtime - can be used independently. + */ + +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import { + logger, + lookupOpenRouterModel, + refreshOpenRouterModelCache, + getOpenRouterModelInfo, +} from '@dexto/core'; + +const ValidateModelParamsSchema = z + .object({ + modelId: z + .string() + .min(1) + .describe('OpenRouter model ID to validate (e.g., anthropic/claude-3.5-sonnet)'), + }) + .describe('Path parameters for model validation'); + +const ValidateModelResponseSchema = z + .object({ + valid: z.boolean().describe('Whether the model ID is valid'), + modelId: z.string().describe('The model ID that was validated'), + status: z + .enum(['valid', 'invalid', 'unknown']) + .describe('Validation status: valid, invalid, or unknown (cache empty)'), + error: z.string().optional().describe('Error message if invalid'), + info: z + .object({ + contextLength: z.number().describe('Model context length in tokens'), + }) + .optional() + .describe('Model information if valid'), + }) + .describe('Model validation response'); + +/** + * Create OpenRouter validation router. + * No agent dependency - purely utility routes. + */ +export function createOpenRouterRouter() { + const app = new OpenAPIHono(); + + const validateRoute = createRoute({ + method: 'get', + path: '/openrouter/validate/{modelId}', + summary: 'Validate OpenRouter Model', + description: + 'Validates an OpenRouter model ID against the cached model registry. Refreshes cache if stale.', + tags: ['openrouter'], + request: { + params: ValidateModelParamsSchema, + }, + responses: { + 200: { + description: 'Validation result', + content: { + 'application/json': { + schema: ValidateModelResponseSchema, + }, + }, + }, + }, + }); + + const refreshRoute = createRoute({ + method: 'post', + path: '/openrouter/refresh-cache', + summary: 'Refresh OpenRouter Model Cache', + description: 'Forces a refresh of the OpenRouter model registry cache from the API.', + tags: ['openrouter'], + responses: { + 200: { + description: 'Cache refreshed successfully', + content: { + 'application/json': { + schema: z.object({ + ok: z.literal(true).describe('Success indicator'), + message: z.string().describe('Status message'), + }), + }, + }, + }, + 500: { + description: 'Cache refresh failed', + content: { + 'application/json': { + schema: z.object({ + ok: z.literal(false).describe('Failure indicator'), + message: z.string().describe('Error message'), + }), + }, + }, + }, + }, + }); + + return app + .openapi(validateRoute, async (ctx) => { + const { modelId: encodedModelId } = ctx.req.valid('param'); + // Decode URL-encoded model ID to handle slashes (e.g., anthropic/claude-3.5-sonnet) + const modelId = decodeURIComponent(encodedModelId); + + // First lookup against current cache + let status = lookupOpenRouterModel(modelId); + + // If unknown (cache empty/stale), try refreshing + if (status === 'unknown') { + try { + await refreshOpenRouterModelCache(); + status = lookupOpenRouterModel(modelId); + } catch (error) { + // Network failed - return unknown status + logger.warn( + `OpenRouter cache refresh failed during validation: ${error instanceof Error ? error.message : String(error)}` + ); + return ctx.json({ + valid: false, + modelId, + status: 'unknown' as const, + error: 'Could not validate model - cache refresh failed', + }); + } + } + + if (status === 'invalid') { + return ctx.json({ + valid: false, + modelId, + status: 'invalid' as const, + error: `Model '${modelId}' not found in OpenRouter. Check the model ID at https://openrouter.ai/models`, + }); + } + + // Valid - include model info + const info = getOpenRouterModelInfo(modelId); + return ctx.json({ + valid: true, + modelId, + status: 'valid' as const, + ...(info && { info: { contextLength: info.contextLength } }), + }); + }) + .openapi(refreshRoute, async (ctx) => { + try { + await refreshOpenRouterModelCache(); + return ctx.json( + { + ok: true as const, + message: 'OpenRouter model cache refreshed successfully', + }, + 200 + ); + } catch (error) { + logger.error( + `Failed to refresh OpenRouter cache: ${error instanceof Error ? error.message : String(error)}` + ); + return ctx.json( + { + ok: false as const, + message: 'Failed to refresh OpenRouter model cache', + }, + 500 + ); + } + }); +} diff --git a/dexto/packages/server/src/hono/routes/prompts.ts b/dexto/packages/server/src/hono/routes/prompts.ts new file mode 100644 index 00000000..15318eb6 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/prompts.ts @@ -0,0 +1,294 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { DextoAgent } from '@dexto/core'; +import { PromptError } from '@dexto/core'; +import { PromptInfoSchema, PromptDefinitionSchema } from '../schemas/responses.js'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +const CustomPromptRequestSchema = z + .object({ + name: z + .string() + .min(1, 'Prompt name is required') + .describe('Unique name for the custom prompt'), + title: z.string().optional().describe('Display title for the prompt'), + description: z.string().optional().describe('Description of what the prompt does'), + content: z + .string() + .min(1, 'Prompt content is required') + .describe('The prompt content text (can include {{argumentName}} placeholders)'), + arguments: z + .array( + z + .object({ + name: z + .string() + .min(1, 'Argument name is required') + .describe('Argument name'), + description: z.string().optional().describe('Argument description'), + required: z + .boolean() + .optional() + .describe('Whether the argument is required'), + }) + .strict() + ) + .optional() + .describe('Array of argument definitions'), + resource: z + .object({ + data: z + .string() + .min(1, 'Resource data is required') + .describe('Base64-encoded resource data'), + mimeType: z + .string() + .min(1, 'Resource MIME type is required') + .describe('MIME type of the resource (e.g., text/plain, application/pdf)'), + filename: z.string().optional().describe('Resource filename'), + }) + .strict() + .optional() + .describe('Attach a resource to this prompt'), + }) + .strict() + .describe('Request body for creating a custom prompt with optional resource attachment'); + +const PromptNameParamSchema = z + .object({ + name: z.string().min(1, 'Prompt name is required').describe('The prompt name'), + }) + .describe('Path parameters for prompt endpoints'); + +const ResolvePromptQuerySchema = z + .object({ + context: z.string().optional().describe('Additional context for prompt resolution'), + args: z + .string() + .optional() + .describe('Arguments to substitute in the prompt template (pass as a JSON string)'), + }) + .describe('Query parameters for resolving prompt templates'); + +export function createPromptsRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const listRoute = createRoute({ + method: 'get', + path: '/prompts', + summary: 'List Prompts', + description: 'Retrieves all available prompts, including both built-in and custom prompts', + tags: ['prompts'], + responses: { + 200: { + description: 'List all prompts', + content: { + 'application/json': { + schema: z + .object({ + prompts: z + .array(PromptInfoSchema) + .describe('Array of available prompts'), + }) + .strict() + .describe('Prompts list response'), + }, + }, + }, + }, + }); + + const createCustomRoute = createRoute({ + method: 'post', + path: '/prompts/custom', + summary: 'Create Custom Prompt', + description: + 'Creates a new custom prompt with optional resource attachment. Maximum request size: 10MB', + tags: ['prompts'], + request: { + body: { + content: { + 'application/json': { + schema: CustomPromptRequestSchema, + }, + }, + }, + }, + responses: { + 201: { + description: 'Custom prompt created', + content: { + 'application/json': { + schema: z + .object({ + prompt: PromptInfoSchema.describe('Created prompt information'), + }) + .strict() + .describe('Create prompt response'), + }, + }, + }, + }, + }); + + const deleteCustomRoute = createRoute({ + method: 'delete', + path: '/prompts/custom/{name}', + summary: 'Delete Custom Prompt', + description: 'Permanently deletes a custom prompt. Built-in prompts cannot be deleted', + tags: ['prompts'], + request: { + params: z.object({ + name: z.string().min(1, 'Prompt name is required').describe('The prompt name'), + }), + }, + responses: { + 204: { description: 'Prompt deleted' }, + }, + }); + + const getPromptRoute = createRoute({ + method: 'get', + path: '/prompts/{name}', + summary: 'Get Prompt Definition', + description: 'Fetches the definition for a specific prompt', + tags: ['prompts'], + request: { + params: PromptNameParamSchema, + }, + responses: { + 200: { + description: 'Prompt definition', + content: { + 'application/json': { + schema: z + .object({ + definition: PromptDefinitionSchema.describe('Prompt definition'), + }) + .strict() + .describe('Get prompt definition response'), + }, + }, + }, + 404: { description: 'Prompt not found' }, + }, + }); + + const resolvePromptRoute = createRoute({ + method: 'get', + path: '/prompts/{name}/resolve', + summary: 'Resolve Prompt', + description: + 'Resolves a prompt template with provided arguments and returns the final text with resources', + tags: ['prompts'], + request: { + params: PromptNameParamSchema, + query: ResolvePromptQuerySchema, + }, + responses: { + 200: { + description: 'Resolved prompt content', + content: { + 'application/json': { + schema: z + .object({ + text: z.string().describe('Resolved prompt text'), + resources: z + .array(z.string()) + .describe('Array of resource identifiers'), + }) + .strict() + .describe('Resolve prompt response'), + }, + }, + }, + 404: { description: 'Prompt not found' }, + }, + }); + + return app + .openapi(listRoute, async (ctx) => { + const agent = await getAgent(ctx); + const prompts = await agent.listPrompts(); + const list = Object.values(prompts); + return ctx.json({ prompts: list }); + }) + .openapi(createCustomRoute, async (ctx) => { + const agent = await getAgent(ctx); + const payload = ctx.req.valid('json'); + const promptArguments = payload.arguments + ?.map((arg) => ({ + name: arg.name, + ...(arg.description ? { description: arg.description } : {}), + ...(typeof arg.required === 'boolean' ? { required: arg.required } : {}), + })) + .filter(Boolean); + + const createPayload = { + name: payload.name, + content: payload.content, + ...(payload.title ? { title: payload.title } : {}), + ...(payload.description ? { description: payload.description } : {}), + ...(promptArguments && promptArguments.length > 0 + ? { arguments: promptArguments } + : {}), + ...(payload.resource + ? { + resource: { + data: payload.resource.data, + mimeType: payload.resource.mimeType, + ...(payload.resource.filename + ? { filename: payload.resource.filename } + : {}), + }, + } + : {}), + }; + const prompt = await agent.createCustomPrompt(createPayload); + return ctx.json({ prompt }, 201); + }) + .openapi(deleteCustomRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { name } = ctx.req.valid('param'); + // Hono automatically decodes path parameters, no manual decode needed + await agent.deleteCustomPrompt(name); + return ctx.body(null, 204); + }) + .openapi(getPromptRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { name } = ctx.req.valid('param'); + const definition = await agent.getPromptDefinition(name); + if (!definition) throw PromptError.notFound(name); + return ctx.json({ definition }); + }) + .openapi(resolvePromptRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { name } = ctx.req.valid('param'); + const { context, args: argsString } = ctx.req.valid('query'); + + // Optional structured args in `args` query param as JSON + let parsedArgs: Record | undefined; + if (argsString) { + try { + const parsed = JSON.parse(argsString); + if (parsed && typeof parsed === 'object') { + parsedArgs = parsed as Record; + } + } catch { + // Ignore malformed args JSON; continue with whatever we have + } + } + + // Build options object with only defined values + const options: { + context?: string; + args?: Record; + } = {}; + if (context !== undefined) options.context = context; + if (parsedArgs !== undefined) options.args = parsedArgs; + + // Use DextoAgent's resolvePrompt method + const result = await agent.resolvePrompt(name, options); + return ctx.json({ text: result.text, resources: result.resources }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/queue.ts b/dexto/packages/server/src/hono/routes/queue.ts new file mode 100644 index 00000000..ee366f8f --- /dev/null +++ b/dexto/packages/server/src/hono/routes/queue.ts @@ -0,0 +1,238 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { DextoAgent, ContentPart } from '@dexto/core'; +import { ContentPartSchema } from '../schemas/responses.js'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +// Schema for queued message in responses +const QueuedMessageSchema = z + .object({ + id: z.string().describe('Unique identifier for the queued message'), + content: z.array(ContentPartSchema).describe('Message content parts'), + queuedAt: z.number().describe('Unix timestamp when message was queued'), + metadata: z.record(z.unknown()).optional().describe('Optional metadata'), + }) + .strict() + .describe('A message waiting in the queue'); + +// ContentPart schemas matching @dexto/core types +// TODO: Same as messages.ts - Zod-inferred types don't exactly match core's ContentInput +// due to exactOptionalPropertyTypes. We cast to ContentPart after validation. +const TextPartSchema = z + .object({ + type: z.literal('text').describe('Content type identifier'), + text: z.string().describe('Text content'), + }) + .describe('Text content part'); + +const ImagePartSchema = z + .object({ + type: z.literal('image').describe('Content type identifier'), + image: z.string().describe('Base64-encoded image data or URL'), + mimeType: z.string().optional().describe('MIME type (e.g., image/png)'), + }) + .describe('Image content part'); + +const FilePartSchema = z + .object({ + type: z.literal('file').describe('Content type identifier'), + data: z.string().describe('Base64-encoded file data or URL'), + mimeType: z.string().describe('MIME type (e.g., application/pdf)'), + filename: z.string().optional().describe('Optional filename'), + }) + .describe('File content part'); + +const QueueContentPartSchema = z + .discriminatedUnion('type', [TextPartSchema, ImagePartSchema, FilePartSchema]) + .describe('Content part - text, image, or file'); + +// Schema for queue message request body - matches messages.ts MessageBodySchema +const QueueMessageBodySchema = z + .object({ + content: z + .union([z.string(), z.array(QueueContentPartSchema)]) + .describe('Message content - string for text, or ContentPart[] for multimodal'), + }) + .describe('Request body for queueing a message'); + +export function createQueueRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + // GET /queue/:sessionId - Get all queued messages + const getQueueRoute = createRoute({ + method: 'get', + path: '/queue/{sessionId}', + summary: 'Get queued messages', + description: 'Returns all messages waiting in the queue for a session', + tags: ['queue'], + request: { + params: z.object({ + sessionId: z.string().min(1).describe('Session ID'), + }), + }, + responses: { + 200: { + description: 'List of queued messages', + content: { + 'application/json': { + schema: z + .object({ + messages: z.array(QueuedMessageSchema).describe('Queued messages'), + count: z.number().describe('Number of messages in queue'), + }) + .strict(), + }, + }, + }, + 404: { description: 'Session not found' }, + }, + }); + + // POST /queue/:sessionId - Queue a new message + const queueMessageRoute = createRoute({ + method: 'post', + path: '/queue/{sessionId}', + summary: 'Queue a message', + description: + 'Adds a message to the queue for processing when the session is no longer busy', + tags: ['queue'], + request: { + params: z.object({ + sessionId: z.string().min(1).describe('Session ID'), + }), + body: { + content: { 'application/json': { schema: QueueMessageBodySchema } }, + }, + }, + responses: { + 201: { + description: 'Message queued successfully', + content: { + 'application/json': { + schema: z + .object({ + queued: z.literal(true).describe('Indicates message was queued'), + id: z.string().describe('ID of the queued message'), + position: z.number().describe('Position in the queue (1-based)'), + }) + .strict(), + }, + }, + }, + 404: { description: 'Session not found' }, + }, + }); + + // DELETE /queue/:sessionId/:messageId - Remove a specific queued message + const removeQueuedMessageRoute = createRoute({ + method: 'delete', + path: '/queue/{sessionId}/{messageId}', + summary: 'Remove queued message', + description: 'Removes a specific message from the queue', + tags: ['queue'], + request: { + params: z.object({ + sessionId: z.string().min(1).describe('Session ID'), + messageId: z.string().min(1).describe('ID of the queued message to remove'), + }), + }, + responses: { + 200: { + description: 'Message removed successfully', + content: { + 'application/json': { + schema: z + .object({ + removed: z.literal(true).describe('Indicates message was removed'), + id: z.string().describe('ID of the removed message'), + }) + .strict(), + }, + }, + }, + 404: { description: 'Session or message not found' }, + }, + }); + + // DELETE /queue/:sessionId - Clear all queued messages + const clearQueueRoute = createRoute({ + method: 'delete', + path: '/queue/{sessionId}', + summary: 'Clear message queue', + description: 'Removes all messages from the queue for a session', + tags: ['queue'], + request: { + params: z.object({ + sessionId: z.string().min(1).describe('Session ID'), + }), + }, + responses: { + 200: { + description: 'Queue cleared successfully', + content: { + 'application/json': { + schema: z + .object({ + cleared: z.literal(true).describe('Indicates queue was cleared'), + count: z.number().describe('Number of messages that were removed'), + }) + .strict(), + }, + }, + }, + 404: { description: 'Session not found' }, + }, + }); + + return app + .openapi(getQueueRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('param'); + + const messages = await agent.getQueuedMessages(sessionId); + return ctx.json({ + messages, + count: messages.length, + }); + }) + .openapi(queueMessageRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('param'); + const { content: rawContent } = ctx.req.valid('json'); + + // Normalize content to array format and cast to ContentPart[] + // (same exactOptionalPropertyTypes issue as messages.ts - see TODO there) + const content = ( + typeof rawContent === 'string' + ? [{ type: 'text' as const, text: rawContent }] + : rawContent + ) as ContentPart[]; + + const result = await agent.queueMessage(sessionId, { content }); + return ctx.json( + { + queued: result.queued, + id: result.id, + position: result.position, + }, + 201 + ); + }) + .openapi(removeQueuedMessageRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId, messageId } = ctx.req.valid('param'); + + const removed = await agent.removeQueuedMessage(sessionId, messageId); + if (!removed) { + return ctx.json({ error: 'Message not found in queue' }, 404); + } + return ctx.json({ removed: true, id: messageId }); + }) + .openapi(clearQueueRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('param'); + + const count = await agent.clearMessageQueue(sessionId); + return ctx.json({ cleared: true, count }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/resources.ts b/dexto/packages/server/src/hono/routes/resources.ts new file mode 100644 index 00000000..6b79e6ed --- /dev/null +++ b/dexto/packages/server/src/hono/routes/resources.ts @@ -0,0 +1,128 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import { ResourceSchema } from '../schemas/responses.js'; +import type { GetAgentFn } from '../index.js'; + +const ResourceIdParamSchema = z + .object({ + resourceId: z + .string() + .min(1, 'Resource ID is required') + .transform((encoded) => decodeURIComponent(encoded)) + .describe('The URI-encoded resource identifier'), + }) + .describe('Path parameters for resource endpoints'); + +// Response schemas for resources endpoints + +const ListResourcesResponseSchema = z + .object({ + ok: z.literal(true).describe('Indicates successful response'), + resources: z + .array(ResourceSchema) + .describe('Array of all available resources from all sources'), + }) + .strict() + .describe('List of all resources'); + +const ResourceContentItemSchema = z + .object({ + uri: z.string().describe('Resource URI'), + mimeType: z.string().optional().describe('MIME type of the content'), + text: z.string().optional().describe('Text content (for text resources)'), + blob: z + .string() + .optional() + .describe('Base64-encoded binary content (for binary resources)'), + }) + .strict() + .describe('Resource content item'); + +const ReadResourceResponseSchema = z + .object({ + ok: z.literal(true).describe('Indicates successful response'), + content: z + .object({ + contents: z + .array(ResourceContentItemSchema) + .describe('Array of content items (typically one item)'), + _meta: z + .record(z.any()) + .optional() + .describe('Optional metadata about the resource'), + }) + .strict() + .describe('Resource content from MCP ReadResourceResult'), + }) + .strict() + .describe('Resource content response'); + +export function createResourcesRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const listRoute = createRoute({ + method: 'get', + path: '/resources', + summary: 'List All Resources', + description: + 'Retrieves a list of all available resources from all sources (MCP servers and internal providers)', + tags: ['resources'], + responses: { + 200: { + description: 'List all resources', + content: { 'application/json': { schema: ListResourcesResponseSchema } }, + }, + }, + }); + + const getContentRoute = createRoute({ + method: 'get', + path: '/resources/{resourceId}/content', + summary: 'Read Resource Content', + description: + 'Reads the content of a specific resource by its URI. The resource ID in the URL must be URI-encoded', + tags: ['resources'], + request: { + params: ResourceIdParamSchema, + }, + responses: { + 200: { + description: 'Resource content', + content: { 'application/json': { schema: ReadResourceResponseSchema } }, + }, + }, + }); + + const headRoute = createRoute({ + method: 'head', + path: '/resources/{resourceId}', + summary: 'Check Resource Exists', + description: 'Checks if a resource exists by its URI without retrieving its content', + tags: ['resources'], + request: { + params: ResourceIdParamSchema, + }, + responses: { + 200: { description: 'Resource exists' }, + 404: { description: 'Resource not found' }, + }, + }); + + return app + .openapi(listRoute, async (ctx) => { + const agent = await getAgent(ctx); + const resources = await agent.listResources(); + return ctx.json({ ok: true, resources: Object.values(resources) }); + }) + .openapi(getContentRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { resourceId } = ctx.req.valid('param'); + const content = await agent.readResource(resourceId); + return ctx.json({ ok: true, content }); + }) + .openapi(headRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { resourceId } = ctx.req.valid('param'); + const exists = await agent.hasResource(resourceId); + return ctx.body(null, exists ? 200 : 404); + }); +} diff --git a/dexto/packages/server/src/hono/routes/search.ts b/dexto/packages/server/src/hono/routes/search.ts new file mode 100644 index 00000000..52bd4ae8 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/search.ts @@ -0,0 +1,87 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { DextoAgent } from '@dexto/core'; +import { MessageSearchResponseSchema, SessionSearchResponseSchema } from '../schemas/responses.js'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +const MessageSearchQuery = z.object({ + q: z.string().min(1, 'Search query is required').describe('Search query string'), + limit: z.coerce + .number() + .min(1) + .max(100) + .optional() + .describe('Maximum number of results to return (default: 20)'), + offset: z.coerce + .number() + .min(0) + .optional() + .describe('Number of results to skip for pagination (default: 0)'), + sessionId: z.string().optional().describe('Limit search to a specific session'), + role: z + .enum(['user', 'assistant', 'system', 'tool']) + .optional() + .describe('Filter by message role'), +}); + +const SessionSearchQuery = z.object({ + q: z.string().min(1, 'Search query is required').describe('Search query string'), +}); + +export function createSearchRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const messagesRoute = createRoute({ + method: 'get', + path: '/search/messages', + summary: 'Search Messages', + description: 'Searches for messages across all sessions or within a specific session', + tags: ['search'], + request: { query: MessageSearchQuery }, + responses: { + 200: { + description: 'Message search results', + content: { 'application/json': { schema: MessageSearchResponseSchema } }, + }, + }, + }); + + const sessionsRoute = createRoute({ + method: 'get', + path: '/search/sessions', + summary: 'Search Sessions', + description: 'Searches for sessions that contain the specified query', + tags: ['search'], + request: { query: SessionSearchQuery }, + responses: { + 200: { + description: 'Session search results', + content: { 'application/json': { schema: SessionSearchResponseSchema } }, + }, + }, + }); + + return app + .openapi(messagesRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { q, limit, offset, sessionId, role } = ctx.req.valid('query'); + const options = { + limit: limit || 20, + offset: offset || 0, + ...(sessionId && { sessionId }), + ...(role && { role }), + }; + + const searchResults = await agent.searchMessages(q, options); + // TODO: Improve type alignment between core and server schemas. + // Core's InternalMessage has union types for binary data, but JSON responses are strings. + return ctx.json(searchResults as z.output); + }) + .openapi(sessionsRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { q } = ctx.req.valid('query'); + const searchResults = await agent.searchSessions(q); + // TODO: Improve type alignment between core and server schemas. + return ctx.json(searchResults as z.output); + }); +} diff --git a/dexto/packages/server/src/hono/routes/sessions.ts b/dexto/packages/server/src/hono/routes/sessions.ts new file mode 100644 index 00000000..427bbe35 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/sessions.ts @@ -0,0 +1,492 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import { SessionMetadataSchema, InternalMessageSchema } from '../schemas/responses.js'; +import type { GetAgentFn } from '../index.js'; + +const CreateSessionSchema = z + .object({ + sessionId: z.string().optional().describe('A custom ID for the new session'), + }) + .describe('Request body for creating a new session'); + +export function createSessionsRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const listRoute = createRoute({ + method: 'get', + path: '/sessions', + summary: 'List Sessions', + description: 'Retrieves a list of all active sessions', + tags: ['sessions'], + responses: { + 200: { + description: 'List of all active sessions', + content: { + 'application/json': { + schema: z + .object({ + sessions: z + .array(SessionMetadataSchema) + .describe('Array of session metadata objects'), + }) + .strict(), + }, + }, + }, + }, + }); + + const createRouteDef = createRoute({ + method: 'post', + path: '/sessions', + summary: 'Create Session', + description: 'Creates a new session', + tags: ['sessions'], + request: { body: { content: { 'application/json': { schema: CreateSessionSchema } } } }, + responses: { + 201: { + description: 'Session created successfully', + content: { + 'application/json': { + schema: z + .object({ + session: SessionMetadataSchema.describe( + 'Newly created session metadata' + ), + }) + .strict(), + }, + }, + }, + }, + }); + + const getRoute = createRoute({ + method: 'get', + path: '/sessions/{sessionId}', + summary: 'Get Session Details', + description: 'Fetches details for a specific session', + tags: ['sessions'], + request: { params: z.object({ sessionId: z.string().describe('Session identifier') }) }, + responses: { + 200: { + description: 'Session details with metadata', + content: { + 'application/json': { + schema: z + .object({ + session: SessionMetadataSchema.extend({ + history: z + .number() + .int() + .nonnegative() + .describe('Number of messages in history'), + }) + .strict() + .describe('Session metadata with history count'), + }) + .strict(), + }, + }, + }, + }, + }); + + const historyRoute = createRoute({ + method: 'get', + path: '/sessions/{sessionId}/history', + summary: 'Get Session History', + description: + 'Retrieves the conversation history for a session along with processing status', + tags: ['sessions'], + request: { params: z.object({ sessionId: z.string().describe('Session identifier') }) }, + responses: { + 200: { + description: 'Session conversation history', + content: { + 'application/json': { + schema: z + .object({ + history: z + .array(InternalMessageSchema) + .describe('Array of messages in conversation history'), + isBusy: z + .boolean() + .describe( + 'Whether the session is currently processing a message' + ), + }) + .strict(), + }, + }, + }, + }, + }); + + const deleteRoute = createRoute({ + method: 'delete', + path: '/sessions/{sessionId}', + summary: 'Delete Session', + description: + 'Permanently deletes a session and all its conversation history. This action cannot be undone', + tags: ['sessions'], + request: { params: z.object({ sessionId: z.string().describe('Session identifier') }) }, + responses: { + 200: { + description: 'Session deleted successfully', + content: { + 'application/json': { + schema: z + .object({ + status: z.literal('deleted').describe('Deletion status'), + sessionId: z.string().describe('ID of the deleted session'), + }) + .strict(), + }, + }, + }, + }, + }); + + const cancelRoute = createRoute({ + method: 'post', + path: '/sessions/{sessionId}/cancel', + summary: 'Cancel Session Run', + description: + 'Cancels an in-flight agent run for the specified session. ' + + 'By default (soft cancel), only the current LLM call is cancelled and queued messages continue processing. ' + + 'Set clearQueue=true for hard cancel to also clear any queued messages.', + tags: ['sessions'], + request: { + params: z.object({ sessionId: z.string().describe('Session identifier') }), + body: { + content: { + 'application/json': { + schema: z + .object({ + clearQueue: z + .boolean() + .optional() + .default(false) + .describe( + 'If true (hard cancel), clears queued messages. If false (soft cancel, default), queued messages continue processing.' + ), + }) + .strict(), + }, + }, + required: false, + }, + }, + responses: { + 200: { + description: 'Cancel operation result', + content: { + 'application/json': { + schema: z + .object({ + cancelled: z.boolean().describe('Whether a run was cancelled'), + sessionId: z.string().describe('Session ID'), + queueCleared: z + .boolean() + .describe('Whether queued messages were cleared'), + clearedCount: z + .number() + .describe( + 'Number of queued messages cleared (0 if soft cancel)' + ), + }) + .strict(), + }, + }, + }, + }, + }); + + const loadRoute = createRoute({ + method: 'get', + path: '/sessions/{sessionId}/load', + summary: 'Load Session', + description: + 'Validates and retrieves session information including processing status. The client should track the active session.', + tags: ['sessions'], + request: { + params: z.object({ sessionId: z.string().describe('Session identifier') }), + }, + responses: { + 200: { + description: 'Session information retrieved successfully', + content: { + 'application/json': { + schema: z + .object({ + session: SessionMetadataSchema.extend({ + isBusy: z + .boolean() + .describe( + 'Whether the session is currently processing a message' + ), + }).describe('Session metadata with processing status'), + }) + .strict(), + }, + }, + }, + 404: { + description: 'Session not found', + content: { + 'application/json': { + schema: z + .object({ + error: z.string().describe('Error message'), + }) + .strict(), + }, + }, + }, + }, + }); + + const patchRoute = createRoute({ + method: 'patch', + path: '/sessions/{sessionId}', + summary: 'Update Session Title', + description: 'Updates the title of an existing session', + tags: ['sessions'], + request: { + params: z.object({ sessionId: z.string().describe('Session identifier') }), + body: { + content: { + 'application/json': { + schema: z.object({ + title: z + .string() + .min(1, 'Title is required') + .max(120, 'Title too long') + .describe('New title for the session (maximum 120 characters)'), + }), + }, + }, + }, + }, + responses: { + 200: { + description: 'Session updated successfully', + content: { + 'application/json': { + schema: z + .object({ + session: SessionMetadataSchema.describe('Updated session metadata'), + }) + .strict(), + }, + }, + }, + }, + }); + + const generateTitleRoute = createRoute({ + method: 'post', + path: '/sessions/{sessionId}/generate-title', + summary: 'Generate Session Title', + description: + 'Generates a descriptive title for the session using the first user message. Returns existing title if already set.', + tags: ['sessions'], + request: { + params: z.object({ sessionId: z.string().describe('Session identifier') }), + }, + responses: { + 200: { + description: 'Title generated successfully', + content: { + 'application/json': { + schema: z + .object({ + title: z + .string() + .nullable() + .describe('Generated title, or null if generation failed'), + sessionId: z.string().describe('Session ID'), + }) + .strict(), + }, + }, + }, + 404: { + description: 'Session not found (error format handled by middleware)', + }, + }, + }); + + return app + .openapi(listRoute, async (ctx) => { + const agent = await getAgent(ctx); + const sessionIds = await agent.listSessions(); + const sessions = await Promise.all( + sessionIds.map(async (id) => { + try { + const metadata = await agent.getSessionMetadata(id); + return { + id, + createdAt: metadata?.createdAt || null, + lastActivity: metadata?.lastActivity || null, + messageCount: metadata?.messageCount || 0, + title: metadata?.title || null, + }; + } catch { + // Skip sessions that no longer exist + return { + id, + createdAt: null, + lastActivity: null, + messageCount: 0, + title: null, + }; + } + }) + ); + return ctx.json({ sessions }); + }) + .openapi(createRouteDef, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('json'); + const session = await agent.createSession(sessionId); + const metadata = await agent.getSessionMetadata(session.id); + return ctx.json( + { + session: { + id: session.id, + createdAt: metadata?.createdAt || Date.now(), + lastActivity: metadata?.lastActivity || Date.now(), + messageCount: metadata?.messageCount || 0, + title: metadata?.title || null, + }, + }, + 201 + ); + }) + .openapi(getRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.param(); + const metadata = await agent.getSessionMetadata(sessionId); + const history = await agent.getSessionHistory(sessionId); + return ctx.json({ + session: { + id: sessionId, + createdAt: metadata?.createdAt || null, + lastActivity: metadata?.lastActivity || null, + messageCount: metadata?.messageCount || 0, + title: metadata?.title || null, + history: history.length, + }, + }); + }) + .openapi(historyRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.param(); + const [history, isBusy] = await Promise.all([ + agent.getSessionHistory(sessionId), + agent.isSessionBusy(sessionId), + ]); + // TODO: Improve type alignment between core and server schemas. + // Core's InternalMessage has union types (string | Uint8Array | Buffer | URL) + // for binary data, but JSON responses are always base64 strings. + return ctx.json({ + history: history as z.output[], + isBusy, + }); + }) + .openapi(deleteRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.param(); + await agent.deleteSession(sessionId); + return ctx.json({ status: 'deleted', sessionId }); + }) + .openapi(cancelRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('param'); + + // Get clearQueue from body, default to false (soft cancel) + let clearQueue = false; + try { + const body = ctx.req.valid('json'); + clearQueue = body?.clearQueue ?? false; + } catch { + // No body or invalid body - use default (soft cancel) + } + + // If hard cancel, clear the queue first + let clearedCount = 0; + if (clearQueue) { + try { + clearedCount = await agent.clearMessageQueue(sessionId); + agent.logger.debug( + `Hard cancel: cleared ${clearedCount} queued message(s) for session: ${sessionId}` + ); + } catch { + // Session might not exist or queue not accessible - continue with cancel + } + } + + // Then cancel the current run + const cancelled = await agent.cancel(sessionId); + if (!cancelled) { + agent.logger.debug(`No in-flight run to cancel for session: ${sessionId}`); + } + + return ctx.json({ + cancelled, + sessionId, + queueCleared: clearQueue, + clearedCount, + }); + }) + .openapi(loadRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('param'); + + // Validate that session exists + const sessionIds = await agent.listSessions(); + if (!sessionIds.includes(sessionId)) { + return ctx.json({ error: `Session not found: ${sessionId}` }, 404); + } + + // Return session metadata with processing status + const metadata = await agent.getSessionMetadata(sessionId); + const isBusy = await agent.isSessionBusy(sessionId); + return ctx.json( + { + session: { + id: sessionId, + createdAt: metadata?.createdAt || null, + lastActivity: metadata?.lastActivity || null, + messageCount: metadata?.messageCount || 0, + title: metadata?.title || null, + isBusy, + }, + }, + 200 + ); + }) + .openapi(patchRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('param'); + const { title } = ctx.req.valid('json'); + await agent.setSessionTitle(sessionId, title); + const metadata = await agent.getSessionMetadata(sessionId); + return ctx.json({ + session: { + id: sessionId, + createdAt: metadata?.createdAt || null, + lastActivity: metadata?.lastActivity || null, + messageCount: metadata?.messageCount || 0, + title: metadata?.title || title, + }, + }); + }) + .openapi(generateTitleRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { sessionId } = ctx.req.valid('param'); + const title = await agent.generateSessionTitle(sessionId); + return ctx.json({ title, sessionId }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/static.ts b/dexto/packages/server/src/hono/routes/static.ts new file mode 100644 index 00000000..a995051a --- /dev/null +++ b/dexto/packages/server/src/hono/routes/static.ts @@ -0,0 +1,118 @@ +import { Hono } from 'hono'; +import type { NotFoundHandler } from 'hono'; +import { serveStatic } from '@hono/node-server/serve-static'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +/** + * Runtime configuration injected into WebUI via window globals. + * This replaces the Next.js SSR injection that was lost in the Vite migration. + * + * TODO: This injection only works in production mode where Hono serves index.html. + * In dev mode (`pnpm dev`), Vite serves index.html directly, bypassing this injection. + * To support dev mode analytics, add a `/api/config/analytics` endpoint that the + * WebUI can fetch as a fallback when `window.__DEXTO_ANALYTICS__` is undefined. + */ +export interface WebUIRuntimeConfig { + analytics?: { + distinctId: string; + posthogKey: string; + posthogHost: string; + appVersion: string; + } | null; +} + +/** + * Create a static file router for serving WebUI assets. + * + * Serves static files from the specified webRoot directory. + * Note: SPA fallback is handled separately via createSpaFallbackHandler. + * + * @param webRoot - Absolute path to the directory containing WebUI build output + */ +export function createStaticRouter(webRoot: string) { + const app = new Hono(); + + // Serve static assets from /assets/ + app.use('/assets/*', serveStatic({ root: webRoot })); + + // Serve static files from /logos/ + app.use('/logos/*', serveStatic({ root: webRoot })); + + // Serve other static files (favicon, etc.) + app.use('/favicon.ico', serveStatic({ root: webRoot })); + + return app; +} + +/** + * Build the injection script for runtime config. + * Escapes values to prevent XSS and script injection. + */ +function buildInjectionScript(config: WebUIRuntimeConfig): string { + const scripts: string[] = []; + + if (config.analytics) { + // Escape < to prevent script injection via JSON values + const safeJson = JSON.stringify(config.analytics).replace(/${scripts.join('\n')}`; +} + +/** + * Create a notFound handler for SPA fallback. + * + * This handler serves index.html for client-side routes (paths without file extensions). + * For paths with file extensions (like /openapi.json), it returns a standard 404. + * + * This should be registered as app.notFound() to run after all routes fail to match. + * + * @param webRoot - Absolute path to the directory containing WebUI build output + * @param runtimeConfig - Optional runtime configuration to inject into the HTML + */ +export function createSpaFallbackHandler( + webRoot: string, + runtimeConfig?: WebUIRuntimeConfig +): NotFoundHandler { + // Pre-build the injection script once (not per-request) + const injectionScript = runtimeConfig ? buildInjectionScript(runtimeConfig) : ''; + + return async (c) => { + const path = c.req.path; + + // If path ends with a file extension, it's a real 404 (not an SPA route) + // This allows /openapi.json, /.well-known/agent-card.json etc. to 404 properly + // Uses regex to avoid false positives like /session/2024.01.01 + if (/\.[a-zA-Z0-9]+$/.test(path)) { + return c.json({ error: 'Not Found', path }, 404); + } + + // SPA fallback - serve index.html for client-side routes + try { + let html = await readFile(join(webRoot, 'index.html'), 'utf-8'); + + // Inject runtime config into if provided + if (injectionScript) { + html = html.replace('', `${injectionScript}`); + } + + return c.html(html); + } catch { + // index.html not found - WebUI not available + return c.html( + ` + +Dexto API Server + +

Dexto API Server

+

WebUI is not available. API endpoints are accessible at /api/*

+ +`, + 200 + ); + } + }; +} diff --git a/dexto/packages/server/src/hono/routes/tools.ts b/dexto/packages/server/src/hono/routes/tools.ts new file mode 100644 index 00000000..03ef2ad6 --- /dev/null +++ b/dexto/packages/server/src/hono/routes/tools.ts @@ -0,0 +1,147 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { DextoAgent } from '@dexto/core'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +// JSON Schema definition for tool input parameters +const JsonSchemaProperty = z + .object({ + type: z + .enum(['string', 'number', 'integer', 'boolean', 'object', 'array']) + .optional() + .describe('Property type'), + description: z.string().optional().describe('Property description'), + enum: z + .array(z.union([z.string(), z.number(), z.boolean()])) + .optional() + .describe('Enum values'), + default: z.any().optional().describe('Default value'), + }) + .passthrough() + .describe('JSON Schema property definition'); + +const ToolInputSchema = z + .object({ + type: z.literal('object').optional().describe('Schema type, always "object" when present'), + properties: z.record(JsonSchemaProperty).optional().describe('Property definitions'), + required: z.array(z.string()).optional().describe('Required property names'), + }) + .passthrough() + .describe('JSON Schema for tool input parameters'); + +const ToolInfoSchema = z + .object({ + id: z.string().describe('Tool identifier'), + name: z.string().describe('Tool name'), + description: z.string().describe('Tool description'), + source: z + .enum(['internal', 'custom', 'mcp']) + .describe('Source of the tool (internal, custom, or mcp)'), + serverName: z.string().optional().describe('MCP server name (if source is mcp)'), + inputSchema: ToolInputSchema.optional().describe('JSON Schema for tool input parameters'), + }) + .strict() + .describe('Tool information'); + +const AllToolsResponseSchema = z + .object({ + tools: z.array(ToolInfoSchema).describe('Array of all available tools'), + totalCount: z.number().describe('Total number of tools'), + internalCount: z.number().describe('Number of internal tools'), + customCount: z.number().describe('Number of custom tools'), + mcpCount: z.number().describe('Number of MCP tools'), + }) + .strict() + .describe('All available tools from all sources'); + +export function createToolsRouter(getAgent: GetAgentFn) { + const app = new OpenAPIHono(); + + const allToolsRoute = createRoute({ + method: 'get', + path: '/tools', + summary: 'List All Tools', + description: + 'Retrieves all available tools from all sources (internal, custom, and MCP servers)', + tags: ['tools'], + responses: { + 200: { + description: 'All tools', + content: { 'application/json': { schema: AllToolsResponseSchema } }, + }, + }, + }); + + return app.openapi(allToolsRoute, async (ctx) => { + const agent = await getAgent(ctx); + + // Get all tools from all sources + const allTools = await agent.getAllTools(); + + // Get MCP tools with server metadata for proper grouping + const mcpToolsWithServerInfo = agent.getAllMcpToolsWithServerInfo(); + + const toolList: z.output[] = []; + + let internalCount = 0; + let customCount = 0; + let mcpCount = 0; + + for (const [toolName, toolInfo] of Object.entries(allTools)) { + // Determine source and extract server name + let source: 'internal' | 'custom' | 'mcp'; + let serverName: string | undefined; + + if (toolName.startsWith('mcp--')) { + // MCP tool - strip the mcp-- prefix to look up in cache + const mcpToolName = toolName.substring(5); // Remove 'mcp--' prefix + const mcpToolInfo = mcpToolsWithServerInfo.get(mcpToolName); + if (mcpToolInfo) { + source = 'mcp'; + serverName = mcpToolInfo.serverName; + mcpCount++; + } else { + // Fallback if not found in cache + source = 'mcp'; + mcpCount++; + } + } else if (toolName.startsWith('internal--')) { + source = 'internal'; + internalCount++; + } else if (toolName.startsWith('custom--')) { + source = 'custom'; + customCount++; + } else { + // Default to internal + source = 'internal'; + internalCount++; + } + + toolList.push({ + id: toolName, + name: toolName, + description: toolInfo.description || 'No description available', + source, + serverName, + inputSchema: toolInfo.parameters as z.output | undefined, + }); + } + + // Sort: internal first, then custom, then MCP + toolList.sort((a, b) => { + const sourceOrder = { internal: 0, custom: 1, mcp: 2 }; + if (a.source !== b.source) { + return sourceOrder[a.source] - sourceOrder[b.source]; + } + return a.name.localeCompare(b.name); + }); + + return ctx.json({ + tools: toolList, + totalCount: toolList.length, + internalCount, + customCount, + mcpCount, + }); + }); +} diff --git a/dexto/packages/server/src/hono/routes/webhooks.ts b/dexto/packages/server/src/hono/routes/webhooks.ts new file mode 100644 index 00000000..6483f33d --- /dev/null +++ b/dexto/packages/server/src/hono/routes/webhooks.ts @@ -0,0 +1,260 @@ +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; +import type { DextoAgent } from '@dexto/core'; +import { WebhookEventSubscriber } from '../../events/webhook-subscriber.js'; +import type { WebhookConfig } from '../../events/webhook-types.js'; +import type { Context } from 'hono'; +type GetAgentFn = (ctx: Context) => DextoAgent | Promise; + +// Response schemas +const WebhookResponseSchema = z + .object({ + id: z.string().describe('Unique webhook identifier'), + url: z.string().url().describe('Webhook URL'), + description: z.string().optional().describe('Webhook description'), + createdAt: z.union([z.date(), z.number()]).describe('Creation timestamp (Date or Unix ms)'), + }) + .strict() + .describe('Webhook response object'); + +const WebhookTestResultSchema = z + .object({ + success: z.boolean().describe('Whether the webhook test succeeded'), + statusCode: z.number().optional().describe('HTTP status code from webhook'), + responseTime: z.number().optional().describe('Response time in milliseconds'), + error: z.string().optional().describe('Error message if test failed'), + }) + .strict() + .describe('Webhook test result'); + +const WebhookBodySchema = z + .object({ + url: z + .string() + .url('Invalid URL format') + .describe('The URL to send webhook events to (must be a valid HTTP/HTTPS URL)'), + secret: z.string().optional().describe('A secret key for HMAC signature verification'), + description: z.string().optional().describe('A description of the webhook for reference'), + }) + .describe('Request body for registering a webhook'); + +export function createWebhooksRouter( + getAgent: GetAgentFn, + webhookSubscriber: WebhookEventSubscriber +) { + const app = new OpenAPIHono(); + + const registerRoute = createRoute({ + method: 'post', + path: '/webhooks', + summary: 'Register Webhook', + description: 'Registers a new webhook endpoint to receive agent events', + tags: ['webhooks'], + request: { body: { content: { 'application/json': { schema: WebhookBodySchema } } } }, + responses: { + 201: { + description: 'Webhook registered', + content: { + 'application/json': { + schema: z + .object({ + webhook: WebhookResponseSchema.describe( + 'Registered webhook details' + ), + }) + .strict(), + }, + }, + }, + }, + }); + + const listRoute = createRoute({ + method: 'get', + path: '/webhooks', + summary: 'List Webhooks', + description: 'Retrieves a list of all registered webhooks', + tags: ['webhooks'], + responses: { + 200: { + description: 'List webhooks', + content: { + 'application/json': { + schema: z + .object({ + webhooks: z + .array(WebhookResponseSchema) + .describe('Array of registered webhooks'), + }) + .strict(), + }, + }, + }, + }, + }); + + const getRoute = createRoute({ + method: 'get', + path: '/webhooks/{webhookId}', + summary: 'Get Webhook Details', + description: 'Fetches details for a specific webhook', + tags: ['webhooks'], + request: { params: z.object({ webhookId: z.string().describe('The webhook identifier') }) }, + responses: { + 200: { + description: 'Webhook', + content: { + 'application/json': { + schema: z + .object({ + webhook: WebhookResponseSchema.describe('Webhook details'), + }) + .strict(), + }, + }, + }, + 404: { description: 'Not found' }, + }, + }); + + const deleteRoute = createRoute({ + method: 'delete', + path: '/webhooks/{webhookId}', + summary: 'Delete Webhook', + description: 'Permanently removes a webhook endpoint. This action cannot be undone', + tags: ['webhooks'], + request: { params: z.object({ webhookId: z.string().describe('The webhook identifier') }) }, + responses: { + 200: { + description: 'Removed', + content: { + 'application/json': { + schema: z + .object({ + status: z + .literal('removed') + .describe('Operation status indicating successful removal'), + webhookId: z.string().describe('ID of the removed webhook'), + }) + .strict(), + }, + }, + }, + 404: { description: 'Not found' }, + }, + }); + + const testRoute = createRoute({ + method: 'post', + path: '/webhooks/{webhookId}/test', + summary: 'Test Webhook', + description: 'Sends a sample event to test webhook connectivity and configuration', + tags: ['webhooks'], + request: { params: z.object({ webhookId: z.string().describe('The webhook identifier') }) }, + responses: { + 200: { + description: 'Test result', + content: { + 'application/json': { + schema: z + .object({ + test: z + .literal('completed') + .describe('Test status indicating completion'), + result: WebhookTestResultSchema.describe('Test execution results'), + }) + .strict(), + }, + }, + }, + 404: { description: 'Not found' }, + }, + }); + + return app + .openapi(registerRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { url, secret, description } = ctx.req.valid('json'); + + const webhookId = `wh_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`; + const webhook: WebhookConfig = { + id: webhookId, + url, + createdAt: new Date(), + ...(secret && { secret }), + ...(description && { description }), + }; + + webhookSubscriber.addWebhook(webhook); + agent.logger.info(`Webhook registered: ${webhookId} -> ${url}`); + + return ctx.json( + { + webhook: { + id: webhook.id, + url: webhook.url, + description: webhook.description, + createdAt: webhook.createdAt, + }, + }, + 201 + ); + }) + .openapi(listRoute, async (ctx) => { + const webhooks = webhookSubscriber.getWebhooks().map((webhook) => ({ + id: webhook.id, + url: webhook.url, + description: webhook.description, + createdAt: webhook.createdAt, + })); + + return ctx.json({ webhooks }); + }) + .openapi(getRoute, (ctx) => { + const { webhookId } = ctx.req.valid('param'); + const webhook = webhookSubscriber.getWebhook(webhookId); + if (!webhook) { + return ctx.json({ error: 'Webhook not found' }, 404); + } + + return ctx.json({ + webhook: { + id: webhook.id, + url: webhook.url, + description: webhook.description, + createdAt: webhook.createdAt, + }, + }); + }) + .openapi(deleteRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { webhookId } = ctx.req.valid('param'); + const removed = webhookSubscriber.removeWebhook(webhookId); + if (!removed) { + return ctx.json({ error: 'Webhook not found' }, 404); + } + agent.logger.info(`Webhook removed: ${webhookId}`); + return ctx.json({ status: 'removed', webhookId }); + }) + .openapi(testRoute, async (ctx) => { + const agent = await getAgent(ctx); + const { webhookId } = ctx.req.valid('param'); + const webhook = webhookSubscriber.getWebhook(webhookId); + + if (!webhook) { + return ctx.json({ error: 'Webhook not found' }, 404); + } + + agent.logger.info(`Testing webhook: ${webhookId}`); + const result = await webhookSubscriber.testWebhook(webhookId); + + return ctx.json({ + test: 'completed', + result: { + success: result.success, + statusCode: result.statusCode, + responseTime: result.responseTime, + error: result.error, + }, + }); + }); +} diff --git a/dexto/packages/server/src/hono/schemas/responses.ts b/dexto/packages/server/src/hono/schemas/responses.ts new file mode 100644 index 00000000..4377c79e --- /dev/null +++ b/dexto/packages/server/src/hono/schemas/responses.ts @@ -0,0 +1,577 @@ +/** + * Response schemas for OpenAPI documentation + * + * This file defines Zod schemas for all API response types, following these principles: + * 1. Import reusable schemas from @dexto/core where available + * 2. Define message/context schemas HERE (not in core) - see note below + * 3. All schemas follow Zod best practices from CLAUDE.md (strict, describe, etc.) + * + * TYPE BOUNDARY: Core vs Server Schemas + * ------------------------------------- + * Core's TypeScript interfaces use rich union types for binary data: + * `image: string | Uint8Array | Buffer | ArrayBuffer | URL` + * + * This allows internal code to work with various binary formats before serialization. + * However, JSON API responses can only contain strings (base64-encoded). + * + * Server schemas use `z.string()` for these fields because: + * 1. JSON serialization converts all binary data to base64 strings + * 2. Hono client type inference works correctly with concrete types + * 3. WebUI receives properly typed `string` instead of `JSONValue` + * + * CONSEQUENCE: Route handlers that return core types (e.g., `InternalMessage[]`) + * need type casts when passing to `ctx.json()` because TypeScript sees the union + * type from core but the schema expects just `string`. At runtime the data IS + * already strings - the cast just bridges the static type mismatch. + * + * See routes/sessions.ts, routes/search.ts for examples with TODO comments. + */ + +import { z } from 'zod'; +import { LLMConfigBaseSchema as CoreLLMConfigBaseSchema, LLM_PROVIDERS } from '@dexto/core'; + +// TODO: Implement shared error response schemas for OpenAPI documentation. +// Currently, 404 and other error responses lack body schemas because @hono/zod-openapi +// enforces strict type matching between route definitions and handlers. When a 404 schema +// is defined, TypeScript expects handler return types to be a union of all response types, +// but the type system tries to match every return against every schema instead of by status code. +// +// Solution: Create a typed helper or wrapper that: +// 1. Defines a shared ErrorResponseSchema (e.g., { error: string, code?: string }) +// 2. Properly types handlers to return discriminated unions by status code +// 3. Can be reused across all routes for consistent error documentation +// +// See: https://github.com/honojs/middleware/tree/main/packages/zod-openapi for patterns + +// ============================================================================ +// Imports from @dexto/core - Reusable schemas +// ============================================================================ + +// Memory schemas +export { MemorySchema } from '@dexto/core'; + +// LLM schemas +export { LLMConfigBaseSchema, type ValidatedLLMConfig } from '@dexto/core'; + +// ============================================================================ +// Message/Context Schemas (defined here, not in core - see header comment) +// ============================================================================ + +export const TextPartSchema = z + .object({ + type: z.literal('text').describe('Part type: text'), + text: z.string().describe('Text content'), + }) + .strict() + .describe('Text content part'); + +export const ImagePartSchema = z + .object({ + type: z.literal('image').describe('Part type: image'), + image: z.string().describe('Base64-encoded image data'), + mimeType: z.string().optional().describe('MIME type of the image'), + }) + .strict() + .describe('Image content part'); + +export const FilePartSchema = z + .object({ + type: z.literal('file').describe('Part type: file'), + data: z.string().describe('Base64-encoded file data'), + mimeType: z.string().describe('MIME type of the file'), + filename: z.string().optional().describe('Optional filename'), + }) + .strict() + .describe('File content part'); + +export const UIResourcePartSchema = z + .object({ + type: z.literal('ui-resource').describe('Part type: ui-resource'), + uri: z.string().describe('URI identifying the UI resource (must start with ui://)'), + mimeType: z + .string() + .describe('MIME type: text/html, text/uri-list, or application/vnd.mcp-ui.remote-dom'), + content: z.string().optional().describe('Inline HTML content or URL'), + blob: z.string().optional().describe('Base64-encoded content (alternative to content)'), + metadata: z + .object({ + title: z.string().optional().describe('Display title for the UI resource'), + preferredSize: z + .object({ + width: z.number().describe('Preferred width in pixels'), + height: z.number().describe('Preferred height in pixels'), + }) + .strict() + .optional() + .describe('Preferred rendering size'), + }) + .strict() + .optional() + .describe('Optional metadata for the UI resource'), + }) + .strict() + .describe('UI Resource content part for MCP-UI interactive components'); + +export const ContentPartSchema = z + .discriminatedUnion('type', [ + TextPartSchema, + ImagePartSchema, + FilePartSchema, + UIResourcePartSchema, + ]) + .describe('Message content part (text, image, file, or UI resource)'); + +export const ToolCallSchema = z + .object({ + id: z.string().describe('Unique identifier for this tool call'), + type: z + .literal('function') + .describe('Tool call type (currently only function is supported)'), + function: z + .object({ + name: z.string().describe('Name of the function to call'), + arguments: z.string().describe('Arguments for the function in JSON string format'), + }) + .strict() + .describe('Function call details'), + }) + .strict() + .describe('Tool call made by the assistant'); + +export const TokenUsageSchema = z + .object({ + inputTokens: z.number().int().nonnegative().optional().describe('Number of input tokens'), + outputTokens: z.number().int().nonnegative().optional().describe('Number of output tokens'), + reasoningTokens: z + .number() + .int() + .nonnegative() + .optional() + .describe('Number of reasoning tokens'), + totalTokens: z.number().int().nonnegative().optional().describe('Total tokens used'), + }) + .strict() + .describe('Token usage accounting'); + +export const InternalMessageSchema = z + .object({ + id: z.string().uuid().optional().describe('Unique message identifier (UUID)'), + role: z + .enum(['system', 'user', 'assistant', 'tool']) + .describe('Role of the message sender'), + timestamp: z.number().int().positive().optional().describe('Creation timestamp (Unix ms)'), + content: z + .union([z.string(), z.null(), z.array(ContentPartSchema)]) + .describe('Message content (string, null, or array of parts)'), + reasoning: z.string().optional().describe('Optional model reasoning text'), + tokenUsage: TokenUsageSchema.optional().describe('Optional token usage accounting'), + model: z.string().optional().describe('Model identifier for assistant messages'), + provider: z + .enum(LLM_PROVIDERS) + .optional() + .describe('Provider identifier for assistant messages'), + toolCalls: z.array(ToolCallSchema).optional().describe('Tool calls made by the assistant'), + toolCallId: z.string().optional().describe('ID of the tool call this message responds to'), + name: z.string().optional().describe('Name of the tool that produced this result'), + success: z + .boolean() + .optional() + .describe('Whether tool execution succeeded (present for role=tool messages)'), + }) + .strict() + .describe('Internal message representation'); + +// Derived types for consumers +export type TextPart = z.output; +export type ImagePart = z.output; +export type FilePart = z.output; +export type ContentPart = z.output; +export type ToolCall = z.output; +export type TokenUsage = z.output; +export type InternalMessage = z.output; + +// ============================================================================ +// LLM Config Schemas +// ============================================================================ + +// LLM config response schema - omits apiKey for security +// API keys should never be returned in responses +export const LLMConfigResponseSchema = CoreLLMConfigBaseSchema.omit({ apiKey: true }) + .extend({ + hasApiKey: z.boolean().optional().describe('Whether an API key is configured'), + }) + .describe('LLM configuration (apiKey omitted for security)'); + +// Full LLM config schema for requests (includes apiKey with writeOnly) +export const LLMConfigSchema = CoreLLMConfigBaseSchema.describe('LLM configuration with API key'); + +export type LLMConfigResponse = z.output; + +// Agent schemas +export { AgentCardSchema, type AgentCard } from '@dexto/core'; + +// MCP schemas +export { + McpServerConfigSchema, + StdioServerConfigSchema, + SseServerConfigSchema, + HttpServerConfigSchema, + type McpServerConfig, + type ValidatedMcpServerConfig, +} from '@dexto/core'; + +// Tool schemas +export { ToolConfirmationConfigSchema } from '@dexto/core'; + +// Resource schemas +export { InternalResourceConfigSchema } from '@dexto/core'; + +// ============================================================================ +// New schemas for types that don't have Zod equivalents in core +// ============================================================================ + +// --- Session Schemas --- + +export const SessionMetadataSchema = z + .object({ + id: z.string().describe('Unique session identifier'), + createdAt: z + .number() + .int() + .positive() + .nullable() + .describe('Creation timestamp (Unix ms, null if unavailable)'), + lastActivity: z + .number() + .int() + .positive() + .nullable() + .describe('Last activity timestamp (Unix ms, null if unavailable)'), + messageCount: z + .number() + .int() + .nonnegative() + .describe('Total number of messages in session'), + title: z.string().optional().nullable().describe('Optional session title'), + }) + .strict() + .describe('Session metadata'); + +export type SessionMetadata = z.output; + +// --- Search Schemas --- + +export const SearchResultSchema = z + .object({ + sessionId: z.string().describe('Session ID where the message was found'), + message: InternalMessageSchema.describe('The message that matched the search'), + matchedText: z.string().describe('The specific text that matched the search query'), + context: z.string().describe('Context around the match for preview'), + messageIndex: z + .number() + .int() + .nonnegative() + .describe('Index of the message within the session'), + }) + .strict() + .describe('Result of a message search'); + +export type SearchResult = z.output; + +export const SessionSearchResultSchema = z + .object({ + sessionId: z.string().describe('Session ID'), + matchCount: z + .number() + .int() + .nonnegative() + .describe('Number of messages that matched in this session'), + firstMatch: SearchResultSchema.describe('Preview of the first matching message'), + metadata: z + .object({ + createdAt: z.number().int().positive().describe('Session creation timestamp'), + lastActivity: z.number().int().positive().describe('Last activity timestamp'), + messageCount: z.number().int().nonnegative().describe('Total messages in session'), + }) + .strict() + .describe('Session metadata'), + }) + .strict() + .describe('Result of a session search'); + +export type SessionSearchResult = z.output; + +export const MessageSearchResponseSchema = z + .object({ + results: z.array(SearchResultSchema).describe('Array of search results'), + total: z.number().int().nonnegative().describe('Total number of results available'), + hasMore: z.boolean().describe('Whether there are more results beyond the current page'), + query: z.string().describe('Query that was searched'), + }) + .strict() + .describe('Message search response'); + +export type MessageSearchResponse = z.output; + +export const SessionSearchResponseSchema = z + .object({ + results: z.array(SessionSearchResultSchema).describe('Array of session search results'), + total: z.number().int().nonnegative().describe('Total number of sessions with matches'), + hasMore: z + .boolean() + .describe( + 'Always false - session search returns all matching sessions without pagination' + ), + query: z.string().describe('Query that was searched'), + }) + .strict() + .describe('Session search response'); + +export type SessionSearchResponse = z.output; + +// --- Webhook Schemas --- + +export const WebhookSchema = z + .object({ + id: z.string().describe('Unique webhook identifier'), + url: z.string().url().describe('Webhook URL to send events to'), + events: z.array(z.string()).describe('Array of event types this webhook subscribes to'), + createdAt: z.number().int().positive().describe('Creation timestamp (Unix ms)'), + }) + .strict() + .describe('Webhook subscription'); + +export type Webhook = z.output; + +// --- LLM Provider/Model Schemas --- + +// Schema for ModelInfo from core registry +export const CatalogModelInfoSchema = z + .object({ + name: z.string().describe('Model name identifier'), + maxInputTokens: z.number().int().positive().describe('Maximum input tokens'), + default: z.boolean().optional().describe('Whether this is a default model'), + supportedFileTypes: z + .array(z.enum(['audio', 'pdf', 'image'])) + .describe('File types this model supports'), + displayName: z.string().optional().describe('Human-readable display name'), + pricing: z + .object({ + inputPerM: z.number().describe('Input cost per million tokens (USD)'), + outputPerM: z.number().describe('Output cost per million tokens (USD)'), + cacheReadPerM: z.number().optional().describe('Cache read cost per million tokens'), + cacheWritePerM: z + .number() + .optional() + .describe('Cache write cost per million tokens'), + currency: z.literal('USD').optional().describe('Currency'), + unit: z.literal('per_million_tokens').optional().describe('Unit'), + }) + .optional() + .describe('Pricing information in USD per million tokens'), + }) + .strict() + .describe('Model information from LLM registry'); + +export type CatalogModelInfo = z.output; + +// Schema for ProviderCatalog returned by /llm/catalog (grouped mode) +export const ProviderCatalogSchema = z + .object({ + name: z.string().describe('Provider display name'), + hasApiKey: z.boolean().describe('Whether API key is configured'), + primaryEnvVar: z.string().describe('Primary environment variable for API key'), + supportsBaseURL: z.boolean().describe('Whether custom base URLs are supported'), + models: z.array(CatalogModelInfoSchema).describe('Models available from this provider'), + supportedFileTypes: z + .array(z.enum(['audio', 'pdf', 'image'])) + .describe('Provider-level file type support'), + }) + .strict() + .describe('Provider catalog entry with models and capabilities'); + +export type ProviderCatalog = z.output; + +// Schema for flat model list (includes provider field) +export const ModelFlatSchema = CatalogModelInfoSchema.extend({ + provider: z.string().describe('Provider identifier for this model'), +}).describe('Flattened model entry with provider information'); + +export type ModelFlat = z.output; + +// --- Agent Registry Schemas --- + +export const AgentRegistryEntrySchema = z + .object({ + id: z.string().describe('Unique agent identifier'), + name: z.string().describe('Agent name'), + description: z.string().describe('Agent description'), + author: z.string().optional().describe('Agent author'), + tags: z.array(z.string()).optional().describe('Agent tags'), + type: z.enum(['builtin', 'custom']).describe('Agent type'), + }) + .strict() + .describe('Agent registry entry'); + +export type AgentRegistryEntry = z.output; + +// --- Resource Schemas --- + +// TODO: Consider refactoring to use discriminated union for better type safety: +// - MCP resources (source: 'mcp') should require serverName field +// - Internal resources (source: 'internal') should not have serverName field +// This would require updating core's ResourceMetadata interface to also use discriminated union +export const ResourceSchema = z + .object({ + uri: z.string().describe('Resource URI'), + name: z.string().optional().describe('Resource name'), + description: z.string().optional().describe('Resource description'), + mimeType: z.string().optional().describe('MIME type of the resource'), + source: z.enum(['mcp', 'internal']).describe('Source system that provides this resource'), + serverName: z + .string() + .optional() + .describe('Original server/provider name (for MCP resources)'), + size: z.number().optional().describe('Size of the resource in bytes (if known)'), + lastModified: z + .string() + .datetime() + .optional() + .describe('Last modified timestamp (ISO 8601 string)'), + metadata: z + .record(z.unknown()) + .optional() + .describe('Additional metadata specific to the resource type'), + }) + .strict() + .describe('Resource metadata'); + +export type Resource = z.output; + +// --- Tool Schemas --- + +export const ToolSchema = z + .object({ + name: z.string().describe('Tool name'), + description: z.string().describe('Tool description'), + inputSchema: z.record(z.unknown()).describe('JSON Schema for tool input parameters'), + }) + .strict() + .describe('Tool metadata'); + +export type Tool = z.output; + +// --- Prompt Schemas --- + +export const PromptArgumentSchema = z + .object({ + name: z.string().describe('Argument name'), + description: z.string().optional().describe('Argument description'), + required: z.boolean().optional().describe('Whether the argument is required'), + }) + .strict() + .describe('Prompt argument definition'); + +export type PromptArgument = z.output; + +export const PromptDefinitionSchema = z + .object({ + name: z.string().describe('Prompt name'), + title: z.string().optional().describe('Prompt title'), + description: z.string().optional().describe('Prompt description'), + arguments: z + .array(PromptArgumentSchema) + .optional() + .describe('Array of argument definitions'), + }) + .strict() + .describe('Prompt definition (MCP-compliant)'); + +export type PromptDefinition = z.output; + +export const PromptInfoSchema = z + .object({ + name: z.string().describe('Prompt name'), + title: z.string().optional().describe('Prompt title'), + description: z.string().optional().describe('Prompt description'), + arguments: z + .array(PromptArgumentSchema) + .optional() + .describe('Array of argument definitions'), + source: z.enum(['mcp', 'config', 'custom']).describe('Source of the prompt'), + metadata: z.record(z.unknown()).optional().describe('Additional metadata'), + }) + .strict() + .describe('Enhanced prompt information'); + +export type PromptInfo = z.output; + +export const PromptSchema = z + .object({ + id: z.string().describe('Unique prompt identifier'), + name: z.string().describe('Prompt name'), + description: z.string().optional().describe('Prompt description'), + content: z.string().describe('Prompt template content'), + variables: z + .array(z.string()) + .optional() + .describe('List of variable placeholders in the prompt'), + }) + .strict() + .describe('Prompt template'); + +export type Prompt = z.output; + +// ============================================================================ +// Common Response Patterns +// ============================================================================ + +// Generic success response with data +export const OkResponseSchema = (dataSchema: T) => + z + .object({ + ok: z.literal(true).describe('Indicates successful response'), + data: dataSchema.describe('Response data'), + }) + .strict() + .describe('Successful API response'); + +// Generic error response +export const ErrorResponseSchema = z + .object({ + ok: z.literal(false).describe('Indicates failed response'), + error: z + .object({ + message: z.string().describe('Error message'), + code: z.string().optional().describe('Error code'), + details: z.unknown().optional().describe('Additional error details'), + }) + .strict() + .describe('Error details'), + }) + .strict() + .describe('Error API response'); + +export type ErrorResponse = z.output; + +// Status response (for operations that don't return data) +export const StatusResponseSchema = z + .object({ + status: z.string().describe('Operation status'), + message: z.string().optional().describe('Optional status message'), + }) + .strict() + .describe('Status response'); + +export type StatusResponse = z.output; + +// Delete response +export const DeleteResponseSchema = z + .object({ + status: z.literal('deleted').describe('Indicates successful deletion'), + id: z.string().optional().describe('ID of the deleted resource'), + }) + .strict() + .describe('Delete operation response'); + +export type DeleteResponse = z.output; diff --git a/dexto/packages/server/src/hono/start-server.ts b/dexto/packages/server/src/hono/start-server.ts new file mode 100644 index 00000000..17652a48 --- /dev/null +++ b/dexto/packages/server/src/hono/start-server.ts @@ -0,0 +1,165 @@ +import type { Server } from 'node:http'; +import type { Context } from 'hono'; +import type { DextoAgent, AgentCard } from '@dexto/core'; +import { createAgentCard, logger } from '@dexto/core'; +import { createDextoApp } from './index.js'; +import { createNodeServer } from './node/index.js'; +import type { DextoApp } from './types.js'; +import type { WebUIRuntimeConfig } from './routes/static.js'; +import { WebhookEventSubscriber } from '../events/webhook-subscriber.js'; +import { A2ASseEventSubscriber } from '../events/a2a-sse-subscriber.js'; +import { ApprovalCoordinator } from '../approval/approval-coordinator.js'; +import { createManualApprovalHandler } from '../approval/manual-approval-handler.js'; + +export type StartDextoServerOptions = { + /** Port to listen on. Defaults to 3000 or process.env.PORT */ + port?: number; + /** Hostname to bind to. Defaults to 0.0.0.0 */ + hostname?: string; + /** Override agent card metadata (name, version, etc.) */ + agentCard?: Partial; + /** Absolute path to WebUI build output. If provided, static files will be served. */ + webRoot?: string; + /** Runtime configuration to inject into WebUI (analytics, etc.) */ + webUIConfig?: WebUIRuntimeConfig; + /** Base URL for agent card. Defaults to http://localhost:{port} */ + baseUrl?: string; +}; + +export type StartDextoServerResult = { + /** HTTP server instance */ + server: Server; + /** Hono app instance */ + app: DextoApp; + /** Stop the server and agent gracefully */ + stop: () => Promise; + /** Agent card with resolved metadata */ + agentCard: AgentCard; +}; + +/** + * Start a Dexto server with minimal configuration. + * + * This is a high-level helper that: + * 1. Creates event subscribers and approval coordinator + * 2. Creates and configures the Hono app + * 3. Wires up all the infrastructure (SSE, webhooks, approvals) + * 4. Starts the agent + * 5. Starts the HTTP server + * + * @example + * ```typescript + * // Register providers (filesystem-tools, process-tools, etc.) + * import '@dexto/image-local'; + * + * import { DextoAgent } from '@dexto/core'; + * import { loadAgentConfig } from '@dexto/agent-management'; + * import { startDextoServer } from '@dexto/server'; + * + * const config = await loadAgentConfig('./agents/default.yml'); + * const agent = new DextoAgent(config, './agents/default.yml'); + * + * const { server, stop } = await startDextoServer(agent, { + * port: 3000, + * agentCard: { name: 'My Agent' } + * }); + * + * // Server is now running at http://localhost:3000 + * // To stop: await stop(); + * ``` + */ +export async function startDextoServer( + agent: DextoAgent, + options: StartDextoServerOptions = {} +): Promise { + const { + port: requestedPort, + hostname = '0.0.0.0', + agentCard: agentCardOverride = {}, + webRoot, + webUIConfig, + baseUrl: baseUrlOverride, + } = options; + + // Resolve port from options, env, or default + const resolvedPort = requestedPort ?? (process.env.PORT ? Number(process.env.PORT) : 3000); + const baseUrl = baseUrlOverride ?? `http://localhost:${resolvedPort}`; + + logger.info(`Initializing Dexto server on ${hostname}:${resolvedPort}...`); + + // Create agent card with overrides + const agentCard = createAgentCard( + { + defaultName: agentCardOverride.name ?? 'dexto-agent', + defaultVersion: agentCardOverride.version ?? '1.0.0', + defaultBaseUrl: baseUrl, + }, + agentCardOverride + ); + + // Create event subscribers and approval coordinator + logger.debug('Creating event infrastructure...'); + const webhookSubscriber = new WebhookEventSubscriber(); + const sseSubscriber = new A2ASseEventSubscriber(); + const approvalCoordinator = new ApprovalCoordinator(); + + // Create Hono app + logger.debug('Creating Hono application...'); + const app = createDextoApp({ + getAgent: (_ctx: Context) => agent, + getAgentCard: () => agentCard, + approvalCoordinator, + webhookSubscriber, + sseSubscriber, + ...(webRoot ? { webRoot } : {}), + ...(webUIConfig ? { webUIConfig } : {}), + }); + + // Create Node.js HTTP server + logger.debug('Creating Node.js HTTP server...'); + const { server, webhookSubscriber: bridgeWebhookSubscriber } = createNodeServer(app, { + getAgent: () => agent, + port: resolvedPort, + hostname, + }); + + // Register webhook subscriber with agent for LLM streaming events + if (bridgeWebhookSubscriber) { + logger.debug('Registering webhook subscriber with agent...'); + agent.registerSubscriber(bridgeWebhookSubscriber); + } + + // Set approval handler if manual mode OR elicitation enabled + const needsHandler = + agent.config.toolConfirmation?.mode === 'manual' || agent.config.elicitation.enabled; + + if (needsHandler) { + logger.debug('Setting up manual approval handler...'); + const handler = createManualApprovalHandler(approvalCoordinator); + agent.setApprovalHandler(handler); + } + + // Wire SSE subscribers to agent event bus + logger.debug('Wiring event subscribers to agent...'); + webhookSubscriber.subscribe(agent.agentEventBus); + sseSubscriber.subscribe(agent.agentEventBus); + + // Start the agent + logger.info('Starting agent...'); + await agent.start(); + + logger.info(`Server running at http://${hostname}:${resolvedPort}`, null, 'green'); + + // Return result with stop function + return { + server, + app, + agentCard, + stop: async () => { + logger.info('Stopping Dexto server...'); + await agent.stop(); + server.close(); + logger.info('Server stopped', null, 'yellow'); + }, + }; +} diff --git a/dexto/packages/server/src/hono/types.ts b/dexto/packages/server/src/hono/types.ts new file mode 100644 index 00000000..251aebf7 --- /dev/null +++ b/dexto/packages/server/src/hono/types.ts @@ -0,0 +1,6 @@ +import type { OpenAPIHono } from '@hono/zod-openapi'; +import type { WebhookEventSubscriber } from '../events/webhook-subscriber.js'; + +export type DextoApp = OpenAPIHono & { + webhookSubscriber?: WebhookEventSubscriber; +}; diff --git a/dexto/packages/server/src/index.ts b/dexto/packages/server/src/index.ts new file mode 100644 index 00000000..03369af7 --- /dev/null +++ b/dexto/packages/server/src/index.ts @@ -0,0 +1,11 @@ +export * from './hono/index.js'; +export * from './hono/node/index.js'; +export * from './hono/start-server.js'; +export type { DextoApp } from './hono/types.js'; +export * from './events/webhook-subscriber.js'; +export * from './events/a2a-sse-subscriber.js'; +export * from './events/webhook-types.js'; +export * from './events/types.js'; +export * from './mcp/mcp-handler.js'; +export * from './approval/manual-approval-handler.js'; +export * from './approval/approval-coordinator.js'; diff --git a/dexto/packages/server/src/mcp/mcp-handler.ts b/dexto/packages/server/src/mcp/mcp-handler.ts new file mode 100644 index 00000000..d03180eb --- /dev/null +++ b/dexto/packages/server/src/mcp/mcp-handler.ts @@ -0,0 +1,146 @@ +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { ReadResourceCallback } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js'; +import type { AgentCard, IDextoLogger } from '@dexto/core'; +import { logger } from '@dexto/core'; +import { z } from 'zod'; +import type { DextoAgent } from '@dexto/core'; +import { randomUUID } from 'crypto'; + +export type McpTransportType = 'stdio' | 'sse' | 'http'; + +export async function createMcpTransport( + transportType: McpTransportType = 'http' +): Promise { + logger.info(`Creating MCP transport of type: ${transportType}`); + + switch (transportType) { + case 'stdio': + return new StdioServerTransport(); + case 'sse': + throw new Error( + 'SSE transport requires HTTP response context and should be created per-request' + ); + default: { + // Use stateless mode (no session management) for better compatibility + // with clients like OpenAI that may not properly handle Mcp-Session-Id headers + return new StreamableHTTPServerTransport({ + enableJsonResponse: true, + }) as Transport; + } + } +} + +export async function initializeMcpServer( + agent: DextoAgent, + agentCardData: AgentCard, + mcpTransport: Transport +): Promise { + const mcpServer = new McpServer( + { name: agentCardData.name, version: agentCardData.version }, + { + capabilities: { + resources: {}, + }, + } + ); + + const toolName = 'chat_with_agent'; + const toolDescription = 'Allows you to chat with the an AI agent. Send a message to interact.'; + + mcpServer.tool( + toolName, + toolDescription, + { message: z.string() }, + async ({ message }: { message: string }) => { + agent.logger.info( + `MCP tool '${toolName}' received message: ${message.substring(0, 100)}${message.length > 100 ? '...' : ''}` + ); + // Create ephemeral session for this MCP tool call (stateless MCP interactions) + const session = await agent.createSession(`mcp-${randomUUID()}`); + try { + const text = await agent.run(message, undefined, undefined, session.id); + agent.logger.info( + `MCP tool '${toolName}' sending response: ${text?.substring(0, 100)}${(text?.length ?? 0) > 100 ? '...' : ''}` + ); + return { content: [{ type: 'text', text: text ?? '' }] }; + } finally { + // Always clean up ephemeral session to prevent accumulation + await agent + .deleteSession(session.id) + .catch((err) => + agent.logger.warn(`Failed to cleanup MCP session ${session.id}: ${err}`) + ); + } + } + ); + agent.logger.info(`Registered MCP tool: '${toolName}'`); + + await initializeAgentCardResource(mcpServer, agentCardData, agent.logger); + + agent.logger.info(`Initializing MCP protocol server connection...`); + await mcpServer.connect(mcpTransport); + agent.logger.info(`✅ MCP server protocol connected via transport.`); + return mcpServer; +} + +export async function initializeAgentCardResource( + mcpServer: McpServer, + agentCardData: AgentCard, + agentLogger: IDextoLogger +): Promise { + const agentCardResourceProgrammaticName = 'agentCard'; + const agentCardResourceUri = 'dexto://agent/card'; + try { + const readCallback: ReadResourceCallback = async (uri, _extra) => { + agentLogger.info(`MCP client requesting resource at ${uri.href}`); + return { + contents: [ + { + uri: uri.href, + type: 'application/json', + text: JSON.stringify(agentCardData, null, 2), + }, + ], + }; + }; + mcpServer.resource(agentCardResourceProgrammaticName, agentCardResourceUri, readCallback); + agentLogger.info( + `Registered MCP Resource: '${agentCardResourceProgrammaticName}' at URI '${agentCardResourceUri}'` + ); + } catch (e: any) { + agentLogger.warn( + `Error attempting to register MCP Resource '${agentCardResourceProgrammaticName}': ${e.message}. Check SDK.` + ); + } +} + +export function createMcpHttpHandlers(mcpTransport: Transport) { + if (!(mcpTransport instanceof StreamableHTTPServerTransport)) { + logger.info('Non-HTTP transport detected. Skipping HTTP route setup.'); + return null; + } + + const handlePost = async (req: IncomingMessage, res: ServerResponse, body: unknown) => { + logger.info(`MCP POST /mcp received request body: ${JSON.stringify(body)}`); + try { + await mcpTransport.handleRequest(req, res, body); + } catch (err) { + logger.error(`MCP POST error: ${JSON.stringify(err, null, 2)}`); + } + }; + + const handleGet = async (req: IncomingMessage, res: ServerResponse) => { + logger.info('MCP GET /mcp received request, attempting to establish SSE connection.'); + try { + await mcpTransport.handleRequest(req, res); + } catch (err) { + logger.error(`MCP GET error: ${JSON.stringify(err, null, 2)}`); + } + }; + + return { handlePost, handleGet }; +} diff --git a/dexto/packages/server/tsconfig.json b/dexto/packages/server/tsconfig.json new file mode 100644 index 00000000..93274537 --- /dev/null +++ b/dexto/packages/server/tsconfig.json @@ -0,0 +1,27 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": false, + "paths": { + "@dexto/core": ["../core/dist/index.d.ts"], + "@core/*": ["../core/dist/*"], + "@dexto/agent-management": ["../agent-management/dist/index.d.ts"] + } + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "**/*.test.ts", + "**/*.spec.ts", + "**/*.integration.test.ts", + "dist", + "node_modules" + ] +} diff --git a/dexto/packages/server/tsconfig.typecheck.json b/dexto/packages/server/tsconfig.typecheck.json new file mode 100644 index 00000000..90a93a82 --- /dev/null +++ b/dexto/packages/server/tsconfig.typecheck.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["dist", "node_modules"] +} diff --git a/dexto/packages/server/tsup.config.ts b/dexto/packages/server/tsup.config.ts new file mode 100644 index 00000000..533526f8 --- /dev/null +++ b/dexto/packages/server/tsup.config.ts @@ -0,0 +1,24 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig([ + { + entry: [ + 'src/**/*.ts', + '!src/**/*.test.ts', + '!src/**/*.spec.ts', + '!src/**/*.integration.test.ts', + ], + format: ['esm', 'cjs'], + outDir: 'dist', + dts: false, // Disable DTS generation in tsup to avoid worker memory issues + clean: true, + bundle: false, + platform: 'node', + esbuildOptions(options) { + options.logOverride = { + ...(options.logOverride ?? {}), + 'empty-import-meta': 'silent', + }; + }, + }, +]); diff --git a/dexto/packages/server/vitest.config.ts b/dexto/packages/server/vitest.config.ts new file mode 100644 index 00000000..06a9b068 --- /dev/null +++ b/dexto/packages/server/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + include: ['**/*.test.ts', '**/*.spec.ts', '**/*.integration.test.ts'], + }, +}); diff --git a/dexto/packages/tools-filesystem/CHANGELOG.md b/dexto/packages/tools-filesystem/CHANGELOG.md new file mode 100644 index 00000000..d9144be2 --- /dev/null +++ b/dexto/packages/tools-filesystem/CHANGELOG.md @@ -0,0 +1,110 @@ +# @dexto/tools-filesystem + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +- Updated dependencies [042f4f0] + - @dexto/core@1.5.6 + +## 1.5.5 + +### Patch Changes + +- 6df3ca9: Updated readme. Removed stale filesystem and process tool from dexto/core. +- Updated dependencies [63fa083] +- Updated dependencies [6df3ca9] + - @dexto/core@1.5.5 + +## 1.5.4 + +### Patch Changes + +- aa2c9a0: - new --dev flag for using dev mode with the CLI (for maintainers) (sets DEXTO_DEV_MODE=true and ensures local files are used) + - improved bash tool descriptions + - fixed explore agent task description getting truncated + - fixed some alignment issues + - fix search/find tools not asking approval for working outside directory + - add sound feature (sounds when approval reqd, when loop done) + - configurable in `preferences.yml` (on by default) and in `~/.dexto/sounds`, instructions in comment in `~/.dexto/preferences.yml` + - add new `env` system prompt contributor that includes info about os, working directory, git status. useful for coding agent to get enough context to improve cmd construction without unnecessary directory shifts + - support for loading `.claude/commands` and `.cursor/commands` global and local commands in addition to `.dexto/commands` +- Updated dependencies [0016cd3] +- Updated dependencies [499b890] +- Updated dependencies [aa2c9a0] + - @dexto/core@1.5.4 + +## 1.5.3 + +### Patch Changes + +- 4f00295: Added spawn-agent tools and explore agent. +- 69c944c: File integrity & performance improvements, approval system fixes, and developer experience enhancements + + ### File System Improvements + - **File integrity protection**: Store file hashes to prevent edits from corrupting files when content changes between operations (resolves #516) + - **Performance optimization**: Disable backups and remove redundant reads, switch to async non-blocking reads for faster file writes + + ### Approval System Fixes + - **Coding agent auto-approve**: Fix auto-approve not working due to incorrect tool names in auto-approve policies + - **Parallel tool calls**: Fix multiple parallel same-tool calls requiring redundant approvals - now checks all waiting approvals and resolves ones affected by newly approved commands + - **Refactored CLI approval handler**: Decoupled approval handler pattern from server for better separation of concerns + + ### Shell & Scripting Fixes + - **Bash mode aliases**: Fix bash mode not honoring zsh aliases + - **Script improvements**: Miscellaneous script improvements for better developer experience + +- Updated dependencies [4f00295] +- Updated dependencies [69c944c] + - @dexto/core@1.5.3 + +## 1.5.2 + +### Patch Changes + +- Updated dependencies [8a85ea4] +- Updated dependencies [527f3f9] + - @dexto/core@1.5.2 + +## 1.5.1 + +### Patch Changes + +- Updated dependencies [bfcc7b1] +- Updated dependencies [4aabdb7] + - @dexto/core@1.5.1 + +## 1.5.0 + +### Minor Changes + +- e7722e5: Minor version bump for new release with bundler, custom tool pkgs, etc. + +### Patch Changes + +- 1e7e974: Added image bundler, @dexto/image-local and moved tool services outside core. Added registry providers to select core services. +- Updated dependencies [ee12727] +- Updated dependencies [1e7e974] +- Updated dependencies [4c05310] +- Updated dependencies [5fa79fa] +- Updated dependencies [ef40e60] +- Updated dependencies [e714418] +- Updated dependencies [e7722e5] +- Updated dependencies [7d5ab19] +- Updated dependencies [436a900] + - @dexto/core@1.5.0 diff --git a/dexto/packages/tools-filesystem/package.json b/dexto/packages/tools-filesystem/package.json new file mode 100644 index 00000000..c648be70 --- /dev/null +++ b/dexto/packages/tools-filesystem/package.json @@ -0,0 +1,42 @@ +{ + "name": "@dexto/tools-filesystem", + "version": "1.5.6", + "description": "FileSystem tools provider for Dexto agents", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsup", + "typecheck": "tsc --noEmit", + "clean": "rm -rf dist" + }, + "keywords": [ + "dexto", + "tools", + "filesystem", + "file-operations" + ], + "dependencies": { + "@dexto/core": "workspace:*", + "diff": "^7.0.0", + "glob": "^11.1.0", + "safe-regex": "^2.1.1", + "zod": "^3.25.0" + }, + "devDependencies": { + "@types/diff": "^5.2.3", + "@types/safe-regex": "^1.1.6", + "tsup": "^8.0.0", + "typescript": "^5.3.3" + }, + "files": [ + "dist", + "README.md" + ] +} diff --git a/dexto/packages/tools-filesystem/src/directory-approval.integration.test.ts b/dexto/packages/tools-filesystem/src/directory-approval.integration.test.ts new file mode 100644 index 00000000..c90ba8fc --- /dev/null +++ b/dexto/packages/tools-filesystem/src/directory-approval.integration.test.ts @@ -0,0 +1,641 @@ +/** + * Directory Approval Integration Tests + * + * Tests for the directory access permission system integrated into file tools. + * + * Key behaviors tested: + * 1. Working directory: No directory prompt, normal tool flow + * 2. External dir (first access): Directory prompt via getApprovalOverride + * 3. External dir (after "session" approval): No directory prompt + * 4. External dir (after "once" approval): Directory prompt every time + * 5. Path containment: approving /ext covers /ext/sub/file.txt + */ + +import { describe, it, expect, beforeEach, afterEach, vi, type Mock } from 'vitest'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import { createReadFileTool } from './read-file-tool.js'; +import { createWriteFileTool } from './write-file-tool.js'; +import { createEditFileTool } from './edit-file-tool.js'; +import { FileSystemService } from './filesystem-service.js'; +import type { DirectoryApprovalCallbacks, FileToolOptions } from './file-tool-types.js'; +import { ApprovalType, ApprovalStatus } from '@dexto/core'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +describe('Directory Approval Integration Tests', () => { + let mockLogger: ReturnType; + let tempDir: string; + let fileSystemService: FileSystemService; + let directoryApproval: DirectoryApprovalCallbacks; + let isSessionApprovedMock: Mock; + let addApprovedMock: Mock; + + beforeEach(async () => { + mockLogger = createMockLogger(); + + // Create temp directory for testing + // Resolve real path to handle symlinks (macOS /tmp -> /private/var/folders/...) + const rawTempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-fs-test-')); + tempDir = await fs.realpath(rawTempDir); + + // Create FileSystemService with temp dir as working directory + fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: false, + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + // Create directory approval callbacks + isSessionApprovedMock = vi.fn().mockReturnValue(false); + addApprovedMock = vi.fn(); + directoryApproval = { + isSessionApproved: isSessionApprovedMock, + addApproved: addApprovedMock, + }; + + vi.clearAllMocks(); + }); + + afterEach(async () => { + // Cleanup temp directory + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + // ===================================================================== + // READ FILE TOOL TESTS + // ===================================================================== + + describe('Read File Tool', () => { + describe('getApprovalOverride', () => { + it('should return null for paths within working directory (no prompt needed)', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + // Create test file in working directory + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'test content'); + + const override = await tool.getApprovalOverride?.({ file_path: testFile }); + expect(override).toBeNull(); + }); + + it('should return directory access approval for external paths', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + // External path (outside working directory) + const externalPath = '/external/project/file.ts'; + + const override = await tool.getApprovalOverride?.({ file_path: externalPath }); + + expect(override).not.toBeNull(); + expect(override?.type).toBe(ApprovalType.DIRECTORY_ACCESS); + const metadata = override?.metadata as any; + expect(metadata?.path).toBe(path.resolve(externalPath)); + expect(metadata?.parentDir).toBe(path.dirname(path.resolve(externalPath))); + expect(metadata?.operation).toBe('read'); + expect(metadata?.toolName).toBe('read_file'); + }); + + it('should return null when external path is session-approved', async () => { + isSessionApprovedMock.mockReturnValue(true); + + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath = '/external/project/file.ts'; + + const override = await tool.getApprovalOverride?.({ file_path: externalPath }); + expect(override).toBeNull(); + expect(isSessionApprovedMock).toHaveBeenCalledWith(externalPath); + }); + + it('should return null when file_path is missing', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + const override = await tool.getApprovalOverride?.({}); + expect(override).toBeNull(); + }); + }); + + describe('onApprovalGranted', () => { + it('should add directory as session-approved when rememberDirectory is true', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + // First trigger getApprovalOverride to set pendingApprovalParentDir + const externalPath = '/external/project/file.ts'; + await tool.getApprovalOverride?.({ file_path: externalPath }); + + // Then call onApprovalGranted with rememberDirectory: true + tool.onApprovalGranted?.({ + approvalId: 'test-approval', + status: ApprovalStatus.APPROVED, + data: { rememberDirectory: true }, + }); + + expect(addApprovedMock).toHaveBeenCalledWith( + path.dirname(path.resolve(externalPath)), + 'session' + ); + }); + + it('should add directory as once-approved when rememberDirectory is false', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath = '/external/project/file.ts'; + await tool.getApprovalOverride?.({ file_path: externalPath }); + + tool.onApprovalGranted?.({ + approvalId: 'test-approval', + status: ApprovalStatus.APPROVED, + data: { rememberDirectory: false }, + }); + + expect(addApprovedMock).toHaveBeenCalledWith( + path.dirname(path.resolve(externalPath)), + 'once' + ); + }); + + it('should default to once-approved when rememberDirectory is not specified', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath = '/external/project/file.ts'; + await tool.getApprovalOverride?.({ file_path: externalPath }); + + tool.onApprovalGranted?.({ + approvalId: 'test-approval', + status: ApprovalStatus.APPROVED, + data: {}, + }); + + expect(addApprovedMock).toHaveBeenCalledWith( + path.dirname(path.resolve(externalPath)), + 'once' + ); + }); + + it('should not call addApproved when directoryApproval is not provided', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval: undefined, + }); + + const externalPath = '/external/project/file.ts'; + await tool.getApprovalOverride?.({ file_path: externalPath }); + + // Should not throw + tool.onApprovalGranted?.({ + approvalId: 'test-approval', + status: ApprovalStatus.APPROVED, + data: { rememberDirectory: true }, + }); + + expect(addApprovedMock).not.toHaveBeenCalled(); + }); + }); + + describe('execute', () => { + it('should read file contents within working directory', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + const testFile = path.join(tempDir, 'readable.txt'); + await fs.writeFile(testFile, 'Hello, world!\nLine 2'); + + const result = (await tool.execute({ file_path: testFile }, {})) as any; + + expect(result.content).toBe('Hello, world!\nLine 2'); + expect(result.lines).toBe(2); + }); + }); + }); + + // ===================================================================== + // WRITE FILE TOOL TESTS + // ===================================================================== + + describe('Write File Tool', () => { + describe('getApprovalOverride', () => { + it('should return null for paths within working directory', async () => { + const tool = createWriteFileTool({ + fileSystemService, + directoryApproval, + }); + + const testFile = path.join(tempDir, 'new-file.txt'); + + const override = await tool.getApprovalOverride?.({ + file_path: testFile, + content: 'test', + }); + expect(override).toBeNull(); + }); + + it('should return directory access approval for external paths', async () => { + const tool = createWriteFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath = '/external/project/new.ts'; + + const override = await tool.getApprovalOverride?.({ + file_path: externalPath, + content: 'test', + }); + + expect(override).not.toBeNull(); + expect(override?.type).toBe(ApprovalType.DIRECTORY_ACCESS); + const metadata = override?.metadata as any; + expect(metadata?.operation).toBe('write'); + expect(metadata?.toolName).toBe('write_file'); + }); + + it('should return null when external path is session-approved', async () => { + isSessionApprovedMock.mockReturnValue(true); + + const tool = createWriteFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath = '/external/project/new.ts'; + + const override = await tool.getApprovalOverride?.({ + file_path: externalPath, + content: 'test', + }); + expect(override).toBeNull(); + }); + }); + + describe('onApprovalGranted', () => { + it('should add directory as session-approved when rememberDirectory is true', async () => { + const tool = createWriteFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath = '/external/project/new.ts'; + await tool.getApprovalOverride?.({ file_path: externalPath, content: 'test' }); + + tool.onApprovalGranted?.({ + approvalId: 'test-approval', + status: ApprovalStatus.APPROVED, + data: { rememberDirectory: true }, + }); + + expect(addApprovedMock).toHaveBeenCalledWith( + path.dirname(path.resolve(externalPath)), + 'session' + ); + }); + }); + }); + + // ===================================================================== + // EDIT FILE TOOL TESTS + // ===================================================================== + + describe('Edit File Tool', () => { + describe('getApprovalOverride', () => { + it('should return null for paths within working directory', async () => { + const tool = createEditFileTool({ + fileSystemService, + directoryApproval, + }); + + const testFile = path.join(tempDir, 'existing.txt'); + + const override = await tool.getApprovalOverride?.({ + file_path: testFile, + old_string: 'old', + new_string: 'new', + }); + expect(override).toBeNull(); + }); + + it('should return directory access approval for external paths', async () => { + const tool = createEditFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath = '/external/project/existing.ts'; + + const override = await tool.getApprovalOverride?.({ + file_path: externalPath, + old_string: 'old', + new_string: 'new', + }); + + expect(override).not.toBeNull(); + expect(override?.type).toBe(ApprovalType.DIRECTORY_ACCESS); + const metadata = override?.metadata as any; + expect(metadata?.operation).toBe('edit'); + expect(metadata?.toolName).toBe('edit_file'); + }); + + it('should return null when external path is session-approved', async () => { + isSessionApprovedMock.mockReturnValue(true); + + const tool = createEditFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath = '/external/project/existing.ts'; + + const override = await tool.getApprovalOverride?.({ + file_path: externalPath, + old_string: 'old', + new_string: 'new', + }); + expect(override).toBeNull(); + }); + }); + }); + + // ===================================================================== + // SESSION VS ONCE APPROVAL SCENARIOS + // ===================================================================== + + describe('Session vs Once Approval Scenarios', () => { + it('should not prompt for subsequent requests after session approval', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath1 = '/external/project/file1.ts'; + const externalPath2 = '/external/project/file2.ts'; + + // First request - needs approval + let override = await tool.getApprovalOverride?.({ file_path: externalPath1 }); + expect(override).not.toBeNull(); + + // Simulate session approval + tool.onApprovalGranted?.({ + approvalId: 'approval-1', + status: ApprovalStatus.APPROVED, + data: { rememberDirectory: true }, + }); + + // Verify addApproved was called with 'session' + expect(addApprovedMock).toHaveBeenCalledWith( + path.dirname(path.resolve(externalPath1)), + 'session' + ); + + // Now simulate that isSessionApproved returns true for the approved directory + isSessionApprovedMock.mockReturnValue(true); + + // Second request - should not need approval (session approved) + override = await tool.getApprovalOverride?.({ file_path: externalPath2 }); + expect(override).toBeNull(); + }); + + it('should prompt for subsequent requests after once approval', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + const externalPath1 = '/external/project/file1.ts'; + const externalPath2 = '/external/project/file2.ts'; + + // First request - needs approval + let override = await tool.getApprovalOverride?.({ file_path: externalPath1 }); + expect(override).not.toBeNull(); + + // Simulate once approval + tool.onApprovalGranted?.({ + approvalId: 'approval-1', + status: ApprovalStatus.APPROVED, + data: { rememberDirectory: false }, + }); + + // Verify addApproved was called with 'once' + expect(addApprovedMock).toHaveBeenCalledWith( + path.dirname(path.resolve(externalPath1)), + 'once' + ); + + // isSessionApproved stays false for 'once' approvals + isSessionApprovedMock.mockReturnValue(false); + + // Second request - should still need approval (only 'once') + override = await tool.getApprovalOverride?.({ file_path: externalPath2 }); + expect(override).not.toBeNull(); + }); + }); + + // ===================================================================== + // PATH CONTAINMENT SCENARIOS + // ===================================================================== + + describe('Path Containment Scenarios', () => { + it('should cover child paths when parent directory is session-approved', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + // isSessionApproved checks if file is within any session-approved directory + // If /external/project is session-approved, /external/project/deep/file.ts should also be covered + isSessionApprovedMock.mockImplementation((filePath: string) => { + const normalizedPath = path.resolve(filePath); + const approvedDir = '/external/project'; + return ( + normalizedPath.startsWith(approvedDir + path.sep) || + normalizedPath === approvedDir + ); + }); + + // Direct child path - should be approved + let override = await tool.getApprovalOverride?.({ + file_path: '/external/project/file.ts', + }); + expect(override).toBeNull(); + + // Deep nested path - should also be approved + override = await tool.getApprovalOverride?.({ + file_path: '/external/project/deep/nested/file.ts', + }); + expect(override).toBeNull(); + }); + + it('should NOT cover sibling directories', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + // Only /external/sub is approved + isSessionApprovedMock.mockImplementation((filePath: string) => { + const normalizedPath = path.resolve(filePath); + const approvedDir = '/external/sub'; + return ( + normalizedPath.startsWith(approvedDir + path.sep) || + normalizedPath === approvedDir + ); + }); + + // /external/sub path - approved + let override = await tool.getApprovalOverride?.({ file_path: '/external/sub/file.ts' }); + expect(override).toBeNull(); + + // /external/other path - NOT approved (sibling directory) + override = await tool.getApprovalOverride?.({ file_path: '/external/other/file.ts' }); + expect(override).not.toBeNull(); + }); + }); + + // ===================================================================== + // DIFFERENT EXTERNAL DIRECTORIES SCENARIOS + // ===================================================================== + + describe('Different External Directories Scenarios', () => { + it('should require separate approval for different external directories', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval, + }); + + const dir1Path = '/external/project1/file.ts'; + const dir2Path = '/external/project2/file.ts'; + + // Both directories need approval + const override1 = await tool.getApprovalOverride?.({ file_path: dir1Path }); + expect(override1).not.toBeNull(); + const metadata1 = override1?.metadata as any; + expect(metadata1?.parentDir).toBe('/external/project1'); + + const override2 = await tool.getApprovalOverride?.({ file_path: dir2Path }); + expect(override2).not.toBeNull(); + const metadata2 = override2?.metadata as any; + expect(metadata2?.parentDir).toBe('/external/project2'); + }); + }); + + // ===================================================================== + // MIXED OPERATIONS SCENARIOS + // ===================================================================== + + describe('Mixed Operations Scenarios', () => { + it('should share directory approval across different file operations', async () => { + const readTool = createReadFileTool({ fileSystemService, directoryApproval }); + const writeTool = createWriteFileTool({ fileSystemService, directoryApproval }); + const editTool = createEditFileTool({ fileSystemService, directoryApproval }); + + const externalDir = '/external/project'; + + // All operations need approval initially + expect( + await readTool.getApprovalOverride?.({ file_path: `${externalDir}/file1.ts` }) + ).not.toBeNull(); + expect( + await writeTool.getApprovalOverride?.({ + file_path: `${externalDir}/file2.ts`, + content: 'test', + }) + ).not.toBeNull(); + expect( + await editTool.getApprovalOverride?.({ + file_path: `${externalDir}/file3.ts`, + old_string: 'a', + new_string: 'b', + }) + ).not.toBeNull(); + + // Simulate session approval for the directory + isSessionApprovedMock.mockReturnValue(true); + + // Now all operations should not need approval + expect( + await readTool.getApprovalOverride?.({ file_path: `${externalDir}/file1.ts` }) + ).toBeNull(); + expect( + await writeTool.getApprovalOverride?.({ + file_path: `${externalDir}/file2.ts`, + content: 'test', + }) + ).toBeNull(); + expect( + await editTool.getApprovalOverride?.({ + file_path: `${externalDir}/file3.ts`, + old_string: 'a', + new_string: 'b', + }) + ).toBeNull(); + }); + }); + + // ===================================================================== + // NO DIRECTORY APPROVAL CALLBACKS SCENARIOS + // ===================================================================== + + describe('Without Directory Approval Callbacks', () => { + it('should work without directory approval callbacks (all paths need normal tool confirmation)', async () => { + const tool = createReadFileTool({ + fileSystemService, + directoryApproval: undefined, + }); + + // External path without approval callbacks - no override (normal tool flow) + const override = await tool.getApprovalOverride?.({ + file_path: '/external/project/file.ts', + }); + + // Without directoryApproval, isSessionApproved is never checked, so we fall through to + // returning a directory access request. Let me re-check the implementation... + // Actually looking at the code, when directoryApproval is undefined, isSessionApproved check + // is skipped (directoryApproval?.isSessionApproved), so it would still return the override. + // This is correct behavior - without approval callbacks, the override is still returned + // but onApprovalGranted won't do anything. + expect(override).not.toBeNull(); + }); + }); +}); diff --git a/dexto/packages/tools-filesystem/src/edit-file-tool.test.ts b/dexto/packages/tools-filesystem/src/edit-file-tool.test.ts new file mode 100644 index 00000000..9e97ce0f --- /dev/null +++ b/dexto/packages/tools-filesystem/src/edit-file-tool.test.ts @@ -0,0 +1,266 @@ +/** + * Edit File Tool Tests + * + * Tests for the edit_file tool including file modification detection. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import { createEditFileTool } from './edit-file-tool.js'; +import { FileSystemService } from './filesystem-service.js'; +import { ToolErrorCode } from '@dexto/core'; +import { DextoRuntimeError } from '@dexto/core'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +describe('edit_file tool', () => { + let mockLogger: ReturnType; + let tempDir: string; + let fileSystemService: FileSystemService; + + beforeEach(async () => { + mockLogger = createMockLogger(); + + // Create temp directory for testing + const rawTempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-edit-test-')); + tempDir = await fs.realpath(rawTempDir); + + fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: false, + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + vi.clearAllMocks(); + }); + + afterEach(async () => { + // Cleanup temp directory + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('File Modification Detection', () => { + it('should succeed when file is not modified between preview and execute', async () => { + const tool = createEditFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'hello world'); + + const toolCallId = 'test-call-123'; + const input = { + file_path: testFile, + old_string: 'world', + new_string: 'universe', + }; + + // Generate preview (stores hash) + const preview = await tool.generatePreview!(input, { toolCallId }); + expect(preview).toBeDefined(); + + // Execute without modifying file (should succeed) + const result = (await tool.execute(input, { toolCallId })) as { + success: boolean; + path: string; + }; + + expect(result.success).toBe(true); + expect(result.path).toBe(testFile); + + // Verify file was edited + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('hello universe'); + }); + + it('should fail when file is modified between preview and execute', async () => { + const tool = createEditFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'hello world'); + + const toolCallId = 'test-call-456'; + const input = { + file_path: testFile, + old_string: 'world', + new_string: 'universe', + }; + + // Generate preview (stores hash) + const preview = await tool.generatePreview!(input, { toolCallId }); + expect(preview).toBeDefined(); + + // Simulate user modifying the file externally (keep 'world' so edit would work without hash check) + await fs.writeFile(testFile, 'hello world - user added this'); + + // Execute should fail because file was modified + try { + await tool.execute(input, { toolCallId }); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe( + ToolErrorCode.FILE_MODIFIED_SINCE_PREVIEW + ); + } + + // Verify file was NOT modified by the tool (still has user's changes) + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('hello world - user added this'); + }); + + it('should detect file modification with correct error code', async () => { + const tool = createEditFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'hello world'); + + const toolCallId = 'test-call-789'; + const input = { + file_path: testFile, + old_string: 'world', + new_string: 'universe', + }; + + // Generate preview (stores hash) + await tool.generatePreview!(input, { toolCallId }); + + // Simulate user modifying the file externally + await fs.writeFile(testFile, 'hello world modified'); + + // Execute should fail with FILE_MODIFIED_SINCE_PREVIEW error + try { + await tool.execute(input, { toolCallId }); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe( + ToolErrorCode.FILE_MODIFIED_SINCE_PREVIEW + ); + expect((error as DextoRuntimeError).message).toContain( + 'modified since the preview' + ); + expect((error as DextoRuntimeError).message).toContain('read the file again'); + } + }); + + it('should work without toolCallId (no modification check)', async () => { + const tool = createEditFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'hello world'); + + const input = { + file_path: testFile, + old_string: 'world', + new_string: 'universe', + }; + + // Generate preview without toolCallId + await tool.generatePreview!(input, {}); + + // Modify file + await fs.writeFile(testFile, 'hello world changed'); + + // Execute without toolCallId - should NOT check for modifications + // (but will fail because old_string won't match the new content) + // This tests that the tool doesn't crash when toolCallId is missing + try { + await tool.execute(input, {}); + } catch (error) { + // Expected to fail because 'world' is no longer in the file + // But it should NOT be FILE_MODIFIED_SINCE_PREVIEW error + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).not.toBe( + ToolErrorCode.FILE_MODIFIED_SINCE_PREVIEW + ); + } + }); + + it('should clean up hash cache after successful execution', async () => { + const tool = createEditFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'hello world'); + + const toolCallId = 'test-call-cleanup'; + const input = { + file_path: testFile, + old_string: 'world', + new_string: 'universe', + }; + + // Generate preview + await tool.generatePreview!(input, { toolCallId }); + + // Execute successfully + await tool.execute(input, { toolCallId }); + + // Prepare for second edit + const input2 = { + file_path: testFile, + old_string: 'universe', + new_string: 'galaxy', + }; + + // Second execution with same toolCallId should work + // (hash should have been cleaned up, so no stale check) + await tool.generatePreview!(input2, { toolCallId }); + const result = (await tool.execute(input2, { toolCallId })) as { success: boolean }; + + expect(result.success).toBe(true); + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('hello galaxy'); + }); + + it('should clean up hash cache after failed execution', async () => { + const tool = createEditFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'hello world'); + + const toolCallId = 'test-call-fail-cleanup'; + const input = { + file_path: testFile, + old_string: 'world', + new_string: 'universe', + }; + + // Generate preview + await tool.generatePreview!(input, { toolCallId }); + + // Modify file to cause failure + await fs.writeFile(testFile, 'hello world modified'); + + // Execute should fail + try { + await tool.execute(input, { toolCallId }); + } catch { + // Expected + } + + // Reset file + await fs.writeFile(testFile, 'hello world'); + + // Next execution with same toolCallId should work + // (hash should have been cleaned up even after failure) + await tool.generatePreview!(input, { toolCallId }); + const result = (await tool.execute(input, { toolCallId })) as { success: boolean }; + + expect(result.success).toBe(true); + }); + }); +}); diff --git a/dexto/packages/tools-filesystem/src/edit-file-tool.ts b/dexto/packages/tools-filesystem/src/edit-file-tool.ts new file mode 100644 index 00000000..f01d7230 --- /dev/null +++ b/dexto/packages/tools-filesystem/src/edit-file-tool.ts @@ -0,0 +1,267 @@ +/** + * Edit File Tool + * + * Internal tool for editing files by replacing text (requires approval) + */ + +import * as path from 'node:path'; +import { createHash } from 'node:crypto'; +import { z } from 'zod'; +import { createPatch } from 'diff'; +import { InternalTool, ToolExecutionContext, ApprovalType } from '@dexto/core'; +import type { DiffDisplayData, ApprovalRequestDetails, ApprovalResponse } from '@dexto/core'; +import { ToolError } from '@dexto/core'; +import { ToolErrorCode } from '@dexto/core'; +import { DextoRuntimeError } from '@dexto/core'; +import type { FileToolOptions } from './file-tool-types.js'; +import { FileSystemErrorCode } from './error-codes.js'; + +/** + * Cache for content hashes between preview and execute phases. + * Keyed by toolCallId to ensure proper cleanup after execution. + * This prevents file corruption when user modifies file between preview and execute. + */ +const previewContentHashCache = new Map(); + +/** + * Compute SHA-256 hash of content for change detection + */ +function computeContentHash(content: string): string { + return createHash('sha256').update(content, 'utf8').digest('hex'); +} + +const EditFileInputSchema = z + .object({ + file_path: z.string().describe('Absolute path to the file to edit'), + old_string: z + .string() + .describe('Text to replace (must be unique unless replace_all is true)'), + new_string: z.string().describe('Replacement text'), + replace_all: z + .boolean() + .optional() + .default(false) + .describe('Replace all occurrences (default: false, requires unique match)'), + }) + .strict(); + +type EditFileInput = z.input; + +/** + * Generate diff preview without modifying the file + */ +function generateDiffPreview( + filePath: string, + originalContent: string, + newContent: string +): DiffDisplayData { + const unified = createPatch(filePath, originalContent, newContent, 'before', 'after', { + context: 3, + }); + const additions = (unified.match(/^\+[^+]/gm) || []).length; + const deletions = (unified.match(/^-[^-]/gm) || []).length; + + return { + type: 'diff', + unified, + filename: filePath, + additions, + deletions, + }; +} + +/** + * Create the edit_file internal tool with directory approval support + */ +export function createEditFileTool(options: FileToolOptions): InternalTool { + const { fileSystemService, directoryApproval } = options; + + // Store parent directory for use in onApprovalGranted callback + let pendingApprovalParentDir: string | undefined; + + return { + id: 'edit_file', + description: + 'Edit a file by replacing text. By default, old_string must be unique in the file (will error if found multiple times). Set replace_all=true to replace all occurrences. Automatically creates backup before editing. Requires approval. Returns success status, path, number of changes made, and backup path.', + inputSchema: EditFileInputSchema, + + /** + * Check if this edit operation needs directory access approval. + * Returns custom approval request if the file is outside allowed paths. + */ + getApprovalOverride: async (args: unknown): Promise => { + const { file_path } = args as EditFileInput; + if (!file_path) return null; + + // Check if path is within config-allowed paths (async for non-blocking symlink resolution) + const isAllowed = await fileSystemService.isPathWithinConfigAllowed(file_path); + if (isAllowed) { + return null; // Use normal tool confirmation + } + + // Check if directory is already session-approved + if (directoryApproval?.isSessionApproved(file_path)) { + return null; // Already approved, use normal flow + } + + // Need directory access approval + const absolutePath = path.resolve(file_path); + const parentDir = path.dirname(absolutePath); + pendingApprovalParentDir = parentDir; + + return { + type: ApprovalType.DIRECTORY_ACCESS, + metadata: { + path: absolutePath, + parentDir, + operation: 'edit', + toolName: 'edit_file', + }, + }; + }, + + /** + * Handle approved directory access - remember the directory for session + */ + onApprovalGranted: (response: ApprovalResponse): void => { + if (!directoryApproval || !pendingApprovalParentDir) return; + + // Check if user wants to remember the directory + // Use type assertion to access rememberDirectory since response.data is a union type + const data = response.data as { rememberDirectory?: boolean } | undefined; + const rememberDirectory = data?.rememberDirectory ?? false; + directoryApproval.addApproved( + pendingApprovalParentDir, + rememberDirectory ? 'session' : 'once' + ); + + // Clear pending state + pendingApprovalParentDir = undefined; + }, + + /** + * Generate preview for approval UI - shows diff without modifying file + * Throws ToolError.validationFailed() for validation errors (file not found, string not found) + * Stores content hash for change detection in execute phase. + */ + generatePreview: async (input: unknown, context?: ToolExecutionContext) => { + const { file_path, old_string, new_string, replace_all } = input as EditFileInput; + + try { + // Read current file content + const originalFile = await fileSystemService.readFile(file_path); + const originalContent = originalFile.content; + + // Store content hash for change detection in execute phase + if (context?.toolCallId) { + previewContentHashCache.set( + context.toolCallId, + computeContentHash(originalContent) + ); + } + + // Validate uniqueness constraint when replace_all is false + if (!replace_all) { + const occurrences = originalContent.split(old_string).length - 1; + if (occurrences > 1) { + throw ToolError.validationFailed( + 'edit_file', + `String found ${occurrences} times in file. Set replace_all=true to replace all, or provide more context to make old_string unique.`, + { file_path, occurrences } + ); + } + } + + // Compute what the new content would be + const newContent = replace_all + ? originalContent.split(old_string).join(new_string) + : originalContent.replace(old_string, new_string); + + // If no change, old_string was not found - throw validation error + if (originalContent === newContent) { + throw ToolError.validationFailed( + 'edit_file', + `String not found in file: "${old_string.slice(0, 50)}${old_string.length > 50 ? '...' : ''}"`, + { file_path, old_string_preview: old_string.slice(0, 100) } + ); + } + + return generateDiffPreview(file_path, originalContent, newContent); + } catch (error) { + // Re-throw validation errors as-is + if ( + error instanceof DextoRuntimeError && + error.code === ToolErrorCode.VALIDATION_FAILED + ) { + throw error; + } + // Convert filesystem errors (file not found, etc.) to validation errors + if (error instanceof DextoRuntimeError) { + throw ToolError.validationFailed('edit_file', error.message, { + file_path, + originalErrorCode: error.code, + }); + } + // Unexpected errors - return null to allow approval to proceed + return null; + } + }, + + execute: async (input: unknown, context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { file_path, old_string, new_string, replace_all } = input as EditFileInput; + + // Check if file was modified since preview (safety check) + // This prevents corrupting user edits made between preview approval and execution + if (context?.toolCallId && previewContentHashCache.has(context.toolCallId)) { + const expectedHash = previewContentHashCache.get(context.toolCallId)!; + previewContentHashCache.delete(context.toolCallId); // Clean up regardless of outcome + + // Read current content to verify it hasn't changed + let currentContent: string; + try { + const currentFile = await fileSystemService.readFile(file_path); + currentContent = currentFile.content; + } catch (error) { + // File was deleted between preview and execute - treat as modified + if ( + error instanceof DextoRuntimeError && + error.code === FileSystemErrorCode.FILE_NOT_FOUND + ) { + throw ToolError.fileModifiedSincePreview('edit_file', file_path); + } + throw error; + } + const currentHash = computeContentHash(currentContent); + + if (expectedHash !== currentHash) { + throw ToolError.fileModifiedSincePreview('edit_file', file_path); + } + } + + // Edit file using FileSystemService + // Backup behavior is controlled by config.enableBackups (default: false) + // editFile returns originalContent and newContent, eliminating extra file reads + const result = await fileSystemService.editFile(file_path, { + oldString: old_string, + newString: new_string, + replaceAll: replace_all, + }); + + // Generate display data using content returned from editFile + const _display = generateDiffPreview( + file_path, + result.originalContent, + result.newContent + ); + + return { + success: result.success, + path: result.path, + changes_count: result.changesCount, + ...(result.backupPath && { backup_path: result.backupPath }), + _display, + }; + }, + }; +} diff --git a/dexto/packages/tools-filesystem/src/error-codes.ts b/dexto/packages/tools-filesystem/src/error-codes.ts new file mode 100644 index 00000000..dd7f74c8 --- /dev/null +++ b/dexto/packages/tools-filesystem/src/error-codes.ts @@ -0,0 +1,44 @@ +/** + * FileSystem Service Error Codes + * + * Standardized error codes for file system operations + */ + +export enum FileSystemErrorCode { + // File not found errors + FILE_NOT_FOUND = 'FILESYSTEM_FILE_NOT_FOUND', + DIRECTORY_NOT_FOUND = 'FILESYSTEM_DIRECTORY_NOT_FOUND', + + // Permission errors + PERMISSION_DENIED = 'FILESYSTEM_PERMISSION_DENIED', + PATH_NOT_ALLOWED = 'FILESYSTEM_PATH_NOT_ALLOWED', + PATH_BLOCKED = 'FILESYSTEM_PATH_BLOCKED', + + // Validation errors + INVALID_PATH = 'FILESYSTEM_INVALID_PATH', + PATH_TRAVERSAL_DETECTED = 'FILESYSTEM_PATH_TRAVERSAL_DETECTED', + INVALID_FILE_EXTENSION = 'FILESYSTEM_INVALID_FILE_EXTENSION', + INVALID_ENCODING = 'FILESYSTEM_INVALID_ENCODING', + + // Size errors + FILE_TOO_LARGE = 'FILESYSTEM_FILE_TOO_LARGE', + TOO_MANY_RESULTS = 'FILESYSTEM_TOO_MANY_RESULTS', + + // Operation errors + READ_FAILED = 'FILESYSTEM_READ_FAILED', + WRITE_FAILED = 'FILESYSTEM_WRITE_FAILED', + BACKUP_FAILED = 'FILESYSTEM_BACKUP_FAILED', + EDIT_FAILED = 'FILESYSTEM_EDIT_FAILED', + STRING_NOT_UNIQUE = 'FILESYSTEM_STRING_NOT_UNIQUE', + STRING_NOT_FOUND = 'FILESYSTEM_STRING_NOT_FOUND', + + // Search errors + GLOB_FAILED = 'FILESYSTEM_GLOB_FAILED', + SEARCH_FAILED = 'FILESYSTEM_SEARCH_FAILED', + INVALID_PATTERN = 'FILESYSTEM_INVALID_PATTERN', + REGEX_TIMEOUT = 'FILESYSTEM_REGEX_TIMEOUT', + + // Configuration errors + INVALID_CONFIG = 'FILESYSTEM_INVALID_CONFIG', + SERVICE_NOT_INITIALIZED = 'FILESYSTEM_SERVICE_NOT_INITIALIZED', +} diff --git a/dexto/packages/tools-filesystem/src/errors.ts b/dexto/packages/tools-filesystem/src/errors.ts new file mode 100644 index 00000000..d9ae5d54 --- /dev/null +++ b/dexto/packages/tools-filesystem/src/errors.ts @@ -0,0 +1,324 @@ +/** + * FileSystem Service Errors + * + * Error classes for file system operations + */ + +import { DextoRuntimeError, ErrorType } from '@dexto/core'; + +/** Error scope for filesystem operations */ +const FILESYSTEM_SCOPE = 'filesystem'; +import { FileSystemErrorCode } from './error-codes.js'; + +export interface FileSystemErrorContext { + path?: string; + pattern?: string; + size?: number; + maxSize?: number; + encoding?: string; + operation?: string; +} + +/** + * Factory class for creating FileSystem-related errors + */ +export class FileSystemError { + private constructor() { + // Private constructor prevents instantiation + } + + /** + * File not found error + */ + static fileNotFound(path: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.FILE_NOT_FOUND, + FILESYSTEM_SCOPE, + ErrorType.NOT_FOUND, + `File not found: ${path}`, + { path } + ); + } + + /** + * Directory not found error + */ + static directoryNotFound(path: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.DIRECTORY_NOT_FOUND, + FILESYSTEM_SCOPE, + ErrorType.NOT_FOUND, + `Directory not found: ${path}`, + { path } + ); + } + + /** + * Permission denied error + */ + static permissionDenied(path: string, operation: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.PERMISSION_DENIED, + FILESYSTEM_SCOPE, + ErrorType.FORBIDDEN, + `Permission denied: cannot ${operation} ${path}`, + { path, operation } + ); + } + + /** + * Path not allowed error + */ + static pathNotAllowed(path: string, allowedPaths: string[]): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.PATH_NOT_ALLOWED, + FILESYSTEM_SCOPE, + ErrorType.USER, + `Path not allowed: ${path}. Must be within allowed paths: ${allowedPaths.join(', ')}`, + { path, allowedPaths }, + 'Ensure the path is within the configured allowed paths' + ); + } + + /** + * Path blocked error + */ + static pathBlocked(path: string, reason: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.PATH_BLOCKED, + FILESYSTEM_SCOPE, + ErrorType.FORBIDDEN, + `Path is blocked: ${path}. Reason: ${reason}`, + { path, reason } + ); + } + + /** + * Invalid path error + */ + static invalidPath(path: string, reason: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.INVALID_PATH, + FILESYSTEM_SCOPE, + ErrorType.USER, + `Invalid path: ${path}. ${reason}`, + { path, reason } + ); + } + + /** + * Path traversal detected + */ + static pathTraversal(path: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.PATH_TRAVERSAL_DETECTED, + FILESYSTEM_SCOPE, + ErrorType.FORBIDDEN, + `Path traversal detected in: ${path}`, + { path } + ); + } + + /** + * Invalid file extension error + */ + static invalidExtension(path: string, blockedExtensions: string[]): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.INVALID_FILE_EXTENSION, + FILESYSTEM_SCOPE, + ErrorType.USER, + `Invalid file extension: ${path}. Blocked extensions: ${blockedExtensions.join(', ')}`, + { path, blockedExtensions } + ); + } + + /** + * File too large error + */ + static fileTooLarge(path: string, size: number, maxSize: number): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.FILE_TOO_LARGE, + FILESYSTEM_SCOPE, + ErrorType.USER, + `File too large: ${path} (${size} bytes). Maximum allowed: ${maxSize} bytes`, + { path, size, maxSize } + ); + } + + /** + * Too many results error + */ + static tooManyResults(operation: string, count: number, maxResults: number): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.TOO_MANY_RESULTS, + FILESYSTEM_SCOPE, + ErrorType.USER, + `Too many results from ${operation}: ${count}. Maximum allowed: ${maxResults}`, + { operation, count, maxResults }, + 'Narrow your search pattern or increase maxResults limit' + ); + } + + /** + * Read operation failed + */ + static readFailed(path: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.READ_FAILED, + FILESYSTEM_SCOPE, + ErrorType.SYSTEM, + `Failed to read file: ${path}. ${cause}`, + { path, cause } + ); + } + + /** + * Write operation failed + */ + static writeFailed(path: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.WRITE_FAILED, + FILESYSTEM_SCOPE, + ErrorType.SYSTEM, + `Failed to write file: ${path}. ${cause}`, + { path, cause } + ); + } + + /** + * Backup creation failed + */ + static backupFailed(path: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.BACKUP_FAILED, + FILESYSTEM_SCOPE, + ErrorType.SYSTEM, + `Failed to create backup for: ${path}. ${cause}`, + { path, cause } + ); + } + + /** + * Edit operation failed + */ + static editFailed(path: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.EDIT_FAILED, + FILESYSTEM_SCOPE, + ErrorType.SYSTEM, + `Failed to edit file: ${path}. ${cause}`, + { path, cause } + ); + } + + /** + * String not unique error + */ + static stringNotUnique( + path: string, + searchString: string, + occurrences: number + ): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.STRING_NOT_UNIQUE, + FILESYSTEM_SCOPE, + ErrorType.USER, + `String is not unique in ${path}: "${searchString}" found ${occurrences} times. Use replaceAll=true or provide a more specific string.`, + { path, searchString, occurrences }, + 'Use replaceAll option or provide more context in the search string' + ); + } + + /** + * String not found error + */ + static stringNotFound(path: string, searchString: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.STRING_NOT_FOUND, + FILESYSTEM_SCOPE, + ErrorType.USER, + `String not found in ${path}: "${searchString}"`, + { path, searchString } + ); + } + + /** + * Glob operation failed + */ + static globFailed(pattern: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.GLOB_FAILED, + FILESYSTEM_SCOPE, + ErrorType.SYSTEM, + `Glob operation failed for pattern: ${pattern}. ${cause}`, + { pattern, cause } + ); + } + + /** + * Search operation failed + */ + static searchFailed(pattern: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.SEARCH_FAILED, + FILESYSTEM_SCOPE, + ErrorType.SYSTEM, + `Search operation failed for pattern: ${pattern}. ${cause}`, + { pattern, cause } + ); + } + + /** + * Invalid pattern error + */ + static invalidPattern(pattern: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.INVALID_PATTERN, + FILESYSTEM_SCOPE, + ErrorType.USER, + `Invalid pattern: ${pattern}. ${cause}`, + { pattern, cause } + ); + } + + /** + * Regex timeout error + */ + static regexTimeout(pattern: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.REGEX_TIMEOUT, + FILESYSTEM_SCOPE, + ErrorType.TIMEOUT, + `Regex operation timed out for pattern: ${pattern}`, + { pattern }, + 'Simplify your regex pattern or increase timeout' + ); + } + + /** + * Invalid configuration error + */ + static invalidConfig(reason: string): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.INVALID_CONFIG, + FILESYSTEM_SCOPE, + ErrorType.USER, + `Invalid FileSystem configuration: ${reason}`, + { reason } + ); + } + + /** + * Service not initialized error + */ + static notInitialized(): DextoRuntimeError { + return new DextoRuntimeError( + FileSystemErrorCode.SERVICE_NOT_INITIALIZED, + FILESYSTEM_SCOPE, + ErrorType.SYSTEM, + 'FileSystemService has not been initialized', + {}, + 'Initialize the FileSystemService before using it' + ); + } +} diff --git a/dexto/packages/tools-filesystem/src/file-tool-types.ts b/dexto/packages/tools-filesystem/src/file-tool-types.ts new file mode 100644 index 00000000..8c049b78 --- /dev/null +++ b/dexto/packages/tools-filesystem/src/file-tool-types.ts @@ -0,0 +1,45 @@ +/** + * File Tool Types + * + * Types shared between file tools for directory approval support. + */ + +import type { FileSystemService } from './filesystem-service.js'; + +/** + * Callbacks for directory access approval. + * Allows file tools to check and request approval for accessing paths + * outside the configured working directory. + */ +export interface DirectoryApprovalCallbacks { + /** + * Check if a path is within any session-approved directory. + * Used to determine if directory approval prompt is needed. + * @param filePath The file path to check (absolute or relative) + * @returns true if path is in a session-approved directory + */ + isSessionApproved: (filePath: string) => boolean; + + /** + * Add a directory to the approved list for this session. + * Called after user approves directory access. + * @param directory Absolute path to the directory to approve + * @param type 'session' (remembered) or 'once' (single use) + */ + addApproved: (directory: string, type: 'session' | 'once') => void; +} + +/** + * Options for creating file tools with directory approval support + */ +export interface FileToolOptions { + /** FileSystemService instance for file operations */ + fileSystemService: FileSystemService; + + /** + * Optional callbacks for directory approval. + * If provided, file tools can request approval for accessing paths + * outside the configured working directory. + */ + directoryApproval?: DirectoryApprovalCallbacks | undefined; +} diff --git a/dexto/packages/tools-filesystem/src/filesystem-service.test.ts b/dexto/packages/tools-filesystem/src/filesystem-service.test.ts new file mode 100644 index 00000000..9437385f --- /dev/null +++ b/dexto/packages/tools-filesystem/src/filesystem-service.test.ts @@ -0,0 +1,277 @@ +/** + * FileSystemService Tests + * + * Tests for the core filesystem service including backup behavior. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import { FileSystemService } from './filesystem-service.js'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +describe('FileSystemService', () => { + let mockLogger: ReturnType; + let tempDir: string; + + beforeEach(async () => { + mockLogger = createMockLogger(); + + // Create temp directory for testing + const rawTempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-fs-test-')); + tempDir = await fs.realpath(rawTempDir); + + vi.clearAllMocks(); + }); + + afterEach(async () => { + // Cleanup temp directory + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('Backup Behavior', () => { + describe('writeFile', () => { + it('should NOT create backup when enableBackups is false (default)', async () => { + const fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: false, + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + // Create initial file + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'original content'); + + // Write new content (should NOT create backup) + const result = await fileSystemService.writeFile(testFile, 'new content'); + + expect(result.success).toBe(true); + expect(result.backupPath).toBeUndefined(); + + // Verify backup directory doesn't exist or is empty + const backupDir = path.join(tempDir, '.dexto-backups'); + try { + const files = await fs.readdir(backupDir); + expect(files.length).toBe(0); + } catch { + // Backup dir doesn't exist, which is expected + } + }); + + it('should create backup when enableBackups is true', async () => { + const fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: true, + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + // Create initial file + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'original content'); + + // Write new content (should create backup) + const result = await fileSystemService.writeFile(testFile, 'new content'); + + expect(result.success).toBe(true); + expect(result.backupPath).toBeDefined(); + expect(result.backupPath).toContain('.dexto'); + expect(result.backupPath).toContain('backup'); + + // Verify backup file exists and contains original content + const backupContent = await fs.readFile(result.backupPath!, 'utf-8'); + expect(backupContent).toBe('original content'); + + // Verify new content was written + const newContent = await fs.readFile(testFile, 'utf-8'); + expect(newContent).toBe('new content'); + }); + + it('should NOT create backup for new files even when enableBackups is true', async () => { + const fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: true, + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + // Write to a new file (no backup needed since file doesn't exist) + const testFile = path.join(tempDir, 'new-file.txt'); + const result = await fileSystemService.writeFile(testFile, 'content', { + createDirs: true, + }); + + expect(result.success).toBe(true); + expect(result.backupPath).toBeUndefined(); + }); + + it('should respect per-call backup option over config', async () => { + const fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: false, // Config says no backups + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + // Create initial file + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'original content'); + + // Write with explicit backup: true (should override config) + const result = await fileSystemService.writeFile(testFile, 'new content', { + backup: true, + }); + + expect(result.success).toBe(true); + expect(result.backupPath).toBeDefined(); + }); + }); + + describe('editFile', () => { + it('should NOT create backup when enableBackups is false (default)', async () => { + const fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: false, + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + // Create initial file + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'hello world'); + + // Edit file (should NOT create backup) + const result = await fileSystemService.editFile(testFile, { + oldString: 'world', + newString: 'universe', + }); + + expect(result.success).toBe(true); + expect(result.backupPath).toBeUndefined(); + + // Verify content was changed + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('hello universe'); + }); + + it('should create backup when enableBackups is true', async () => { + const fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: true, + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + // Create initial file + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'hello world'); + + // Edit file (should create backup) + const result = await fileSystemService.editFile(testFile, { + oldString: 'world', + newString: 'universe', + }); + + expect(result.success).toBe(true); + expect(result.backupPath).toBeDefined(); + + // Verify backup contains original content + const backupContent = await fs.readFile(result.backupPath!, 'utf-8'); + expect(backupContent).toBe('hello world'); + + // Verify new content + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('hello universe'); + }); + + it('should respect per-call backup option over config', async () => { + const fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: false, // Config says no backups + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + // Create initial file + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'hello world'); + + // Edit with explicit backup: true (should override config) + const result = await fileSystemService.editFile( + testFile, + { + oldString: 'world', + newString: 'universe', + }, + { backup: true } + ); + + expect(result.success).toBe(true); + expect(result.backupPath).toBeDefined(); + }); + }); + }); +}); diff --git a/dexto/packages/tools-filesystem/src/filesystem-service.ts b/dexto/packages/tools-filesystem/src/filesystem-service.ts new file mode 100644 index 00000000..92a5a34a --- /dev/null +++ b/dexto/packages/tools-filesystem/src/filesystem-service.ts @@ -0,0 +1,666 @@ +/** + * FileSystem Service + * + * Secure file system operations for Dexto internal tools + */ + +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { glob } from 'glob'; +import safeRegex from 'safe-regex'; +import { getDextoPath, IDextoLogger, DextoLogComponent } from '@dexto/core'; +import { + FileSystemConfig, + FileContent, + ReadFileOptions, + GlobOptions, + GlobResult, + GrepOptions, + SearchResult, + SearchMatch, + WriteFileOptions, + WriteResult, + EditFileOptions, + EditResult, + EditOperation, + FileMetadata, + BufferEncoding, +} from './types.js'; +import { PathValidator } from './path-validator.js'; +import { FileSystemError } from './errors.js'; + +const DEFAULT_ENCODING: BufferEncoding = 'utf-8'; +const DEFAULT_MAX_RESULTS = 1000; +const DEFAULT_MAX_SEARCH_RESULTS = 100; + +/** + * FileSystemService - Handles all file system operations with security checks + * + * This service receives fully-validated configuration from the FileSystem Tools Provider. + * All defaults have been applied by the provider's schema, so the service trusts the config + * and uses it as-is without any fallback logic. + * + * TODO: Add tests for this class + * TODO: instantiate only when internal file tools are enabled to avoid file dependencies which won't work in serverless + */ +export class FileSystemService { + private config: FileSystemConfig; + private pathValidator: PathValidator; + private initialized: boolean = false; + private initPromise: Promise | null = null; + private logger: IDextoLogger; + + /** + * Create a new FileSystemService with validated configuration. + * + * @param config - Fully-validated configuration from provider schema. + * All required fields have values, defaults already applied. + * @param logger - Logger instance for this service + */ + constructor(config: FileSystemConfig, logger: IDextoLogger) { + // Config is already fully validated with defaults applied - just use it + this.config = config; + + this.logger = logger.createChild(DextoLogComponent.FILESYSTEM); + this.pathValidator = new PathValidator(this.config, this.logger); + } + + /** + * Get backup directory path (context-aware with optional override) + * TODO: Migrate to explicit configuration via CLI enrichment layer (per-agent paths) + */ + private getBackupDir(): string { + // Use custom path if provided (absolute), otherwise use context-aware default + return this.config.backupPath || getDextoPath('backups'); + } + + /** + * Get the effective working directory for file operations. + * Falls back to process.cwd() if not configured. + */ + getWorkingDirectory(): string { + return this.config.workingDirectory || process.cwd(); + } + + /** + * Initialize the service. + * Safe to call multiple times - subsequent calls return the same promise. + */ + initialize(): Promise { + if (this.initPromise) { + return this.initPromise; + } + + this.initPromise = this.doInitialize(); + return this.initPromise; + } + + /** + * Internal initialization logic. + */ + private async doInitialize(): Promise { + if (this.initialized) { + this.logger.debug('FileSystemService already initialized'); + return; + } + + // Create backup directory if backups are enabled + if (this.config.enableBackups) { + try { + const backupDir = this.getBackupDir(); + await fs.mkdir(backupDir, { recursive: true }); + this.logger.debug(`Backup directory created/verified: ${backupDir}`); + } catch (error) { + this.logger.warn( + `Failed to create backup directory: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + this.initialized = true; + this.logger.info('FileSystemService initialized successfully'); + } + + /** + * Ensure the service is initialized before use. + * Tools should call this at the start of their execute methods. + * Safe to call multiple times - will await the same initialization promise. + */ + async ensureInitialized(): Promise { + if (this.initialized) { + return; + } + await this.initialize(); + } + + /** + * Set a callback to check if a path is in an approved directory. + * This allows PathValidator to consult ApprovalManager without a direct dependency. + * + * @param checker Function that returns true if path is in an approved directory + */ + setDirectoryApprovalChecker(checker: (filePath: string) => boolean): void { + this.pathValidator.setDirectoryApprovalChecker(checker); + } + + /** + * Check if a file path is within the configured allowed paths (config only). + * This is used by file tools to determine if directory approval is needed. + * + * @param filePath The file path to check (can be relative or absolute) + * @returns true if the path is within config-allowed paths, false otherwise + */ + async isPathWithinConfigAllowed(filePath: string): Promise { + return this.pathValidator.isPathWithinAllowed(filePath); + } + + /** + * Read a file with validation and size limits + */ + async readFile(filePath: string, options: ReadFileOptions = {}): Promise { + await this.ensureInitialized(); + + // Validate path (async for non-blocking symlink resolution) + const validation = await this.pathValidator.validatePath(filePath); + if (!validation.isValid || !validation.normalizedPath) { + throw FileSystemError.invalidPath(filePath, validation.error || 'Unknown error'); + } + + const normalizedPath = validation.normalizedPath; + + // Check if file exists + try { + const stats = await fs.stat(normalizedPath); + + if (!stats.isFile()) { + throw FileSystemError.invalidPath(normalizedPath, 'Path is not a file'); + } + + // Check file size + if (stats.size > this.config.maxFileSize) { + throw FileSystemError.fileTooLarge( + normalizedPath, + stats.size, + this.config.maxFileSize + ); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + throw FileSystemError.fileNotFound(normalizedPath); + } + if ((error as NodeJS.ErrnoException).code === 'EACCES') { + throw FileSystemError.permissionDenied(normalizedPath, 'read'); + } + throw FileSystemError.readFailed( + normalizedPath, + error instanceof Error ? error.message : String(error) + ); + } + + // Read file + try { + const encoding = options.encoding || DEFAULT_ENCODING; + const content = await fs.readFile(normalizedPath, encoding); + const lines = content.split('\n'); + + // Handle offset (1-based per types) and limit + const limit = options.limit; + const offset1 = options.offset; // 1-based if provided + + let selectedLines: string[]; + let truncated = false; + + if ((offset1 && offset1 > 0) || limit !== undefined) { + const start = offset1 && offset1 > 0 ? Math.max(0, offset1 - 1) : 0; + const end = limit !== undefined ? start + limit : lines.length; + selectedLines = lines.slice(start, end); + truncated = end < lines.length; + } else { + selectedLines = lines; + } + + return { + content: selectedLines.join('\n'), + lines: selectedLines.length, + encoding, + truncated, + size: Buffer.byteLength(content, encoding), + }; + } catch (error) { + throw FileSystemError.readFailed( + normalizedPath, + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Find files matching a glob pattern + */ + async globFiles(pattern: string, options: GlobOptions = {}): Promise { + await this.ensureInitialized(); + + const cwd: string = options.cwd || this.config.workingDirectory || process.cwd(); + const maxResults = options.maxResults || DEFAULT_MAX_RESULTS; + + try { + // Execute glob search + const files = await glob(pattern, { + cwd, + absolute: true, + nodir: true, // Only files + follow: false, // Don't follow symlinks + }); + + // Validate each path and collect metadata + const validFiles: FileMetadata[] = []; + + for (const file of files) { + // Validate path (async for non-blocking symlink resolution) + const validation = await this.pathValidator.validatePath(file); + if (!validation.isValid || !validation.normalizedPath) { + this.logger.debug(`Skipping invalid path: ${file}`); + continue; + } + + // Get metadata if requested + if (options.includeMetadata !== false) { + try { + const stats = await fs.stat(validation.normalizedPath); + validFiles.push({ + path: validation.normalizedPath, + size: stats.size, + modified: stats.mtime, + isDirectory: stats.isDirectory(), + }); + } catch (error) { + this.logger.debug( + `Failed to stat file ${file}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } else { + validFiles.push({ + path: validation.normalizedPath, + size: 0, + modified: new Date(), + isDirectory: false, + }); + } + + // Check if we've reached the limit + if (validFiles.length >= maxResults) { + break; + } + } + + const limited = validFiles.length >= maxResults; + return { + files: validFiles, + truncated: limited, + totalFound: validFiles.length, + }; + } catch (error) { + throw FileSystemError.globFailed( + pattern, + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Search for content in files (grep-like functionality) + */ + async searchContent(pattern: string, options: GrepOptions = {}): Promise { + await this.ensureInitialized(); + + const searchPath: string = options.path || this.config.workingDirectory || process.cwd(); + const globPattern = options.glob || '**/*'; + const maxResults = options.maxResults || DEFAULT_MAX_SEARCH_RESULTS; + const contextLines = options.contextLines || 0; + + try { + // Validate regex pattern for ReDoS safety before creating RegExp + // See: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + if (!safeRegex(pattern)) { + throw FileSystemError.invalidPattern( + pattern, + 'Pattern may cause catastrophic backtracking (ReDoS). Please simplify the regex.' + ); + } + + const flags = options.caseInsensitive ? 'i' : ''; + const regex = new RegExp(pattern, flags); + + // Find files to search + const globResult = await this.globFiles(globPattern, { + cwd: searchPath, + maxResults: 10000, // Search more files, but limit results + }); + + const matches: SearchMatch[] = []; + let filesSearched = 0; + + for (const fileInfo of globResult.files) { + try { + // Read file + const fileContent = await this.readFile(fileInfo.path); + const lines = fileContent.content.split('\n'); + + filesSearched++; + + // Search for pattern in each line + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; // Safe: we're iterating within bounds + if (regex.test(line)) { + // Collect context lines if requested + let context: { before: string[]; after: string[] } | undefined; + + if (contextLines > 0) { + const before: string[] = []; + const after: string[] = []; + + for (let j = Math.max(0, i - contextLines); j < i; j++) { + before.push(lines[j]!); // Safe: j is within bounds + } + + for ( + let j = i + 1; + j < Math.min(lines.length, i + contextLines + 1); + j++ + ) { + after.push(lines[j]!); // Safe: j is within bounds + } + + context = { before, after }; + } + + matches.push({ + file: fileInfo.path, + lineNumber: i + 1, // 1-based line numbers + line, + ...(context !== undefined && { context }), + }); + + // Check if we've reached max results + if (matches.length >= maxResults) { + return { + matches, + totalMatches: matches.length, + truncated: true, + filesSearched, + }; + } + } + } + } catch (error) { + // Skip files that can't be read + this.logger.debug( + `Skipping file ${fileInfo.path}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + return { + matches, + totalMatches: matches.length, + truncated: false, + filesSearched, + }; + } catch (error) { + if (error instanceof Error && error.message.includes('Invalid regular expression')) { + throw FileSystemError.invalidPattern(pattern, 'Invalid regular expression syntax'); + } + throw FileSystemError.searchFailed( + pattern, + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Write content to a file + */ + async writeFile( + filePath: string, + content: string, + options: WriteFileOptions = {} + ): Promise { + await this.ensureInitialized(); + + // Validate path (async for non-blocking symlink resolution) + const validation = await this.pathValidator.validatePath(filePath); + if (!validation.isValid || !validation.normalizedPath) { + throw FileSystemError.invalidPath(filePath, validation.error || 'Unknown error'); + } + + const normalizedPath = validation.normalizedPath; + const encoding = options.encoding || DEFAULT_ENCODING; + + // Check if file exists for backup + let backupPath: string | undefined; + let fileExists = false; + + try { + await fs.access(normalizedPath); + fileExists = true; + } catch { + // File doesn't exist, which is fine + } + + // Create backup if file exists and backups are enabled + if (fileExists && (options.backup ?? this.config.enableBackups)) { + backupPath = await this.createBackup(normalizedPath); + } + + try { + // Create parent directories if needed + if (options.createDirs) { + const dir = path.dirname(normalizedPath); + await fs.mkdir(dir, { recursive: true }); + } + + // Write file + await fs.writeFile(normalizedPath, content, encoding); + + const bytesWritten = Buffer.byteLength(content, encoding); + + this.logger.debug(`File written: ${normalizedPath} (${bytesWritten} bytes)`); + + return { + success: true, + path: normalizedPath, + bytesWritten, + backupPath, + }; + } catch (error) { + throw FileSystemError.writeFailed( + normalizedPath, + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Edit a file by replacing text + */ + async editFile( + filePath: string, + operation: EditOperation, + options: EditFileOptions = {} + ): Promise { + await this.ensureInitialized(); + + // Validate path (async for non-blocking symlink resolution) + const validation = await this.pathValidator.validatePath(filePath); + if (!validation.isValid || !validation.normalizedPath) { + throw FileSystemError.invalidPath(filePath, validation.error || 'Unknown error'); + } + + const normalizedPath = validation.normalizedPath; + + // Read current file content + const fileContent = await this.readFile(normalizedPath); + const originalContent = fileContent.content; + + // Count occurrences of old string + const occurrences = ( + originalContent.match( + new RegExp(operation.oldString.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g') + ) || [] + ).length; + + if (occurrences === 0) { + throw FileSystemError.stringNotFound(normalizedPath, operation.oldString); + } + + if (!operation.replaceAll && occurrences > 1) { + throw FileSystemError.stringNotUnique(normalizedPath, operation.oldString, occurrences); + } + + // Create backup if enabled + let backupPath: string | undefined; + if (options.backup ?? this.config.enableBackups) { + backupPath = await this.createBackup(normalizedPath); + } + + try { + // Perform replacement + let newContent: string; + if (operation.replaceAll) { + newContent = originalContent.replace( + new RegExp(operation.oldString.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), + operation.newString + ); + } else { + newContent = originalContent.replace(operation.oldString, operation.newString); + } + + // Write updated content + await fs.writeFile(normalizedPath, newContent, options.encoding || DEFAULT_ENCODING); + + this.logger.debug(`File edited: ${normalizedPath} (${occurrences} replacements)`); + + return { + success: true, + path: normalizedPath, + changesCount: occurrences, + backupPath, + originalContent, + newContent, + }; + } catch (error) { + throw FileSystemError.editFailed( + normalizedPath, + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Create a backup of a file + */ + private async createBackup(filePath: string): Promise { + const backupDir = this.getBackupDir(); + + // Generate backup filename with timestamp + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const basename = path.basename(filePath); + const backupFilename = `${basename}.${timestamp}.backup`; + const backupPath = path.join(backupDir, backupFilename); + + try { + await fs.mkdir(backupDir, { recursive: true }); + await fs.copyFile(filePath, backupPath); + this.logger.debug(`Backup created: ${backupPath}`); + + // Clean up old backups after creating new one + await this.cleanupOldBackups(); + + return backupPath; + } catch (error) { + throw FileSystemError.backupFailed( + filePath, + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Clean up old backup files based on retention policy + */ + async cleanupOldBackups(): Promise { + if (!this.config.enableBackups) { + return 0; + } + + let backupDir: string; + try { + backupDir = this.getBackupDir(); + } catch (error) { + this.logger.warn( + `Failed to resolve backup directory: ${error instanceof Error ? error.message : String(error)}` + ); + return 0; + } + + try { + // Check if backup directory exists + await fs.access(backupDir); + } catch { + // Directory doesn't exist, nothing to clean + return 0; + } + + const cutoffDate = new Date( + Date.now() - this.config.backupRetentionDays * 24 * 60 * 60 * 1000 + ); + let deletedCount = 0; + + try { + const files = await fs.readdir(backupDir); + const backupFiles = files.filter((file) => file.endsWith('.backup')); + + for (const file of backupFiles) { + const filePath = path.join(backupDir, file); + try { + const stats = await fs.stat(filePath); + if (stats.mtime < cutoffDate) { + await fs.unlink(filePath); + deletedCount++; + this.logger.debug(`Cleaned up old backup: ${file}`); + } + } catch (error) { + this.logger.warn( + `Failed to process backup file ${file}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + if (deletedCount > 0) { + this.logger.info(`Backup cleanup: removed ${deletedCount} old backup files`); + } + + return deletedCount; + } catch (error) { + this.logger.warn( + `Failed to cleanup backup directory: ${error instanceof Error ? error.message : String(error)}` + ); + return 0; + } + } + + /** + * Get service configuration + */ + getConfig(): Readonly { + return { ...this.config }; + } + + /** + * Check if a path is allowed (async for non-blocking symlink resolution) + */ + async isPathAllowed(filePath: string): Promise { + const validation = await this.pathValidator.validatePath(filePath); + return validation.isValid; + } +} diff --git a/dexto/packages/tools-filesystem/src/glob-files-tool.ts b/dexto/packages/tools-filesystem/src/glob-files-tool.ts new file mode 100644 index 00000000..b7da64fc --- /dev/null +++ b/dexto/packages/tools-filesystem/src/glob-files-tool.ts @@ -0,0 +1,140 @@ +/** + * Glob Files Tool + * + * Internal tool for finding files using glob patterns + */ + +import * as path from 'node:path'; +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext, ApprovalType } from '@dexto/core'; +import type { SearchDisplayData, ApprovalRequestDetails, ApprovalResponse } from '@dexto/core'; +import type { FileToolOptions } from './file-tool-types.js'; + +const GlobFilesInputSchema = z + .object({ + pattern: z + .string() + .describe('Glob pattern to match files (e.g., "**/*.ts", "src/**/*.js")'), + path: z + .string() + .optional() + .describe('Base directory to search from (defaults to working directory)'), + max_results: z + .number() + .int() + .positive() + .optional() + .default(1000) + .describe('Maximum number of results to return (default: 1000)'), + }) + .strict(); + +type GlobFilesInput = z.input; + +/** + * Create the glob_files internal tool with directory approval support + */ +export function createGlobFilesTool(options: FileToolOptions): InternalTool { + const { fileSystemService, directoryApproval } = options; + + // Store search directory for use in onApprovalGranted callback + let pendingApprovalSearchDir: string | undefined; + + return { + id: 'glob_files', + description: + 'Find files matching a glob pattern. Supports standard glob syntax like **/*.js for recursive matches, *.ts for files in current directory, and src/**/*.tsx for nested paths. Returns array of file paths with metadata (size, modified date). Results are limited to allowed paths only.', + inputSchema: GlobFilesInputSchema, + + /** + * Check if this glob operation needs directory access approval. + * Returns custom approval request if the search directory is outside allowed paths. + */ + getApprovalOverride: async (args: unknown): Promise => { + const { path: searchPath } = args as GlobFilesInput; + + // Resolve the search directory using the same base the service uses + // This ensures approval decisions align with actual execution context + const baseDir = fileSystemService.getWorkingDirectory(); + const searchDir = path.resolve(baseDir, searchPath || '.'); + + // Check if path is within config-allowed paths + const isAllowed = await fileSystemService.isPathWithinConfigAllowed(searchDir); + if (isAllowed) { + return null; // Use normal tool confirmation + } + + // Check if directory is already session-approved + if (directoryApproval?.isSessionApproved(searchDir)) { + return null; // Already approved, use normal flow + } + + // Need directory access approval + pendingApprovalSearchDir = searchDir; + + return { + type: ApprovalType.DIRECTORY_ACCESS, + metadata: { + path: searchDir, + parentDir: searchDir, + operation: 'search', + toolName: 'glob_files', + }, + }; + }, + + /** + * Handle approved directory access - remember the directory for session + */ + onApprovalGranted: (response: ApprovalResponse): void => { + if (!directoryApproval || !pendingApprovalSearchDir) return; + + // Check if user wants to remember the directory + const data = response.data as { rememberDirectory?: boolean } | undefined; + const rememberDirectory = data?.rememberDirectory ?? false; + directoryApproval.addApproved( + pendingApprovalSearchDir, + rememberDirectory ? 'session' : 'once' + ); + + // Clear pending state + pendingApprovalSearchDir = undefined; + }, + + execute: async (input: unknown, _context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { pattern, path, max_results } = input as GlobFilesInput; + + // Search for files using FileSystemService + const result = await fileSystemService.globFiles(pattern, { + cwd: path, + maxResults: max_results, + includeMetadata: true, + }); + + // Build display data (reuse SearchDisplayData for file list) + const _display: SearchDisplayData = { + type: 'search', + pattern, + matches: result.files.map((file) => ({ + file: file.path, + line: 0, // No line number for glob + content: file.path, + })), + totalMatches: result.totalFound, + truncated: result.truncated, + }; + + return { + files: result.files.map((file) => ({ + path: file.path, + size: file.size, + modified: file.modified.toISOString(), + })), + total_found: result.totalFound, + truncated: result.truncated, + _display, + }; + }, + }; +} diff --git a/dexto/packages/tools-filesystem/src/grep-content-tool.ts b/dexto/packages/tools-filesystem/src/grep-content-tool.ts new file mode 100644 index 00000000..24d23e7c --- /dev/null +++ b/dexto/packages/tools-filesystem/src/grep-content-tool.ts @@ -0,0 +1,167 @@ +/** + * Grep Content Tool + * + * Internal tool for searching file contents using regex patterns + */ + +import * as path from 'node:path'; +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext, ApprovalType } from '@dexto/core'; +import type { SearchDisplayData, ApprovalRequestDetails, ApprovalResponse } from '@dexto/core'; +import type { FileToolOptions } from './file-tool-types.js'; + +const GrepContentInputSchema = z + .object({ + pattern: z.string().describe('Regular expression pattern to search for'), + path: z + .string() + .optional() + .describe('Directory to search in (defaults to working directory)'), + glob: z + .string() + .optional() + .describe('Glob pattern to filter files (e.g., "*.ts", "**/*.js")'), + context_lines: z + .number() + .int() + .min(0) + .optional() + .default(0) + .describe( + 'Number of context lines to include before and after each match (default: 0)' + ), + case_insensitive: z + .boolean() + .optional() + .default(false) + .describe('Perform case-insensitive search (default: false)'), + max_results: z + .number() + .int() + .positive() + .optional() + .default(100) + .describe('Maximum number of results to return (default: 100)'), + }) + .strict(); + +type GrepContentInput = z.input; + +/** + * Create the grep_content internal tool with directory approval support + */ +export function createGrepContentTool(options: FileToolOptions): InternalTool { + const { fileSystemService, directoryApproval } = options; + + // Store search directory for use in onApprovalGranted callback + let pendingApprovalSearchDir: string | undefined; + + return { + id: 'grep_content', + description: + 'Search for text patterns in files using regular expressions. Returns matching lines with file path, line number, and optional context lines. Use glob parameter to filter specific file types (e.g., "*.ts"). Supports case-insensitive search. Great for finding code patterns, function definitions, or specific text across multiple files.', + inputSchema: GrepContentInputSchema, + + /** + * Check if this grep operation needs directory access approval. + * Returns custom approval request if the search directory is outside allowed paths. + */ + getApprovalOverride: async (args: unknown): Promise => { + const { path: searchPath } = args as GrepContentInput; + + // Resolve the search directory (use cwd if not specified) + const searchDir = path.resolve(searchPath || process.cwd()); + + // Check if path is within config-allowed paths + const isAllowed = await fileSystemService.isPathWithinConfigAllowed(searchDir); + if (isAllowed) { + return null; // Use normal tool confirmation + } + + // Check if directory is already session-approved + if (directoryApproval?.isSessionApproved(searchDir)) { + return null; // Already approved, use normal flow + } + + // Need directory access approval + pendingApprovalSearchDir = searchDir; + + return { + type: ApprovalType.DIRECTORY_ACCESS, + metadata: { + path: searchDir, + parentDir: searchDir, + operation: 'search', + toolName: 'grep_content', + }, + }; + }, + + /** + * Handle approved directory access - remember the directory for session + */ + onApprovalGranted: (response: ApprovalResponse): void => { + if (!directoryApproval || !pendingApprovalSearchDir) return; + + // Check if user wants to remember the directory + const data = response.data as { rememberDirectory?: boolean } | undefined; + const rememberDirectory = data?.rememberDirectory ?? false; + directoryApproval.addApproved( + pendingApprovalSearchDir, + rememberDirectory ? 'session' : 'once' + ); + + // Clear pending state + pendingApprovalSearchDir = undefined; + }, + + execute: async (input: unknown, _context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { pattern, path, glob, context_lines, case_insensitive, max_results } = + input as GrepContentInput; + + // Search for content using FileSystemService + const result = await fileSystemService.searchContent(pattern, { + path, + glob, + contextLines: context_lines, + caseInsensitive: case_insensitive, + maxResults: max_results, + }); + + // Build display data + const _display: SearchDisplayData = { + type: 'search', + pattern, + matches: result.matches.map((match) => ({ + file: match.file, + line: match.lineNumber, + content: match.line, + ...(match.context && { + context: [...match.context.before, ...match.context.after], + }), + })), + totalMatches: result.totalMatches, + truncated: result.truncated, + }; + + return { + matches: result.matches.map((match) => ({ + file: match.file, + line_number: match.lineNumber, + line: match.line, + ...(match.context && { + context: { + before: match.context.before, + after: match.context.after, + }, + }), + })), + total_matches: result.totalMatches, + files_searched: result.filesSearched, + truncated: result.truncated, + _display, + }; + }, + }; +} diff --git a/dexto/packages/tools-filesystem/src/index.ts b/dexto/packages/tools-filesystem/src/index.ts new file mode 100644 index 00000000..c6b8bc91 --- /dev/null +++ b/dexto/packages/tools-filesystem/src/index.ts @@ -0,0 +1,43 @@ +/** + * @dexto/tools-filesystem + * + * FileSystem tools provider for Dexto agents. + * Provides file operation tools: read, write, edit, glob, grep. + */ + +// Main provider export +export { fileSystemToolsProvider } from './tool-provider.js'; +export type { FileToolOptions, DirectoryApprovalCallbacks } from './file-tool-types.js'; + +// Service and utilities (for advanced use cases) +export { FileSystemService } from './filesystem-service.js'; +export { PathValidator } from './path-validator.js'; +export { FileSystemError } from './errors.js'; +export { FileSystemErrorCode } from './error-codes.js'; + +// Types +export type { + FileSystemConfig, + FileContent, + ReadFileOptions, + GlobOptions, + GlobResult, + GrepOptions, + SearchResult, + SearchMatch, + WriteFileOptions, + WriteResult, + EditFileOptions, + EditResult, + EditOperation, + FileMetadata, + PathValidation, + BufferEncoding, +} from './types.js'; + +// Tool implementations (for custom integrations) +export { createReadFileTool } from './read-file-tool.js'; +export { createWriteFileTool } from './write-file-tool.js'; +export { createEditFileTool } from './edit-file-tool.js'; +export { createGlobFilesTool } from './glob-files-tool.js'; +export { createGrepContentTool } from './grep-content-tool.js'; diff --git a/dexto/packages/tools-filesystem/src/path-validator.test.ts b/dexto/packages/tools-filesystem/src/path-validator.test.ts new file mode 100644 index 00000000..a8ef6912 --- /dev/null +++ b/dexto/packages/tools-filesystem/src/path-validator.test.ts @@ -0,0 +1,517 @@ +/** + * PathValidator Unit Tests + * + * Tests for path validation, security checks, and allowed path logic. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { PathValidator, DirectoryApprovalChecker } from './path-validator.js'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +describe('PathValidator', () => { + let mockLogger: ReturnType; + + beforeEach(() => { + mockLogger = createMockLogger(); + vi.clearAllMocks(); + }); + + describe('validatePath', () => { + describe('Empty and Invalid Paths', () => { + it('should reject empty path', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath(''); + expect(result.isValid).toBe(false); + expect(result.error).toBe('Path cannot be empty'); + }); + + it('should reject whitespace-only path', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath(' '); + expect(result.isValid).toBe(false); + expect(result.error).toBe('Path cannot be empty'); + }); + }); + + describe('Allowed Paths', () => { + it('should allow paths within allowed directories', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath('/home/user/project/src/file.ts'); + expect(result.isValid).toBe(true); + expect(result.normalizedPath).toBeDefined(); + }); + + it('should allow relative paths within working directory', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath('src/file.ts'); + expect(result.isValid).toBe(true); + }); + + it('should reject paths outside allowed directories', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath('/external/project/file.ts'); + expect(result.isValid).toBe(false); + expect(result.error).toContain('not within allowed paths'); + }); + + it('should allow all paths when allowedPaths is empty', async () => { + const validator = new PathValidator( + { + allowedPaths: [], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath('/anywhere/file.ts'); + expect(result.isValid).toBe(true); + }); + }); + + describe('Path Traversal Detection', () => { + it('should reject path traversal attempts', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath( + '/home/user/project/../../../etc/passwd' + ); + expect(result.isValid).toBe(false); + expect(result.error).toBe('Path traversal detected'); + }); + }); + + describe('Blocked Paths', () => { + it('should reject paths in blocked directories', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: ['.git', 'node_modules'], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath('/home/user/project/.git/config'); + expect(result.isValid).toBe(false); + expect(result.error).toContain('blocked'); + }); + + it('should reject paths in node_modules', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: ['node_modules'], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath( + '/home/user/project/node_modules/lodash/index.js' + ); + expect(result.isValid).toBe(false); + expect(result.error).toContain('blocked'); + }); + }); + + describe('Blocked Extensions', () => { + it('should reject files with blocked extensions', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: ['.exe', '.dll'], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath('/home/user/project/malware.exe'); + expect(result.isValid).toBe(false); + expect(result.error).toContain('.exe is not allowed'); + }); + + it('should handle extensions without leading dot', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: ['exe', 'dll'], // No leading dot + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath('/home/user/project/file.exe'); + expect(result.isValid).toBe(false); + }); + + it('should be case-insensitive for extensions', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: ['.exe'], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const result = await validator.validatePath('/home/user/project/file.EXE'); + expect(result.isValid).toBe(false); + }); + }); + + describe('Directory Approval Checker Integration', () => { + it('should consult approval checker for external paths', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + // Without approval checker, external path should fail + let result = await validator.validatePath('/external/project/file.ts'); + expect(result.isValid).toBe(false); + + // Set approval checker that approves external path + const approvalChecker: DirectoryApprovalChecker = (filePath) => { + return filePath.startsWith('/external/project'); + }; + validator.setDirectoryApprovalChecker(approvalChecker); + + // Now external path should succeed + result = await validator.validatePath('/external/project/file.ts'); + expect(result.isValid).toBe(true); + }); + + it('should not use approval checker for config-allowed paths', async () => { + const approvalChecker = vi.fn().mockReturnValue(false); + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + validator.setDirectoryApprovalChecker(approvalChecker); + + // Config-allowed path should not invoke checker + const result = await validator.validatePath('/home/user/project/src/file.ts'); + expect(result.isValid).toBe(true); + expect(approvalChecker).not.toHaveBeenCalled(); + }); + }); + }); + + describe('isPathWithinAllowed', () => { + it('should return true for paths within config-allowed directories', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + expect(await validator.isPathWithinAllowed('/home/user/project/src/file.ts')).toBe( + true + ); + expect( + await validator.isPathWithinAllowed('/home/user/project/deep/nested/file.ts') + ).toBe(true); + }); + + it('should return false for paths outside config-allowed directories', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + expect(await validator.isPathWithinAllowed('/external/project/file.ts')).toBe(false); + expect(await validator.isPathWithinAllowed('/home/user/other/file.ts')).toBe(false); + }); + + it('should NOT consult approval checker (used for prompting decisions)', async () => { + const approvalChecker = vi.fn().mockReturnValue(true); + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + validator.setDirectoryApprovalChecker(approvalChecker); + + // Even with approval checker that returns true, isPathWithinAllowed should return false + // for external paths (it only checks config paths, not approval checker) + expect(await validator.isPathWithinAllowed('/external/project/file.ts')).toBe(false); + expect(approvalChecker).not.toHaveBeenCalled(); + }); + + it('should return false for empty path', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + expect(await validator.isPathWithinAllowed('')).toBe(false); + expect(await validator.isPathWithinAllowed(' ')).toBe(false); + }); + + it('should return true when allowedPaths is empty (all paths allowed)', async () => { + const validator = new PathValidator( + { + allowedPaths: [], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + expect(await validator.isPathWithinAllowed('/anywhere/file.ts')).toBe(true); + }); + }); + + describe('Path Containment (Parent Directory Coverage)', () => { + it('should recognize that approving parent covers child paths', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/external/sub'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + // Child path of allowed directory + expect(await validator.isPathWithinAllowed('/external/sub/deep/nested/file.ts')).toBe( + true + ); + }); + + it('should not allow sibling directories', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/external/sub'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + // /external/other is sibling, not child + expect(await validator.isPathWithinAllowed('/external/other/file.ts')).toBe(false); + }); + + it('should not allow parent directories when child is approved', async () => { + const validator = new PathValidator( + { + allowedPaths: ['/external/sub/deep'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + // /external/sub is parent, should not be allowed + expect(await validator.isPathWithinAllowed('/external/sub/file.ts')).toBe(false); + }); + }); + + describe('getAllowedPaths and getBlockedPaths', () => { + it('should return normalized allowed paths', () => { + const validator = new PathValidator( + { + allowedPaths: ['.', './src'], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const allowedPaths = validator.getAllowedPaths(); + expect(allowedPaths).toHaveLength(2); + expect(allowedPaths[0]).toBe('/home/user/project'); + expect(allowedPaths[1]).toBe('/home/user/project/src'); + }); + + it('should return blocked paths', () => { + const validator = new PathValidator( + { + allowedPaths: ['/home/user/project'], + blockedPaths: ['.git', 'node_modules'], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + enableBackups: false, + backupRetentionDays: 7, + workingDirectory: '/home/user/project', + }, + mockLogger as any + ); + + const blockedPaths = validator.getBlockedPaths(); + expect(blockedPaths).toContain('.git'); + expect(blockedPaths).toContain('node_modules'); + }); + }); +}); diff --git a/dexto/packages/tools-filesystem/src/path-validator.ts b/dexto/packages/tools-filesystem/src/path-validator.ts new file mode 100644 index 00000000..037c5e93 --- /dev/null +++ b/dexto/packages/tools-filesystem/src/path-validator.ts @@ -0,0 +1,307 @@ +/** + * Path Validator + * + * Security-focused path validation for file system operations + */ + +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import { FileSystemConfig, PathValidation } from './types.js'; +import type { IDextoLogger } from '@dexto/core'; + +/** + * Callback type for checking if a path is in an approved directory. + * Used to consult ApprovalManager without creating a direct dependency. + */ +export type DirectoryApprovalChecker = (filePath: string) => boolean; + +/** + * PathValidator - Validates file paths for security and policy compliance + * + * Security checks: + * 1. Path traversal detection (../, symbolic links) + * 2. Allowed paths enforcement (whitelist + approved directories) + * 3. Blocked paths detection (blacklist) + * 4. File extension restrictions + * 5. Absolute path normalization + * + * PathValidator can optionally consult an external approval checker (e.g., ApprovalManager) + * to determine if paths outside the config's allowed paths are accessible. + */ +export class PathValidator { + private config: FileSystemConfig; + private normalizedAllowedPaths: string[]; + private normalizedBlockedPaths: string[]; + private normalizedBlockedExtensions: string[]; + private logger: IDextoLogger; + private directoryApprovalChecker: DirectoryApprovalChecker | undefined; + + constructor(config: FileSystemConfig, logger: IDextoLogger) { + this.config = config; + this.logger = logger; + + // Normalize allowed paths to absolute paths + const workingDir = config.workingDirectory || process.cwd(); + this.normalizedAllowedPaths = config.allowedPaths.map((p) => path.resolve(workingDir, p)); + + // Normalize blocked paths + this.normalizedBlockedPaths = config.blockedPaths.map((p) => path.normalize(p)); + + // Normalize blocked extensions: ensure leading dot and lowercase + this.normalizedBlockedExtensions = (config.blockedExtensions || []).map((ext) => { + const e = ext.startsWith('.') ? ext : `.${ext}`; + return e.toLowerCase(); + }); + + this.logger.debug( + `PathValidator initialized with ${this.normalizedAllowedPaths.length} allowed paths` + ); + } + + /** + * Set a callback to check if a path is in an approved directory. + * This allows PathValidator to consult ApprovalManager without a direct dependency. + * + * @param checker Function that returns true if path is in an approved directory + */ + setDirectoryApprovalChecker(checker: DirectoryApprovalChecker): void { + this.directoryApprovalChecker = checker; + this.logger.debug('Directory approval checker configured'); + } + + /** + * Validate a file path for security and policy compliance + */ + async validatePath(filePath: string): Promise { + // 1. Check for empty path + if (!filePath || filePath.trim() === '') { + return { + isValid: false, + error: 'Path cannot be empty', + }; + } + + // 2. Normalize the path to absolute + const workingDir = this.config.workingDirectory || process.cwd(); + let normalizedPath: string; + + try { + // Handle both absolute and relative paths + normalizedPath = path.isAbsolute(filePath) + ? path.resolve(filePath) + : path.resolve(workingDir, filePath); + + // Canonicalize to handle symlinks and resolve real paths (async, non-blocking) + try { + normalizedPath = await fs.realpath(normalizedPath); + } catch { + // If the path doesn't exist yet (e.g., writes), fallback to the resolved path + // Policy checks continue to use normalizedPath + } + } catch (error) { + return { + isValid: false, + error: `Failed to normalize path: ${error instanceof Error ? error.message : String(error)}`, + }; + } + + // 3. Check for path traversal attempts + if (this.hasPathTraversal(filePath, normalizedPath)) { + return { + isValid: false, + error: 'Path traversal detected', + }; + } + + // 4. Check if path is within allowed paths + if (!this.isPathAllowed(normalizedPath)) { + return { + isValid: false, + error: `Path is not within allowed paths. Allowed: ${this.normalizedAllowedPaths.join(', ')}`, + }; + } + + // 5. Check if path is blocked + const blockedReason = this.isPathBlocked(normalizedPath); + if (blockedReason) { + return { + isValid: false, + error: `Path is blocked: ${blockedReason}`, + }; + } + + // 6. Check file extension if applicable + const ext = path.extname(normalizedPath).toLowerCase(); + if (ext && this.normalizedBlockedExtensions.includes(ext)) { + return { + isValid: false, + error: `File extension ${ext} is not allowed`, + }; + } + + return { + isValid: true, + normalizedPath, + }; + } + + /** + * Check if path contains traversal attempts + */ + private hasPathTraversal(originalPath: string, normalizedPath: string): boolean { + // Check for ../ patterns in original path + if (originalPath.includes('../') || originalPath.includes('..\\')) { + // Verify the normalized path still escapes allowed boundaries + const workingDir = this.config.workingDirectory || process.cwd(); + const relative = path.relative(workingDir, normalizedPath); + if (relative.startsWith('..')) { + return true; + } + } + return false; + } + + /** + * Check if path is within allowed paths (whitelist check) + * Also consults the directory approval checker if configured. + * Uses the sync version since the path is already normalized at this point. + */ + private isPathAllowed(normalizedPath: string): boolean { + return this.isPathAllowedSync(normalizedPath); + } + + /** + * Check if path matches blocked patterns (blacklist check) + */ + private isPathBlocked(normalizedPath: string): string | null { + const roots = + this.normalizedAllowedPaths.length > 0 + ? this.normalizedAllowedPaths + : [this.config.workingDirectory || process.cwd()]; + + for (const blocked of this.normalizedBlockedPaths) { + for (const root of roots) { + // Resolve blocked relative to each allowed root unless already absolute + const blockedFull = path.isAbsolute(blocked) + ? path.normalize(blocked) + : path.resolve(root, blocked); + // Segment-aware prefix check + if ( + normalizedPath === blockedFull || + normalizedPath.startsWith(blockedFull + path.sep) + ) { + return `Within blocked directory: ${blocked}`; + } + } + } + + return null; + } + + /** + * Quick check if a path is allowed (for internal use) + * Note: This assumes the path is already normalized/canonicalized + */ + isPathAllowedQuick(normalizedPath: string): boolean { + return this.isPathAllowedSync(normalizedPath) && !this.isPathBlocked(normalizedPath); + } + + /** + * Synchronous path allowed check (for already-normalized paths) + * This is used internally when we already have a canonicalized path + */ + private isPathAllowedSync(normalizedPath: string): boolean { + // Empty allowedPaths means all paths are allowed + if (this.normalizedAllowedPaths.length === 0) { + return true; + } + + // Check if path is within any config-allowed path + const isInConfigPaths = this.normalizedAllowedPaths.some((allowedPath) => { + const relative = path.relative(allowedPath, normalizedPath); + // Path is allowed if it doesn't escape the allowed directory + return !relative.startsWith('..') && !path.isAbsolute(relative); + }); + + if (isInConfigPaths) { + return true; + } + + // Fallback: check ApprovalManager via callback (includes working dir + approved dirs) + if (this.directoryApprovalChecker) { + return this.directoryApprovalChecker(normalizedPath); + } + + return false; + } + + /** + * Check if a file path is within the configured allowed paths (from config only). + * This method does NOT consult ApprovalManager - it only checks the static config paths. + * + * This is used by file tools to determine if a path needs directory approval. + * Paths within config-allowed directories don't need directory approval prompts. + * + * @param filePath The file path to check (can be relative or absolute) + * @returns true if the path is within config-allowed paths, false otherwise + */ + async isPathWithinAllowed(filePath: string): Promise { + if (!filePath || filePath.trim() === '') { + return false; + } + + // Normalize the path to absolute + const workingDir = this.config.workingDirectory || process.cwd(); + let normalizedPath: string; + + try { + normalizedPath = path.isAbsolute(filePath) + ? path.resolve(filePath) + : path.resolve(workingDir, filePath); + + // Try to resolve symlinks for existing files (async, non-blocking) + try { + normalizedPath = await fs.realpath(normalizedPath); + } catch { + // Path doesn't exist yet, use resolved path + } + } catch { + // Failed to normalize, treat as not within allowed + return false; + } + + // Only check config paths - do NOT consult approval checker here + // This method is used for prompting decisions, not execution decisions + return this.isInConfigAllowedPaths(normalizedPath); + } + + /** + * Check if path is within config-allowed paths only (no approval checker). + * Used for prompting decisions. + */ + private isInConfigAllowedPaths(normalizedPath: string): boolean { + // Empty allowedPaths means all paths are allowed + if (this.normalizedAllowedPaths.length === 0) { + return true; + } + + return this.normalizedAllowedPaths.some((allowedPath) => { + const relative = path.relative(allowedPath, normalizedPath); + return !relative.startsWith('..') && !path.isAbsolute(relative); + }); + } + + /** + * Get normalized allowed paths + */ + getAllowedPaths(): string[] { + return [...this.normalizedAllowedPaths]; + } + + /** + * Get blocked paths + */ + getBlockedPaths(): string[] { + return [...this.normalizedBlockedPaths]; + } +} diff --git a/dexto/packages/tools-filesystem/src/read-file-tool.ts b/dexto/packages/tools-filesystem/src/read-file-tool.ts new file mode 100644 index 00000000..8a056345 --- /dev/null +++ b/dexto/packages/tools-filesystem/src/read-file-tool.ts @@ -0,0 +1,132 @@ +/** + * Read File Tool + * + * Internal tool for reading file contents with size limits and pagination + */ + +import * as path from 'node:path'; +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext, ApprovalType } from '@dexto/core'; +import type { FileDisplayData, ApprovalRequestDetails, ApprovalResponse } from '@dexto/core'; +import type { FileToolOptions } from './file-tool-types.js'; + +const ReadFileInputSchema = z + .object({ + file_path: z.string().describe('Absolute path to the file to read'), + limit: z + .number() + .int() + .positive() + .optional() + .describe('Maximum number of lines to read (optional)'), + offset: z + .number() + .int() + .min(1) + .optional() + .describe('Starting line number (1-based, optional)'), + }) + .strict(); + +type ReadFileInput = z.input; + +/** + * Create the read_file internal tool with directory approval support + */ +export function createReadFileTool(options: FileToolOptions): InternalTool { + const { fileSystemService, directoryApproval } = options; + + // Store parent directory for use in onApprovalGranted callback + let pendingApprovalParentDir: string | undefined; + + return { + id: 'read_file', + description: + 'Read the contents of a file with optional pagination. Returns file content, line count, encoding, and whether the output was truncated. Use limit and offset parameters for large files to read specific sections. This tool is for reading files within allowed paths only.', + inputSchema: ReadFileInputSchema, + + /** + * Check if this read operation needs directory access approval. + * Returns custom approval request if the file is outside allowed paths. + */ + getApprovalOverride: async (args: unknown): Promise => { + const { file_path } = args as ReadFileInput; + if (!file_path) return null; + + // Check if path is within config-allowed paths (async for non-blocking symlink resolution) + const isAllowed = await fileSystemService.isPathWithinConfigAllowed(file_path); + if (isAllowed) { + return null; // Use normal tool confirmation + } + + // Check if directory is already session-approved + if (directoryApproval?.isSessionApproved(file_path)) { + return null; // Already approved, use normal flow + } + + // Need directory access approval + const absolutePath = path.resolve(file_path); + const parentDir = path.dirname(absolutePath); + pendingApprovalParentDir = parentDir; + + return { + type: ApprovalType.DIRECTORY_ACCESS, + metadata: { + path: absolutePath, + parentDir, + operation: 'read', + toolName: 'read_file', + }, + }; + }, + + /** + * Handle approved directory access - remember the directory for session + */ + onApprovalGranted: (response: ApprovalResponse): void => { + if (!directoryApproval || !pendingApprovalParentDir) return; + + // Check if user wants to remember the directory + // Use type assertion to access rememberDirectory since response.data is a union type + const data = response.data as { rememberDirectory?: boolean } | undefined; + const rememberDirectory = data?.rememberDirectory ?? false; + directoryApproval.addApproved( + pendingApprovalParentDir, + rememberDirectory ? 'session' : 'once' + ); + + // Clear pending state + pendingApprovalParentDir = undefined; + }, + + execute: async (input: unknown, _context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { file_path, limit, offset } = input as ReadFileInput; + + // Read file using FileSystemService + const result = await fileSystemService.readFile(file_path, { + limit, + offset, + }); + + // Build display data + const _display: FileDisplayData = { + type: 'file', + path: file_path, + operation: 'read', + size: result.size, + lineCount: result.lines, + }; + + return { + content: result.content, + lines: result.lines, + encoding: result.encoding, + truncated: result.truncated, + size: result.size, + ...(result.mimeType && { mimeType: result.mimeType }), + _display, + }; + }, + }; +} diff --git a/dexto/packages/tools-filesystem/src/tool-provider.ts b/dexto/packages/tools-filesystem/src/tool-provider.ts new file mode 100644 index 00000000..0cff056e --- /dev/null +++ b/dexto/packages/tools-filesystem/src/tool-provider.ts @@ -0,0 +1,215 @@ +/** + * FileSystem Tools Provider + * + * Provides file operation tools by wrapping FileSystemService. + * When registered, the provider initializes FileSystemService and creates tools + * for file operations (read, write, edit, glob, grep). + */ + +import { z } from 'zod'; +import type { CustomToolProvider, ToolCreationContext } from '@dexto/core'; +import type { InternalTool } from '@dexto/core'; +import { FileSystemService } from './filesystem-service.js'; +import { createReadFileTool } from './read-file-tool.js'; +import { createWriteFileTool } from './write-file-tool.js'; +import { createEditFileTool } from './edit-file-tool.js'; +import { createGlobFilesTool } from './glob-files-tool.js'; +import { createGrepContentTool } from './grep-content-tool.js'; +import type { FileToolOptions } from './file-tool-types.js'; + +// Re-export for convenience +export type { FileToolOptions } from './file-tool-types.js'; + +/** + * Default configuration constants for FileSystem tools. + * These are the SINGLE SOURCE OF TRUTH for all default values. + */ +const DEFAULT_ALLOWED_PATHS = ['.']; +const DEFAULT_BLOCKED_PATHS = ['.git', 'node_modules/.bin', '.env']; +const DEFAULT_BLOCKED_EXTENSIONS = ['.exe', '.dll', '.so']; +const DEFAULT_MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB +const DEFAULT_ENABLE_BACKUPS = false; +const DEFAULT_BACKUP_RETENTION_DAYS = 7; + +/** + * Available filesystem tool names for enabledTools configuration. + */ +const FILESYSTEM_TOOL_NAMES = [ + 'read_file', + 'write_file', + 'edit_file', + 'glob_files', + 'grep_content', +] as const; +type FileSystemToolName = (typeof FILESYSTEM_TOOL_NAMES)[number]; + +/** + * Configuration schema for FileSystem tools provider. + * + * This is the SINGLE SOURCE OF TRUTH for all configuration: + * - Validation rules + * - Default values (using constants above) + * - Documentation + * - Type definitions + * + * Services receive fully-validated config from this schema and use it as-is, + * with no additional defaults or fallbacks needed. + */ +const FileSystemToolsConfigSchema = z + .object({ + type: z.literal('filesystem-tools'), + allowedPaths: z + .array(z.string()) + .default(DEFAULT_ALLOWED_PATHS) + .describe('List of allowed base paths for file operations'), + blockedPaths: z + .array(z.string()) + .default(DEFAULT_BLOCKED_PATHS) + .describe('List of blocked paths to exclude from operations'), + blockedExtensions: z + .array(z.string()) + .default(DEFAULT_BLOCKED_EXTENSIONS) + .describe('List of blocked file extensions'), + maxFileSize: z + .number() + .int() + .positive() + .default(DEFAULT_MAX_FILE_SIZE) + .describe( + `Maximum file size in bytes (default: ${DEFAULT_MAX_FILE_SIZE / 1024 / 1024}MB)` + ), + workingDirectory: z + .string() + .optional() + .describe('Working directory for file operations (defaults to process.cwd())'), + enableBackups: z + .boolean() + .default(DEFAULT_ENABLE_BACKUPS) + .describe('Enable automatic backups of modified files'), + backupPath: z + .string() + .optional() + .describe('Absolute path for storing file backups (if enableBackups is true)'), + backupRetentionDays: z + .number() + .int() + .positive() + .default(DEFAULT_BACKUP_RETENTION_DAYS) + .describe( + `Number of days to retain backup files (default: ${DEFAULT_BACKUP_RETENTION_DAYS})` + ), + enabledTools: z + .array(z.enum(FILESYSTEM_TOOL_NAMES)) + .optional() + .describe( + `Subset of tools to enable. If not specified, all tools are enabled. Available: ${FILESYSTEM_TOOL_NAMES.join(', ')}` + ), + }) + .strict(); + +type FileSystemToolsConfig = z.output; + +/** + * FileSystem tools provider. + * + * Wraps FileSystemService and provides file operation tools: + * - read_file: Read file contents with pagination + * - write_file: Write or overwrite file contents + * - edit_file: Edit files using search/replace operations + * - glob_files: Find files matching glob patterns + * - grep_content: Search file contents using regex + * + * When registered via customToolRegistry, FileSystemService is automatically + * initialized and file operation tools become available to the agent. + */ +export const fileSystemToolsProvider: CustomToolProvider< + 'filesystem-tools', + FileSystemToolsConfig +> = { + type: 'filesystem-tools', + configSchema: FileSystemToolsConfigSchema, + + create: (config: FileSystemToolsConfig, context: ToolCreationContext): InternalTool[] => { + const { logger, services } = context; + + logger.debug('Creating FileSystemService for filesystem tools'); + + // Create FileSystemService with validated config + const fileSystemService = new FileSystemService( + { + allowedPaths: config.allowedPaths, + blockedPaths: config.blockedPaths, + blockedExtensions: config.blockedExtensions, + maxFileSize: config.maxFileSize, + workingDirectory: config.workingDirectory || process.cwd(), + enableBackups: config.enableBackups, + backupPath: config.backupPath, + backupRetentionDays: config.backupRetentionDays, + }, + logger + ); + + // Start initialization in background - service methods use ensureInitialized() for lazy init + // This means tools will wait for initialization to complete before executing + fileSystemService.initialize().catch((error) => { + logger.error(`Failed to initialize FileSystemService: ${error.message}`); + }); + + logger.debug('FileSystemService created - initialization will complete on first tool use'); + + // Set up directory approval checker callback if approvalManager is available + // This allows FileSystemService to check approved directories during validation + const approvalManager = services?.approvalManager; + if (approvalManager) { + const approvalChecker = (filePath: string) => { + // Use isDirectoryApproved() for EXECUTION decisions (checks both 'session' and 'once' types) + // isDirectorySessionApproved() is only for PROMPTING decisions (checks 'session' type only) + return approvalManager.isDirectoryApproved(filePath); + }; + fileSystemService.setDirectoryApprovalChecker(approvalChecker); + logger.debug('Directory approval checker configured for FileSystemService'); + } + + // Create directory approval callbacks for file tools + // These allow tools to check and request directory approval + const directoryApproval = approvalManager + ? { + isSessionApproved: (filePath: string) => + approvalManager.isDirectorySessionApproved(filePath), + addApproved: (directory: string, type: 'session' | 'once') => + approvalManager.addApprovedDirectory(directory, type), + } + : undefined; + + // Create options for file tools with directory approval support + const fileToolOptions: FileToolOptions = { + fileSystemService, + directoryApproval, + }; + + // Build tool map for selective enabling + const toolCreators: Record InternalTool> = { + read_file: () => createReadFileTool(fileToolOptions), + write_file: () => createWriteFileTool(fileToolOptions), + edit_file: () => createEditFileTool(fileToolOptions), + glob_files: () => createGlobFilesTool(fileToolOptions), + grep_content: () => createGrepContentTool(fileToolOptions), + }; + + // Determine which tools to create + const toolsToCreate = config.enabledTools ?? FILESYSTEM_TOOL_NAMES; + + if (config.enabledTools) { + logger.debug(`Creating subset of filesystem tools: ${toolsToCreate.join(', ')}`); + } + + // Create and return only the enabled tools + return toolsToCreate.map((toolName) => toolCreators[toolName]()); + }, + + metadata: { + displayName: 'FileSystem Tools', + description: 'File system operations (read, write, edit, glob, grep)', + category: 'filesystem', + }, +}; diff --git a/dexto/packages/tools-filesystem/src/types.ts b/dexto/packages/tools-filesystem/src/types.ts new file mode 100644 index 00000000..5f406f2a --- /dev/null +++ b/dexto/packages/tools-filesystem/src/types.ts @@ -0,0 +1,204 @@ +/** + * FileSystem Service Types + * + * Types and interfaces for file system operations including reading, writing, + * searching, and validation. + */ + +// BufferEncoding type from Node.js +export type BufferEncoding = + | 'ascii' + | 'utf8' + | 'utf-8' + | 'utf16le' + | 'ucs2' + | 'ucs-2' + | 'base64' + | 'base64url' + | 'latin1' + | 'binary' + | 'hex'; + +/** + * File content with metadata + */ +export interface FileContent { + content: string; + lines: number; + encoding: string; + mimeType?: string; + truncated: boolean; + size: number; +} + +/** + * Options for reading files + */ +export interface ReadFileOptions { + /** Maximum number of lines to read */ + limit?: number | undefined; + /** Starting line number (1-based) */ + offset?: number | undefined; + /** File encoding (default: utf-8) */ + encoding?: BufferEncoding | undefined; +} + +/** + * File metadata for glob results + */ +export interface FileMetadata { + path: string; + size: number; + modified: Date; + isDirectory: boolean; +} + +/** + * Options for glob operations + */ +export interface GlobOptions { + /** Base directory to search from */ + cwd?: string | undefined; + /** Maximum number of results */ + maxResults?: number | undefined; + /** Include file metadata */ + includeMetadata?: boolean | undefined; +} + +/** + * Glob result + */ +export interface GlobResult { + files: FileMetadata[]; + truncated: boolean; + totalFound: number; +} + +/** + * Search match with context + */ +export interface SearchMatch { + file: string; + lineNumber: number; + line: string; + context?: { + before: string[]; + after: string[]; + }; +} + +/** + * Options for content search (grep) + */ +export interface GrepOptions { + /** Base directory to search */ + path?: string | undefined; + /** Glob pattern to filter files */ + glob?: string | undefined; + /** Number of context lines before/after match */ + contextLines?: number | undefined; + /** Case-insensitive search */ + caseInsensitive?: boolean | undefined; + /** Maximum number of results */ + maxResults?: number | undefined; + /** Include line numbers */ + lineNumbers?: boolean | undefined; +} + +/** + * Search result + */ +export interface SearchResult { + matches: SearchMatch[]; + totalMatches: number; + truncated: boolean; + filesSearched: number; +} + +/** + * Options for writing files + */ +export interface WriteFileOptions { + /** Create parent directories if they don't exist */ + createDirs?: boolean | undefined; + /** File encoding (default: utf-8) */ + encoding?: BufferEncoding | undefined; + /** Create backup before overwriting */ + backup?: boolean | undefined; +} + +/** + * Write result + */ +export interface WriteResult { + success: boolean; + path: string; + bytesWritten: number; + backupPath?: string | undefined; + /** Original content if file was overwritten (undefined for new files) */ + originalContent?: string | undefined; +} + +/** + * Edit operation + */ +export interface EditOperation { + oldString: string; + newString: string; + replaceAll?: boolean | undefined; +} + +/** + * Options for editing files + */ +export interface EditFileOptions { + /** Create backup before editing */ + backup?: boolean; + /** File encoding */ + encoding?: BufferEncoding; +} + +/** + * Edit result + */ +export interface EditResult { + success: boolean; + path: string; + changesCount: number; + backupPath?: string | undefined; + /** Original content before edit (for diff generation) */ + originalContent: string; + /** New content after edit (for diff generation) */ + newContent: string; +} + +/** + * Path validation result + */ +export interface PathValidation { + isValid: boolean; + error?: string; + normalizedPath?: string; +} + +/** + * File system configuration + */ +export interface FileSystemConfig { + /** Allowed base paths */ + allowedPaths: string[]; + /** Blocked paths (relative to allowed paths) */ + blockedPaths: string[]; + /** Blocked file extensions */ + blockedExtensions: string[]; + /** Maximum file size in bytes */ + maxFileSize: number; + /** Enable automatic backups */ + enableBackups: boolean; + /** Backup directory absolute path (required when enableBackups is true - provided by CLI enrichment) */ + backupPath?: string | undefined; + /** Backup retention period in days (default: 7) */ + backupRetentionDays: number; + /** Working directory for glob/grep operations (defaults to process.cwd()) */ + workingDirectory?: string | undefined; +} diff --git a/dexto/packages/tools-filesystem/src/write-file-tool.test.ts b/dexto/packages/tools-filesystem/src/write-file-tool.test.ts new file mode 100644 index 00000000..852c2478 --- /dev/null +++ b/dexto/packages/tools-filesystem/src/write-file-tool.test.ts @@ -0,0 +1,281 @@ +/** + * Write File Tool Tests + * + * Tests for the write_file tool including file modification detection. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import { createWriteFileTool } from './write-file-tool.js'; +import { FileSystemService } from './filesystem-service.js'; +import { ToolErrorCode } from '@dexto/core'; +import { DextoRuntimeError } from '@dexto/core'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +describe('write_file tool', () => { + let mockLogger: ReturnType; + let tempDir: string; + let fileSystemService: FileSystemService; + + beforeEach(async () => { + mockLogger = createMockLogger(); + + // Create temp directory for testing + const rawTempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-write-test-')); + tempDir = await fs.realpath(rawTempDir); + + fileSystemService = new FileSystemService( + { + allowedPaths: [tempDir], + blockedPaths: [], + blockedExtensions: [], + maxFileSize: 10 * 1024 * 1024, + workingDirectory: tempDir, + enableBackups: false, + backupRetentionDays: 7, + }, + mockLogger as any + ); + await fileSystemService.initialize(); + + vi.clearAllMocks(); + }); + + afterEach(async () => { + // Cleanup temp directory + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('File Modification Detection - Existing Files', () => { + it('should succeed when existing file is not modified between preview and execute', async () => { + const tool = createWriteFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'original content'); + + const toolCallId = 'test-call-123'; + const input = { + file_path: testFile, + content: 'new content', + }; + + // Generate preview (stores hash) + const preview = await tool.generatePreview!(input, { toolCallId }); + expect(preview).toBeDefined(); + expect(preview?.type).toBe('diff'); + + // Execute without modifying file (should succeed) + const result = (await tool.execute(input, { toolCallId })) as { + success: boolean; + path: string; + }; + + expect(result.success).toBe(true); + expect(result.path).toBe(testFile); + + // Verify file was written + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('new content'); + }); + + it('should fail when existing file is modified between preview and execute', async () => { + const tool = createWriteFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'original content'); + + const toolCallId = 'test-call-456'; + const input = { + file_path: testFile, + content: 'new content', + }; + + // Generate preview (stores hash) + await tool.generatePreview!(input, { toolCallId }); + + // Simulate user modifying the file externally + await fs.writeFile(testFile, 'user modified this'); + + // Execute should fail because file was modified + try { + await tool.execute(input, { toolCallId }); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe( + ToolErrorCode.FILE_MODIFIED_SINCE_PREVIEW + ); + } + + // Verify file was NOT modified by the tool (still has user's changes) + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('user modified this'); + }); + + it('should fail when existing file is deleted between preview and execute', async () => { + const tool = createWriteFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'original content'); + + const toolCallId = 'test-call-deleted'; + const input = { + file_path: testFile, + content: 'new content', + }; + + // Generate preview (stores hash of existing file) + await tool.generatePreview!(input, { toolCallId }); + + // Simulate user deleting the file + await fs.unlink(testFile); + + // Execute should fail because file was deleted + try { + await tool.execute(input, { toolCallId }); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe( + ToolErrorCode.FILE_MODIFIED_SINCE_PREVIEW + ); + } + }); + }); + + describe('File Modification Detection - New Files', () => { + it('should succeed when creating new file that still does not exist', async () => { + const tool = createWriteFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'new-file.txt'); + + const toolCallId = 'test-call-new'; + const input = { + file_path: testFile, + content: 'brand new content', + }; + + // Generate preview (stores marker that file doesn't exist) + const preview = await tool.generatePreview!(input, { toolCallId }); + expect(preview).toBeDefined(); + expect(preview?.type).toBe('file'); + expect((preview as any).operation).toBe('create'); + + // Execute (file still doesn't exist - should succeed) + const result = (await tool.execute(input, { toolCallId })) as { success: boolean }; + + expect(result.success).toBe(true); + + // Verify file was created + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('brand new content'); + }); + + it('should fail when file is created by someone else between preview and execute', async () => { + const tool = createWriteFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'race-condition.txt'); + + const toolCallId = 'test-call-race'; + const input = { + file_path: testFile, + content: 'agent content', + }; + + // Generate preview (file doesn't exist) + const preview = await tool.generatePreview!(input, { toolCallId }); + expect(preview?.type).toBe('file'); + + // Simulate someone else creating the file + await fs.writeFile(testFile, 'someone else created this'); + + // Execute should fail because file now exists + try { + await tool.execute(input, { toolCallId }); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe( + ToolErrorCode.FILE_MODIFIED_SINCE_PREVIEW + ); + } + + // Verify the other person's file is preserved + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('someone else created this'); + }); + }); + + describe('Cache Cleanup', () => { + it('should clean up hash cache after successful execution', async () => { + const tool = createWriteFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'original'); + + const toolCallId = 'test-call-cleanup'; + const input = { + file_path: testFile, + content: 'first write', + }; + + // First write + await tool.generatePreview!(input, { toolCallId }); + await tool.execute(input, { toolCallId }); + + // Second write with same toolCallId should work + const input2 = { + file_path: testFile, + content: 'second write', + }; + await tool.generatePreview!(input2, { toolCallId }); + const result = (await tool.execute(input2, { toolCallId })) as { success: boolean }; + + expect(result.success).toBe(true); + const content = await fs.readFile(testFile, 'utf-8'); + expect(content).toBe('second write'); + }); + + it('should clean up hash cache after failed execution', async () => { + const tool = createWriteFileTool({ fileSystemService }); + const testFile = path.join(tempDir, 'test.txt'); + await fs.writeFile(testFile, 'original'); + + const toolCallId = 'test-call-fail'; + const input = { + file_path: testFile, + content: 'new content', + }; + + // Preview + await tool.generatePreview!(input, { toolCallId }); + + // Modify to cause failure + await fs.writeFile(testFile, 'modified'); + + // Execute fails + try { + await tool.execute(input, { toolCallId }); + } catch { + // Expected + } + + // Reset file + await fs.writeFile(testFile, 'reset content'); + + // Next execution with same toolCallId should work + await tool.generatePreview!(input, { toolCallId }); + const result = (await tool.execute(input, { toolCallId })) as { success: boolean }; + + expect(result.success).toBe(true); + }); + }); +}); diff --git a/dexto/packages/tools-filesystem/src/write-file-tool.ts b/dexto/packages/tools-filesystem/src/write-file-tool.ts new file mode 100644 index 00000000..4f866e5d --- /dev/null +++ b/dexto/packages/tools-filesystem/src/write-file-tool.ts @@ -0,0 +1,294 @@ +/** + * Write File Tool + * + * Internal tool for writing content to files (requires approval) + */ + +import * as path from 'node:path'; +import { createHash } from 'node:crypto'; +import { z } from 'zod'; +import { createPatch } from 'diff'; +import { + InternalTool, + ToolExecutionContext, + DextoRuntimeError, + ApprovalType, + ToolError, +} from '@dexto/core'; +import type { + DiffDisplayData, + FileDisplayData, + ApprovalRequestDetails, + ApprovalResponse, +} from '@dexto/core'; +import { FileSystemErrorCode } from './error-codes.js'; +import { BufferEncoding } from './types.js'; +import type { FileToolOptions } from './file-tool-types.js'; + +/** + * Cache for content hashes between preview and execute phases. + * Keyed by toolCallId to ensure proper cleanup after execution. + * This prevents file corruption when user modifies file between preview and execute. + * + * For new files (file doesn't exist at preview time), we store a special marker + * to detect if the file was created between preview and execute. + */ +const previewContentHashCache = new Map(); + +/** Marker for files that didn't exist at preview time */ +const FILE_NOT_EXISTS_MARKER = null; + +/** + * Compute SHA-256 hash of content for change detection + */ +function computeContentHash(content: string): string { + return createHash('sha256').update(content, 'utf8').digest('hex'); +} + +const WriteFileInputSchema = z + .object({ + file_path: z.string().describe('Absolute path where the file should be written'), + content: z.string().describe('Content to write to the file'), + create_dirs: z + .boolean() + .optional() + .default(false) + .describe("Create parent directories if they don't exist (default: false)"), + encoding: z + .enum(['utf-8', 'ascii', 'latin1', 'utf16le']) + .optional() + .default('utf-8') + .describe('File encoding (default: utf-8)'), + }) + .strict(); + +type WriteFileInput = z.input; + +/** + * Generate diff preview without modifying the file + */ +function generateDiffPreview( + filePath: string, + originalContent: string, + newContent: string +): DiffDisplayData { + const unified = createPatch(filePath, originalContent, newContent, 'before', 'after', { + context: 3, + }); + const additions = (unified.match(/^\+[^+]/gm) || []).length; + const deletions = (unified.match(/^-[^-]/gm) || []).length; + + return { + type: 'diff', + unified, + filename: filePath, + additions, + deletions, + }; +} + +/** + * Create the write_file internal tool with directory approval support + */ +export function createWriteFileTool(options: FileToolOptions): InternalTool { + const { fileSystemService, directoryApproval } = options; + + // Store parent directory for use in onApprovalGranted callback + let pendingApprovalParentDir: string | undefined; + + return { + id: 'write_file', + description: + 'Write content to a file. Creates a new file or overwrites existing file. Automatically creates backup of existing files before overwriting. Use create_dirs to create parent directories. Requires approval for all write operations. Returns success status, path, bytes written, and backup path if applicable.', + inputSchema: WriteFileInputSchema, + + /** + * Check if this write operation needs directory access approval. + * Returns custom approval request if the file is outside allowed paths. + */ + getApprovalOverride: async (args: unknown): Promise => { + const { file_path } = args as WriteFileInput; + if (!file_path) return null; + + // Check if path is within config-allowed paths (async for non-blocking symlink resolution) + const isAllowed = await fileSystemService.isPathWithinConfigAllowed(file_path); + if (isAllowed) { + return null; // Use normal tool confirmation + } + + // Check if directory is already session-approved + if (directoryApproval?.isSessionApproved(file_path)) { + return null; // Already approved, use normal flow + } + + // Need directory access approval + const absolutePath = path.resolve(file_path); + const parentDir = path.dirname(absolutePath); + pendingApprovalParentDir = parentDir; + + return { + type: ApprovalType.DIRECTORY_ACCESS, + metadata: { + path: absolutePath, + parentDir, + operation: 'write', + toolName: 'write_file', + }, + }; + }, + + /** + * Handle approved directory access - remember the directory for session + */ + onApprovalGranted: (response: ApprovalResponse): void => { + if (!directoryApproval || !pendingApprovalParentDir) return; + + // Check if user wants to remember the directory + // Use type assertion to access rememberDirectory since response.data is a union type + const data = response.data as { rememberDirectory?: boolean } | undefined; + const rememberDirectory = data?.rememberDirectory ?? false; + directoryApproval.addApproved( + pendingApprovalParentDir, + rememberDirectory ? 'session' : 'once' + ); + + // Clear pending state + pendingApprovalParentDir = undefined; + }, + + /** + * Generate preview for approval UI - shows diff or file creation info + * Stores content hash for change detection in execute phase. + */ + generatePreview: async (input: unknown, context?: ToolExecutionContext) => { + const { file_path, content } = input as WriteFileInput; + + try { + // Try to read existing file + const originalFile = await fileSystemService.readFile(file_path); + const originalContent = originalFile.content; + + // Store content hash for change detection in execute phase + if (context?.toolCallId) { + previewContentHashCache.set( + context.toolCallId, + computeContentHash(originalContent) + ); + } + + // File exists - show diff preview + return generateDiffPreview(file_path, originalContent, content); + } catch (error) { + // Only treat FILE_NOT_FOUND as "create new file", rethrow other errors + if ( + error instanceof DextoRuntimeError && + error.code === FileSystemErrorCode.FILE_NOT_FOUND + ) { + // Store marker that file didn't exist at preview time + if (context?.toolCallId) { + previewContentHashCache.set(context.toolCallId, FILE_NOT_EXISTS_MARKER); + } + + // File doesn't exist - show as file creation with full content + const lineCount = content.split('\n').length; + const preview: FileDisplayData = { + type: 'file', + path: file_path, + operation: 'create', + size: content.length, + lineCount, + content, // Include content for approval preview + }; + return preview; + } + // Permission denied, I/O errors, etc. - rethrow + throw error; + } + }, + + execute: async (input: unknown, context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { file_path, content, create_dirs, encoding } = input as WriteFileInput; + + // Check if file was modified since preview (safety check) + // This prevents corrupting user edits made between preview approval and execution + let originalContent: string | null = null; + let fileExistsNow = false; + + try { + const originalFile = await fileSystemService.readFile(file_path); + originalContent = originalFile.content; + fileExistsNow = true; + } catch (error) { + // Only treat FILE_NOT_FOUND as "create new file", rethrow other errors + if ( + error instanceof DextoRuntimeError && + error.code === FileSystemErrorCode.FILE_NOT_FOUND + ) { + // File doesn't exist - this is a create operation + originalContent = null; + fileExistsNow = false; + } else { + // Permission denied, I/O errors, etc. - rethrow + throw error; + } + } + + // Verify file hasn't changed since preview + if (context?.toolCallId && previewContentHashCache.has(context.toolCallId)) { + const expectedHash = previewContentHashCache.get(context.toolCallId); + previewContentHashCache.delete(context.toolCallId); // Clean up regardless of outcome + + if (expectedHash === FILE_NOT_EXISTS_MARKER) { + // File didn't exist at preview time - verify it still doesn't exist + if (fileExistsNow) { + throw ToolError.fileModifiedSincePreview('write_file', file_path); + } + } else if (expectedHash !== null) { + // File existed at preview time - verify content hasn't changed + if (!fileExistsNow) { + // File was deleted between preview and execute + throw ToolError.fileModifiedSincePreview('write_file', file_path); + } + const currentHash = computeContentHash(originalContent!); + if (expectedHash !== currentHash) { + throw ToolError.fileModifiedSincePreview('write_file', file_path); + } + } + } + + // Write file using FileSystemService + // Backup behavior is controlled by config.enableBackups (default: false) + const result = await fileSystemService.writeFile(file_path, content, { + createDirs: create_dirs, + encoding: encoding as BufferEncoding, + }); + + // Build display data based on operation type + let _display: DiffDisplayData | FileDisplayData; + + if (originalContent === null) { + // New file creation + const lineCount = content.split('\n').length; + _display = { + type: 'file', + path: file_path, + operation: 'create', + size: result.bytesWritten, + lineCount, + }; + } else { + // File overwrite - generate diff using shared helper + _display = generateDiffPreview(file_path, originalContent, content); + } + + return { + success: result.success, + path: result.path, + bytes_written: result.bytesWritten, + ...(result.backupPath && { backup_path: result.backupPath }), + _display, + }; + }, + }; +} diff --git a/dexto/packages/tools-filesystem/tsconfig.json b/dexto/packages/tools-filesystem/tsconfig.json new file mode 100644 index 00000000..01aae715 --- /dev/null +++ b/dexto/packages/tools-filesystem/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true, + "declarationMap": true, + "skipLibCheck": true + }, + "include": ["src/**/*"], + "exclude": ["dist", "node_modules"] +} diff --git a/dexto/packages/tools-filesystem/tsup.config.ts b/dexto/packages/tools-filesystem/tsup.config.ts new file mode 100644 index 00000000..5f7733a1 --- /dev/null +++ b/dexto/packages/tools-filesystem/tsup.config.ts @@ -0,0 +1,24 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig([ + { + entry: ['src/**/*.ts'], + format: ['cjs', 'esm'], + outDir: 'dist', + dts: { + compilerOptions: { + skipLibCheck: true, + }, + }, + platform: 'node', + bundle: false, + clean: true, + tsconfig: './tsconfig.json', + esbuildOptions(options) { + options.logOverride = { + ...(options.logOverride ?? {}), + 'empty-import-meta': 'silent', + }; + }, + }, +]); diff --git a/dexto/packages/tools-plan/.dexto-plugin/plugin.json b/dexto/packages/tools-plan/.dexto-plugin/plugin.json new file mode 100644 index 00000000..fd2ebdf3 --- /dev/null +++ b/dexto/packages/tools-plan/.dexto-plugin/plugin.json @@ -0,0 +1,7 @@ +{ + "name": "plan-tools", + "version": "0.1.0", + "description": "Implementation planning tools with session-linked plans. Create, read, and update plans tied to your session.", + "author": "Dexto", + "customToolProviders": ["plan-tools"] +} diff --git a/dexto/packages/tools-plan/package.json b/dexto/packages/tools-plan/package.json new file mode 100644 index 00000000..6ab5f8e5 --- /dev/null +++ b/dexto/packages/tools-plan/package.json @@ -0,0 +1,40 @@ +{ + "name": "@dexto/tools-plan", + "version": "1.5.6", + "description": "Implementation planning tools with session-linked plans", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "files": [ + "dist", + ".dexto-plugin", + "skills" + ], + "scripts": { + "build": "tsc", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@dexto/core": "workspace:*", + "diff": "^7.0.0", + "zod": "^3.24.1" + }, + "devDependencies": { + "@types/diff": "^7.0.0", + "@types/node": "^22.10.5", + "dotenv": "^16.4.7", + "typescript": "^5.7.2", + "vitest": "^2.1.8" + }, + "author": "Dexto", + "license": "MIT" +} diff --git a/dexto/packages/tools-plan/skills/plan/SKILL.md b/dexto/packages/tools-plan/skills/plan/SKILL.md new file mode 100644 index 00000000..3f36466c --- /dev/null +++ b/dexto/packages/tools-plan/skills/plan/SKILL.md @@ -0,0 +1,102 @@ +--- +name: plan +description: Enter planning mode to create and manage implementation plans +user-invocable: true +--- + +# Planning Mode - PLAN FIRST, THEN IMPLEMENT + +**CRITICAL**: You are in planning mode. You MUST create and get approval for a plan BEFORE writing any code or making any changes. + +## MANDATORY WORKFLOW + +**DO NOT skip these steps. DO NOT start implementing until the plan is approved.** + +1. **Research first** (if needed): Use the explore agent or read relevant files to understand the codebase +2. **Check for existing plan**: Use `plan_read` to see if a plan exists +3. **Create/update plan**: Use `plan_create` or `plan_update` to define your approach +4. **Request review**: Use `plan_review` to get user approval +5. **WAIT for approval**: Only proceed to implementation after user approves +6. **Implement**: Execute the approved plan, updating checkboxes as you go + +## Research Phase + +Before creating your plan, you should understand the codebase: + +- **Use the explore agent** (spawn_agent with subagent_type="Explore") to search for relevant code, patterns, and existing implementations +- **Read key files** to understand the current architecture +- **Identify dependencies** and files that will need changes + +This research informs your plan and prevents wasted effort from incorrect assumptions. + +## Available Tools + +- **plan_create**: Create a new plan (REQUIRED before any implementation) +- **plan_read**: Read the current plan +- **plan_update**: Update the existing plan (shows diff preview) +- **plan_review**: Request user review - returns approve/iterate/reject with feedback + +## WHAT YOU MUST DO NOW + +1. **Research**: Use the explore agent or read files to understand the relevant parts of the codebase +2. **Check plan**: Use `plan_read` to check if a plan already exists +3. **Create plan**: Use `plan_create` to create a comprehensive plan based on your research +4. **Get approval**: Use `plan_review` to request user approval +5. **STOP and WAIT** - do not write any code until the user approves via plan_review + +## Plan Structure + +```markdown +# {Title} + +## Objective +{Clear statement of what we're building/fixing} + +## Steps + +### 1. {Step Name} +- [ ] {Task description} +- [ ] {Task description} +Files: `path/to/file.ts`, `path/to/other.ts` + +### 2. {Step Name} +- [ ] {Task description} +Files: `path/to/file.ts` + +## Considerations +- {Edge cases to handle} +- {Error scenarios} + +## Success Criteria +- {How we know we're done} +``` + +## Guidelines + +- **Break down complex tasks** into clear, sequential steps +- **Include specific file paths** that will be created or modified +- **Note dependencies** between steps +- **Keep plans concise** but complete + +## Handling Review Responses + +After calling `plan_review`, handle the response: + +- **approve**: User approved - proceed with implementation +- **iterate**: User wants changes - update the plan based on feedback, then call `plan_review` again +- **reject**: User rejected - ask what they want instead + +## DO NOT + +- ❌ Start writing code before creating a plan +- ❌ Skip the plan_review step +- ❌ Assume approval - wait for explicit user response +- ❌ Make changes outside the approved plan without updating it first + +--- + +**START NOW**: +1. Research the codebase using the explore agent if needed +2. Use `plan_read` to check for an existing plan +3. Use `plan_create` to create your plan +4. Use `plan_review` to get approval before any implementation diff --git a/dexto/packages/tools-plan/src/errors.ts b/dexto/packages/tools-plan/src/errors.ts new file mode 100644 index 00000000..2223a62f --- /dev/null +++ b/dexto/packages/tools-plan/src/errors.ts @@ -0,0 +1,118 @@ +/** + * Plan Error Factory + * + * Provides typed errors for plan operations following the DextoRuntimeError pattern. + */ + +import { DextoRuntimeError, ErrorType } from '@dexto/core'; + +/** + * Error codes for plan operations + */ +export const PlanErrorCode = { + /** Plan already exists for session */ + PLAN_ALREADY_EXISTS: 'PLAN_ALREADY_EXISTS', + /** Plan not found for session */ + PLAN_NOT_FOUND: 'PLAN_NOT_FOUND', + /** Invalid plan content */ + INVALID_PLAN_CONTENT: 'INVALID_PLAN_CONTENT', + /** Session ID required */ + SESSION_ID_REQUIRED: 'SESSION_ID_REQUIRED', + /** Invalid session ID (path traversal attempt) */ + INVALID_SESSION_ID: 'INVALID_SESSION_ID', + /** Checkpoint not found */ + CHECKPOINT_NOT_FOUND: 'CHECKPOINT_NOT_FOUND', + /** Storage operation failed */ + STORAGE_ERROR: 'STORAGE_ERROR', +} as const; + +export type PlanErrorCodeType = (typeof PlanErrorCode)[keyof typeof PlanErrorCode]; + +/** + * Error factory for plan operations + */ +export const PlanError = { + /** + * Plan already exists for the given session + */ + planAlreadyExists(sessionId: string): DextoRuntimeError { + return new DextoRuntimeError( + PlanErrorCode.PLAN_ALREADY_EXISTS, + 'plan', + ErrorType.USER, + `A plan already exists for session '${sessionId}'. Use plan_update to modify it.`, + { sessionId }, + 'Use plan_update to modify the existing plan, or plan_read to view it.' + ); + }, + + /** + * Plan not found for the given session + */ + planNotFound(sessionId: string): DextoRuntimeError { + return new DextoRuntimeError( + PlanErrorCode.PLAN_NOT_FOUND, + 'plan', + ErrorType.NOT_FOUND, + `No plan found for session '${sessionId}'.`, + { sessionId }, + 'Use plan_create to create a new plan for this session.' + ); + }, + + /** + * Session ID is required for plan operations + */ + sessionIdRequired(): DextoRuntimeError { + return new DextoRuntimeError( + PlanErrorCode.SESSION_ID_REQUIRED, + 'plan', + ErrorType.USER, + 'Session ID is required for plan operations.', + {}, + 'Ensure the tool is called within a valid session context.' + ); + }, + + /** + * Invalid session ID (path traversal attempt) + */ + invalidSessionId(sessionId: string): DextoRuntimeError { + return new DextoRuntimeError( + PlanErrorCode.INVALID_SESSION_ID, + 'plan', + ErrorType.USER, + `Invalid session ID: '${sessionId}' contains invalid path characters.`, + { sessionId }, + 'Session IDs must not contain path traversal characters like "..".' + ); + }, + + /** + * Checkpoint not found in plan + */ + checkpointNotFound(checkpointId: string, sessionId: string): DextoRuntimeError { + return new DextoRuntimeError( + PlanErrorCode.CHECKPOINT_NOT_FOUND, + 'plan', + ErrorType.NOT_FOUND, + `Checkpoint '${checkpointId}' not found in plan for session '${sessionId}'.`, + { checkpointId, sessionId }, + 'Use plan_read to view available checkpoints.' + ); + }, + + /** + * Storage operation failed + */ + storageError(operation: string, sessionId: string, cause?: Error): DextoRuntimeError { + return new DextoRuntimeError( + PlanErrorCode.STORAGE_ERROR, + 'plan', + ErrorType.SYSTEM, + `Failed to ${operation} plan for session '${sessionId}': ${cause?.message || 'unknown error'}`, + { operation, sessionId, cause: cause?.message }, + 'Check file system permissions and try again.' + ); + }, +}; diff --git a/dexto/packages/tools-plan/src/index.ts b/dexto/packages/tools-plan/src/index.ts new file mode 100644 index 00000000..f2d7e171 --- /dev/null +++ b/dexto/packages/tools-plan/src/index.ts @@ -0,0 +1,48 @@ +/** + * @dexto/tools-plan + * + * Implementation planning tools with session-linked plans. + * Provides tools for creating, reading, updating, and tracking plans. + * + * This package is a Dexto plugin that automatically registers: + * - Custom tool provider: plan-tools + * - Skill: plan (planning mode instructions) + * + * Usage: + * 1. Install the package + * 2. The plugin discovery will find .dexto-plugin/plugin.json + * 3. Tools and skill are automatically registered + */ + +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +/** + * Path to the plugin directory containing .dexto-plugin manifest. + * Use this in image definitions to declare bundled plugins. + * + * @example + * ```typescript + * import { PLUGIN_PATH } from '@dexto/tools-plan'; + * + * export default defineImage({ + * bundledPlugins: [PLUGIN_PATH], + * // ... + * }); + * ``` + */ +export const PLUGIN_PATH = path.resolve(__dirname, '..'); + +// Tool provider (for direct registration if needed) +export { planToolsProvider } from './tool-provider.js'; + +// Service (for advanced use cases) +export { PlanService } from './plan-service.js'; + +// Types +export type { Plan, PlanMeta, PlanStatus, PlanServiceOptions, PlanUpdateResult } from './types.js'; + +// Error utilities +export { PlanError, PlanErrorCode, type PlanErrorCodeType } from './errors.js'; diff --git a/dexto/packages/tools-plan/src/plan-service.test.ts b/dexto/packages/tools-plan/src/plan-service.test.ts new file mode 100644 index 00000000..f60f69b2 --- /dev/null +++ b/dexto/packages/tools-plan/src/plan-service.test.ts @@ -0,0 +1,266 @@ +/** + * Plan Service Tests + * + * Tests for the PlanService CRUD operations and error handling. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import { PlanService } from './plan-service.js'; +import { PlanErrorCode } from './errors.js'; +import { DextoRuntimeError } from '@dexto/core'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +describe('PlanService', () => { + let mockLogger: ReturnType; + let tempDir: string; + let planService: PlanService; + + beforeEach(async () => { + mockLogger = createMockLogger(); + + // Create temp directory for testing + const rawTempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-plan-test-')); + tempDir = await fs.realpath(rawTempDir); + + planService = new PlanService({ basePath: tempDir }, mockLogger as any); + + vi.clearAllMocks(); + }); + + afterEach(async () => { + // Cleanup temp directory + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('exists', () => { + it('should return false for non-existent plan', async () => { + const exists = await planService.exists('non-existent-session'); + expect(exists).toBe(false); + }); + + it('should return true for existing plan', async () => { + const sessionId = 'test-session'; + await planService.create(sessionId, '# Test Plan'); + + const exists = await planService.exists(sessionId); + expect(exists).toBe(true); + }); + }); + + describe('create', () => { + it('should create a new plan with content and metadata', async () => { + const sessionId = 'test-session'; + const content = '# Implementation Plan\n\n## Steps\n1. First step'; + const title = 'Test Plan'; + + const plan = await planService.create(sessionId, content, { title }); + + expect(plan.content).toBe(content); + expect(plan.meta.sessionId).toBe(sessionId); + expect(plan.meta.status).toBe('draft'); + expect(plan.meta.title).toBe(title); + expect(plan.meta.createdAt).toBeGreaterThan(0); + expect(plan.meta.updatedAt).toBeGreaterThan(0); + }); + + it('should throw error when plan already exists', async () => { + const sessionId = 'test-session'; + await planService.create(sessionId, '# First Plan'); + + try { + await planService.create(sessionId, '# Second Plan'); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.PLAN_ALREADY_EXISTS); + } + }); + + it('should store plan files on disk', async () => { + const sessionId = 'test-session'; + const content = '# Test Plan'; + await planService.create(sessionId, content); + + // Verify plan.md exists + const planPath = path.join(tempDir, sessionId, 'plan.md'); + const storedContent = await fs.readFile(planPath, 'utf-8'); + expect(storedContent).toBe(content); + + // Verify plan-meta.json exists + const metaPath = path.join(tempDir, sessionId, 'plan-meta.json'); + const metaContent = await fs.readFile(metaPath, 'utf-8'); + const meta = JSON.parse(metaContent); + expect(meta.sessionId).toBe(sessionId); + }); + }); + + describe('read', () => { + it('should return null for non-existent plan', async () => { + const plan = await planService.read('non-existent-session'); + expect(plan).toBeNull(); + }); + + it('should read existing plan with content and metadata', async () => { + const sessionId = 'test-session'; + const content = '# Test Plan'; + const title = 'My Plan'; + await planService.create(sessionId, content, { title }); + + const plan = await planService.read(sessionId); + + expect(plan).not.toBeNull(); + expect(plan!.content).toBe(content); + expect(plan!.meta.sessionId).toBe(sessionId); + expect(plan!.meta.title).toBe(title); + }); + + it('should handle invalid metadata schema gracefully', async () => { + const sessionId = 'test-session'; + await planService.create(sessionId, '# Test'); + + // Write valid JSON but invalid schema (missing required fields) + const metaPath = path.join(tempDir, sessionId, 'plan-meta.json'); + await fs.writeFile(metaPath, JSON.stringify({ invalidField: 'value' })); + + const plan = await planService.read(sessionId); + + // Should return with default metadata + expect(plan).not.toBeNull(); + expect(plan!.meta.sessionId).toBe(sessionId); + expect(plan!.meta.status).toBe('draft'); + expect(mockLogger.warn).toHaveBeenCalled(); + }); + + it('should return null for corrupted JSON metadata', async () => { + const sessionId = 'test-session'; + await planService.create(sessionId, '# Test'); + + // Corrupt the metadata with invalid JSON + const metaPath = path.join(tempDir, sessionId, 'plan-meta.json'); + await fs.writeFile(metaPath, '{ invalid json }'); + + const plan = await planService.read(sessionId); + + // Should return null and log error + expect(plan).toBeNull(); + expect(mockLogger.error).toHaveBeenCalled(); + }); + }); + + describe('update', () => { + it('should update plan content', async () => { + const sessionId = 'test-session'; + await planService.create(sessionId, '# Original Content'); + + const result = await planService.update(sessionId, '# Updated Content'); + + expect(result.oldContent).toBe('# Original Content'); + expect(result.newContent).toBe('# Updated Content'); + expect(result.meta.updatedAt).toBeGreaterThan(0); + }); + + it('should preserve metadata when updating content', async () => { + const sessionId = 'test-session'; + const plan = await planService.create(sessionId, '# Original', { title: 'My Title' }); + const originalCreatedAt = plan.meta.createdAt; + + await planService.update(sessionId, '# Updated'); + + const updatedPlan = await planService.read(sessionId); + expect(updatedPlan!.meta.title).toBe('My Title'); + expect(updatedPlan!.meta.createdAt).toBe(originalCreatedAt); + }); + + it('should throw error when plan does not exist', async () => { + try { + await planService.update('non-existent', '# Content'); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.PLAN_NOT_FOUND); + } + }); + }); + + describe('updateMeta', () => { + it('should update plan status', async () => { + const sessionId = 'test-session'; + await planService.create(sessionId, '# Plan'); + + const meta = await planService.updateMeta(sessionId, { status: 'approved' }); + + expect(meta.status).toBe('approved'); + }); + + it('should update plan title', async () => { + const sessionId = 'test-session'; + await planService.create(sessionId, '# Plan'); + + const meta = await planService.updateMeta(sessionId, { title: 'New Title' }); + + expect(meta.title).toBe('New Title'); + }); + + it('should throw error when plan does not exist', async () => { + try { + await planService.updateMeta('non-existent', { status: 'approved' }); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.PLAN_NOT_FOUND); + } + }); + }); + + describe('delete', () => { + it('should delete existing plan', async () => { + const sessionId = 'test-session'; + await planService.create(sessionId, '# Plan'); + + await planService.delete(sessionId); + + const exists = await planService.exists(sessionId); + expect(exists).toBe(false); + }); + + it('should throw error when plan does not exist', async () => { + try { + await planService.delete('non-existent'); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.PLAN_NOT_FOUND); + } + }); + + it('should remove plan directory from disk', async () => { + const sessionId = 'test-session'; + await planService.create(sessionId, '# Plan'); + const planDir = path.join(tempDir, sessionId); + + await planService.delete(sessionId); + + try { + await fs.access(planDir); + expect.fail('Directory should not exist'); + } catch { + // Expected - directory should not exist + } + }); + }); +}); diff --git a/dexto/packages/tools-plan/src/plan-service.ts b/dexto/packages/tools-plan/src/plan-service.ts new file mode 100644 index 00000000..a07d89bb --- /dev/null +++ b/dexto/packages/tools-plan/src/plan-service.ts @@ -0,0 +1,273 @@ +/** + * Plan Service + * + * Handles storage and retrieval of implementation plans. + * Plans are stored in .dexto/plans/{sessionId}/ with: + * - plan.md: The plan content + * - plan-meta.json: Metadata (status, checkpoints, timestamps) + */ + +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { existsSync } from 'node:fs'; +import type { IDextoLogger } from '@dexto/core'; +import { PlanMetaSchema } from './types.js'; +import type { Plan, PlanMeta, PlanServiceOptions, PlanUpdateResult } from './types.js'; +import { PlanError } from './errors.js'; + +const PLAN_FILENAME = 'plan.md'; +const META_FILENAME = 'plan-meta.json'; + +/** + * Service for managing implementation plans. + */ +export class PlanService { + private basePath: string; + private logger: IDextoLogger | undefined; + + constructor(options: PlanServiceOptions, logger?: IDextoLogger) { + this.basePath = options.basePath; + this.logger = logger; + } + + /** + * Resolves and validates a session directory path. + * Prevents path traversal attacks by ensuring the resolved path stays within basePath. + */ + private resolveSessionDir(sessionId: string): string { + const base = path.resolve(this.basePath); + const resolved = path.resolve(base, sessionId); + const rel = path.relative(base, resolved); + // Check for path traversal (upward traversal) + if (rel.startsWith('..') || path.isAbsolute(rel)) { + throw PlanError.invalidSessionId(sessionId); + } + return resolved; + } + + /** + * Gets the directory path for a session's plan + */ + private getPlanDir(sessionId: string): string { + return this.resolveSessionDir(sessionId); + } + + /** + * Gets the path to the plan content file. + * Public accessor for tools that need to display the path. + */ + public getPlanPath(sessionId: string): string { + return path.join(this.getPlanDir(sessionId), PLAN_FILENAME); + } + + /** + * Gets the path to the plan metadata file + */ + private getMetaPath(sessionId: string): string { + return path.join(this.getPlanDir(sessionId), META_FILENAME); + } + + /** + * Checks if a plan exists for the given session + */ + async exists(sessionId: string): Promise { + const planPath = this.getPlanPath(sessionId); + return existsSync(planPath); + } + + /** + * Creates a new plan for the session + * + * @throws PlanError.planAlreadyExists if plan already exists + * @throws PlanError.storageError on filesystem errors + */ + async create(sessionId: string, content: string, options?: { title?: string }): Promise { + // Check if plan already exists + if (await this.exists(sessionId)) { + throw PlanError.planAlreadyExists(sessionId); + } + + const planDir = this.getPlanDir(sessionId); + const now = Date.now(); + + // Create metadata + const meta: PlanMeta = { + sessionId, + status: 'draft', + title: options?.title, + createdAt: now, + updatedAt: now, + }; + + try { + // Ensure directory exists + await fs.mkdir(planDir, { recursive: true }); + + // Write plan content and metadata + await Promise.all([ + fs.writeFile(this.getPlanPath(sessionId), content, 'utf-8'), + fs.writeFile(this.getMetaPath(sessionId), JSON.stringify(meta, null, 2), 'utf-8'), + ]); + + this.logger?.debug(`Created plan for session ${sessionId}`); + + return { content, meta }; + } catch (error) { + throw PlanError.storageError('create', sessionId, error as Error); + } + } + + /** + * Reads the plan for the given session + * + * @returns The plan or null if not found + */ + async read(sessionId: string): Promise { + if (!(await this.exists(sessionId))) { + return null; + } + + try { + const [content, metaContent] = await Promise.all([ + fs.readFile(this.getPlanPath(sessionId), 'utf-8'), + fs.readFile(this.getMetaPath(sessionId), 'utf-8'), + ]); + + const metaParsed = JSON.parse(metaContent); + const metaResult = PlanMetaSchema.safeParse(metaParsed); + + if (!metaResult.success) { + this.logger?.warn(`Invalid plan metadata for session ${sessionId}, using defaults`); + // Return with minimal metadata if parsing fails + return { + content, + meta: { + sessionId, + status: 'draft', + createdAt: Date.now(), + updatedAt: Date.now(), + }, + }; + } + + return { content, meta: metaResult.data }; + } catch (error) { + const err = error as NodeJS.ErrnoException; + // ENOENT means file doesn't exist - return null (expected case) + if (err.code === 'ENOENT') { + return null; + } + // JSON parse errors (SyntaxError) mean corrupted data - treat as not found + // but log for debugging + if (error instanceof SyntaxError) { + this.logger?.error( + `Failed to read plan for session ${sessionId}: ${error.message}` + ); + return null; + } + // For real I/O errors (permission denied, disk issues), throw to surface the issue + this.logger?.error( + `Failed to read plan for session ${sessionId}: ${err.message ?? String(err)}` + ); + throw PlanError.storageError('read', sessionId, err); + } + } + + /** + * Updates the plan content for the given session + * + * @throws PlanError.planNotFound if plan doesn't exist + * @throws PlanError.storageError on filesystem errors + */ + async update(sessionId: string, content: string): Promise { + const existing = await this.read(sessionId); + if (!existing) { + throw PlanError.planNotFound(sessionId); + } + + const oldContent = existing.content; + const now = Date.now(); + + // Update metadata + const updatedMeta: PlanMeta = { + ...existing.meta, + updatedAt: now, + }; + + try { + await Promise.all([ + fs.writeFile(this.getPlanPath(sessionId), content, 'utf-8'), + fs.writeFile( + this.getMetaPath(sessionId), + JSON.stringify(updatedMeta, null, 2), + 'utf-8' + ), + ]); + + this.logger?.debug(`Updated plan for session ${sessionId}`); + + return { + oldContent, + newContent: content, + meta: updatedMeta, + }; + } catch (error) { + throw PlanError.storageError('update', sessionId, error as Error); + } + } + + /** + * Updates the plan metadata (status, title) + * + * @throws PlanError.planNotFound if plan doesn't exist + * @throws PlanError.storageError on filesystem errors + */ + async updateMeta( + sessionId: string, + updates: Partial> + ): Promise { + const existing = await this.read(sessionId); + if (!existing) { + throw PlanError.planNotFound(sessionId); + } + + const updatedMeta: PlanMeta = { + ...existing.meta, + ...updates, + updatedAt: Date.now(), + }; + + try { + await fs.writeFile( + this.getMetaPath(sessionId), + JSON.stringify(updatedMeta, null, 2), + 'utf-8' + ); + + this.logger?.debug(`Updated plan metadata for session ${sessionId}`); + + return updatedMeta; + } catch (error) { + throw PlanError.storageError('update metadata', sessionId, error as Error); + } + } + + /** + * Deletes the plan for the given session + * + * @throws PlanError.planNotFound if plan doesn't exist + * @throws PlanError.storageError on filesystem errors + */ + async delete(sessionId: string): Promise { + if (!(await this.exists(sessionId))) { + throw PlanError.planNotFound(sessionId); + } + + try { + await fs.rm(this.getPlanDir(sessionId), { recursive: true, force: true }); + this.logger?.debug(`Deleted plan for session ${sessionId}`); + } catch (error) { + throw PlanError.storageError('delete', sessionId, error as Error); + } + } +} diff --git a/dexto/packages/tools-plan/src/tool-provider.test.ts b/dexto/packages/tools-plan/src/tool-provider.test.ts new file mode 100644 index 00000000..c30dff2f --- /dev/null +++ b/dexto/packages/tools-plan/src/tool-provider.test.ts @@ -0,0 +1,234 @@ +/** + * Plan Tools Provider Tests + * + * Tests for the planToolsProvider configuration and tool creation. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import { planToolsProvider } from './tool-provider.js'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +// Create mock context with logger and minimal agent +const createMockContext = (logger: ReturnType) => ({ + logger: logger as any, + agent: {} as any, // Minimal mock - provider only uses logger +}); + +describe('planToolsProvider', () => { + let mockLogger: ReturnType; + let tempDir: string; + let originalCwd: string; + + beforeEach(async () => { + mockLogger = createMockLogger(); + + // Create temp directory for testing + const rawTempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-provider-test-')); + tempDir = await fs.realpath(rawTempDir); + + // Store original cwd and mock process.cwd to return temp dir + originalCwd = process.cwd(); + vi.spyOn(process, 'cwd').mockReturnValue(tempDir); + + vi.clearAllMocks(); + }); + + afterEach(async () => { + // Restore mocked process.cwd + vi.mocked(process.cwd).mockRestore(); + + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('provider metadata', () => { + it('should have correct type', () => { + expect(planToolsProvider.type).toBe('plan-tools'); + }); + + it('should have metadata', () => { + expect(planToolsProvider.metadata).toBeDefined(); + expect(planToolsProvider.metadata?.displayName).toBe('Plan Tools'); + expect(planToolsProvider.metadata?.category).toBe('planning'); + }); + }); + + describe('config schema', () => { + it('should validate minimal config', () => { + const result = planToolsProvider.configSchema.safeParse({ + type: 'plan-tools', + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.basePath).toBe('.dexto/plans'); + } + }); + + it('should validate config with custom basePath', () => { + const result = planToolsProvider.configSchema.safeParse({ + type: 'plan-tools', + basePath: '/custom/path', + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.basePath).toBe('/custom/path'); + } + }); + + it('should validate config with enabledTools', () => { + const result = planToolsProvider.configSchema.safeParse({ + type: 'plan-tools', + enabledTools: ['plan_create', 'plan_read'], + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.enabledTools).toEqual(['plan_create', 'plan_read']); + } + }); + + it('should reject invalid tool names', () => { + const result = planToolsProvider.configSchema.safeParse({ + type: 'plan-tools', + enabledTools: ['invalid_tool'], + }); + + expect(result.success).toBe(false); + }); + + it('should reject unknown properties', () => { + const result = planToolsProvider.configSchema.safeParse({ + type: 'plan-tools', + unknownProp: 'value', + }); + + expect(result.success).toBe(false); + }); + }); + + describe('create', () => { + it('should create all tools by default', () => { + const config = planToolsProvider.configSchema.parse({ + type: 'plan-tools', + }); + + const tools = planToolsProvider.create(config, createMockContext(mockLogger)); + + expect(tools).toHaveLength(4); + const toolIds = tools.map((t) => t.id); + expect(toolIds).toContain('plan_create'); + expect(toolIds).toContain('plan_read'); + expect(toolIds).toContain('plan_update'); + expect(toolIds).toContain('plan_review'); + }); + + it('should create only enabled tools', () => { + const config = planToolsProvider.configSchema.parse({ + type: 'plan-tools', + enabledTools: ['plan_create', 'plan_read'], + }); + + const tools = planToolsProvider.create(config, createMockContext(mockLogger)); + + expect(tools).toHaveLength(2); + const toolIds = tools.map((t) => t.id); + expect(toolIds).toContain('plan_create'); + expect(toolIds).toContain('plan_read'); + expect(toolIds).not.toContain('plan_update'); + }); + + it('should create single tool', () => { + const config = planToolsProvider.configSchema.parse({ + type: 'plan-tools', + enabledTools: ['plan_update'], + }); + + const tools = planToolsProvider.create(config, createMockContext(mockLogger)); + + expect(tools).toHaveLength(1); + expect(tools[0]!.id).toBe('plan_update'); + }); + + it('should use relative basePath from cwd', () => { + const config = planToolsProvider.configSchema.parse({ + type: 'plan-tools', + basePath: '.dexto/plans', + }); + + planToolsProvider.create(config, createMockContext(mockLogger)); + + // Verify debug log was called with resolved path + expect(mockLogger.debug).toHaveBeenCalledWith( + expect.stringContaining(path.join(tempDir, '.dexto/plans')) + ); + }); + + it('should use absolute basePath as-is', () => { + const absolutePath = '/absolute/path/to/plans'; + const config = planToolsProvider.configSchema.parse({ + type: 'plan-tools', + basePath: absolutePath, + }); + + planToolsProvider.create(config, createMockContext(mockLogger)); + + expect(mockLogger.debug).toHaveBeenCalledWith(expect.stringContaining(absolutePath)); + }); + + it('should log when creating subset of tools', () => { + const config = planToolsProvider.configSchema.parse({ + type: 'plan-tools', + enabledTools: ['plan_create'], + }); + + planToolsProvider.create(config, createMockContext(mockLogger)); + + expect(mockLogger.debug).toHaveBeenCalledWith( + expect.stringContaining('Creating subset of plan tools') + ); + }); + }); + + describe('tool descriptions', () => { + it('should have descriptions for all tools', () => { + const config = planToolsProvider.configSchema.parse({ + type: 'plan-tools', + }); + + const tools = planToolsProvider.create(config, createMockContext(mockLogger)); + + for (const tool of tools) { + expect(tool.description).toBeDefined(); + expect(tool.description.length).toBeGreaterThan(0); + } + }); + + it('should have input schemas for all tools', () => { + const config = planToolsProvider.configSchema.parse({ + type: 'plan-tools', + }); + + const tools = planToolsProvider.create(config, createMockContext(mockLogger)); + + for (const tool of tools) { + expect(tool.inputSchema).toBeDefined(); + } + }); + }); +}); diff --git a/dexto/packages/tools-plan/src/tool-provider.ts b/dexto/packages/tools-plan/src/tool-provider.ts new file mode 100644 index 00000000..7939da90 --- /dev/null +++ b/dexto/packages/tools-plan/src/tool-provider.ts @@ -0,0 +1,100 @@ +/** + * Plan Tools Provider + * + * Provides implementation planning tools: + * - plan_create: Create a new plan for the session + * - plan_read: Read the current plan + * - plan_update: Update the existing plan + * - plan_review: Request user review of the plan (shows plan content with approval options) + */ + +import * as path from 'node:path'; +import { z } from 'zod'; +import type { CustomToolProvider, ToolCreationContext, InternalTool } from '@dexto/core'; +import { PlanService } from './plan-service.js'; +import { createPlanCreateTool } from './tools/plan-create-tool.js'; +import { createPlanReadTool } from './tools/plan-read-tool.js'; +import { createPlanUpdateTool } from './tools/plan-update-tool.js'; +import { createPlanReviewTool } from './tools/plan-review-tool.js'; + +/** + * Available plan tool names for enabledTools configuration + */ +const PLAN_TOOL_NAMES = ['plan_create', 'plan_read', 'plan_update', 'plan_review'] as const; +type PlanToolName = (typeof PLAN_TOOL_NAMES)[number]; + +/** + * Configuration schema for Plan tools provider + */ +const PlanToolsConfigSchema = z + .object({ + type: z.literal('plan-tools'), + basePath: z + .string() + .default('.dexto/plans') + .describe('Base directory for plan storage (relative to working directory)'), + enabledTools: z + .array(z.enum(PLAN_TOOL_NAMES)) + .optional() + .describe( + `Subset of tools to enable. If not specified, all tools are enabled. Available: ${PLAN_TOOL_NAMES.join(', ')}` + ), + }) + .strict(); + +type PlanToolsConfig = z.output; + +/** + * Plan tools provider + * + * Provides implementation planning tools: + * - plan_create: Create a new plan with markdown content + * - plan_read: Read the current plan + * - plan_update: Update existing plan (shows diff preview) + * - plan_review: Request user review of the plan (shows plan with approval options) + * + * Plans are stored in .dexto/plans/{sessionId}/ with: + * - plan.md: Markdown content with checkboxes (- [ ] and - [x]) + * - plan-meta.json: Metadata (status, title, timestamps) + */ +export const planToolsProvider: CustomToolProvider<'plan-tools', PlanToolsConfig> = { + type: 'plan-tools', + configSchema: PlanToolsConfigSchema, + + create: (config: PlanToolsConfig, context: ToolCreationContext): InternalTool[] => { + const { logger } = context; + + // Resolve base path (relative to cwd or absolute) + const basePath = path.isAbsolute(config.basePath) + ? config.basePath + : path.join(process.cwd(), config.basePath); + + logger.debug(`Creating PlanService with basePath: ${basePath}`); + + const planService = new PlanService({ basePath }, logger); + + // Build tool map for selective enabling + const toolCreators: Record InternalTool> = { + plan_create: () => createPlanCreateTool(planService), + plan_read: () => createPlanReadTool(planService), + plan_update: () => createPlanUpdateTool(planService), + plan_review: () => createPlanReviewTool(planService), + }; + + // Determine which tools to create + const toolsToCreate = config.enabledTools ?? PLAN_TOOL_NAMES; + + if (config.enabledTools) { + logger.debug(`Creating subset of plan tools: ${toolsToCreate.join(', ')}`); + } + + // Create and return only the enabled tools + return toolsToCreate.map((toolName) => toolCreators[toolName]()); + }, + + metadata: { + displayName: 'Plan Tools', + description: 'Create and manage implementation plans linked to sessions', + category: 'planning', + }, +}; diff --git a/dexto/packages/tools-plan/src/tools/plan-create-tool.test.ts b/dexto/packages/tools-plan/src/tools/plan-create-tool.test.ts new file mode 100644 index 00000000..14b99474 --- /dev/null +++ b/dexto/packages/tools-plan/src/tools/plan-create-tool.test.ts @@ -0,0 +1,152 @@ +/** + * Plan Create Tool Tests + * + * Tests for the plan_create tool including preview generation. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import { createPlanCreateTool } from './plan-create-tool.js'; +import { PlanService } from '../plan-service.js'; +import { PlanErrorCode } from '../errors.js'; +import { DextoRuntimeError } from '@dexto/core'; +import type { FileDisplayData } from '@dexto/core'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +describe('plan_create tool', () => { + let mockLogger: ReturnType; + let tempDir: string; + let planService: PlanService; + + beforeEach(async () => { + mockLogger = createMockLogger(); + + // Create temp directory for testing + const rawTempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-plan-create-test-')); + tempDir = await fs.realpath(rawTempDir); + + planService = new PlanService({ basePath: tempDir }, mockLogger as any); + + vi.clearAllMocks(); + }); + + afterEach(async () => { + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('generatePreview', () => { + it('should return FileDisplayData for new plan', async () => { + const tool = createPlanCreateTool(planService); + const sessionId = 'test-session'; + const content = '# Implementation Plan\n\n## Steps\n1. First step'; + + const preview = (await tool.generatePreview!( + { title: 'Test Plan', content }, + { sessionId } + )) as FileDisplayData; + + expect(preview.type).toBe('file'); + expect(preview.operation).toBe('create'); + // Path is now absolute, check it ends with the expected suffix + expect(preview.path).toContain(sessionId); + expect(preview.path).toMatch(/plan\.md$/); + expect(preview.content).toBe(content); + expect(preview.lineCount).toBe(4); + }); + + it('should throw error when sessionId is missing', async () => { + const tool = createPlanCreateTool(planService); + + try { + await tool.generatePreview!({ title: 'Test', content: '# Plan' }, {}); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.SESSION_ID_REQUIRED); + } + }); + + it('should throw error when plan already exists', async () => { + const tool = createPlanCreateTool(planService); + const sessionId = 'test-session'; + + // Create existing plan + await planService.create(sessionId, '# Existing Plan'); + + try { + await tool.generatePreview!( + { title: 'New Plan', content: '# New Content' }, + { sessionId } + ); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.PLAN_ALREADY_EXISTS); + } + }); + }); + + describe('execute', () => { + it('should create plan and return success', async () => { + const tool = createPlanCreateTool(planService); + const sessionId = 'test-session'; + const content = '# Implementation Plan'; + const title = 'My Plan'; + + const result = (await tool.execute({ title, content }, { sessionId })) as { + success: boolean; + path: string; + status: string; + title: string; + }; + + expect(result.success).toBe(true); + // Path is now absolute, check it ends with the expected suffix + expect(result.path).toContain(sessionId); + expect(result.path).toMatch(/plan\.md$/); + expect(result.status).toBe('draft'); + expect(result.title).toBe(title); + }); + + it('should throw error when sessionId is missing', async () => { + const tool = createPlanCreateTool(planService); + + try { + await tool.execute({ title: 'Test', content: '# Plan' }, {}); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.SESSION_ID_REQUIRED); + } + }); + + it('should include _display data in result', async () => { + const tool = createPlanCreateTool(planService); + const sessionId = 'test-session'; + const content = '# Plan\n## Steps'; + + const result = (await tool.execute({ title: 'Plan', content }, { sessionId })) as { + _display: FileDisplayData; + }; + + expect(result._display).toBeDefined(); + expect(result._display.type).toBe('file'); + expect(result._display.operation).toBe('create'); + expect(result._display.lineCount).toBe(2); + }); + }); +}); diff --git a/dexto/packages/tools-plan/src/tools/plan-create-tool.ts b/dexto/packages/tools-plan/src/tools/plan-create-tool.ts new file mode 100644 index 00000000..ad7d401c --- /dev/null +++ b/dexto/packages/tools-plan/src/tools/plan-create-tool.ts @@ -0,0 +1,93 @@ +/** + * Plan Create Tool + * + * Creates a new implementation plan for the current session. + * Shows a preview for approval before saving. + */ + +import { z } from 'zod'; +import type { InternalTool, ToolExecutionContext, FileDisplayData } from '@dexto/core'; +import type { PlanService } from '../plan-service.js'; +import { PlanError } from '../errors.js'; + +const PlanCreateInputSchema = z + .object({ + title: z.string().describe('Plan title (e.g., "Add User Authentication")'), + content: z + .string() + .describe( + 'Plan content in markdown format. Use - [ ] and - [x] for checkboxes to track progress.' + ), + }) + .strict(); + +type PlanCreateInput = z.input; + +/** + * Creates the plan_create tool + */ +export function createPlanCreateTool(planService: PlanService): InternalTool { + return { + id: 'plan_create', + description: + 'Create a new implementation plan for the current session. Shows the plan for approval before saving. Use markdown format for the plan content with clear steps and file references.', + inputSchema: PlanCreateInputSchema, + + /** + * Generate preview for approval UI + */ + generatePreview: async ( + input: unknown, + context?: ToolExecutionContext + ): Promise => { + const { content } = input as PlanCreateInput; + + if (!context?.sessionId) { + throw PlanError.sessionIdRequired(); + } + + // Check if plan already exists + const exists = await planService.exists(context.sessionId); + if (exists) { + throw PlanError.planAlreadyExists(context.sessionId); + } + + // Return preview for approval UI + const lineCount = content.split('\n').length; + const planPath = planService.getPlanPath(context.sessionId); + return { + type: 'file', + path: planPath, + operation: 'create', + content, + size: content.length, + lineCount, + }; + }, + + execute: async (input: unknown, context?: ToolExecutionContext) => { + const { title, content } = input as PlanCreateInput; + + if (!context?.sessionId) { + throw PlanError.sessionIdRequired(); + } + + const plan = await planService.create(context.sessionId, content, { title }); + const planPath = planService.getPlanPath(context.sessionId); + + return { + success: true, + path: planPath, + status: plan.meta.status, + title: plan.meta.title, + _display: { + type: 'file', + path: planPath, + operation: 'create', + size: content.length, + lineCount: content.split('\n').length, + } as FileDisplayData, + }; + }, + }; +} diff --git a/dexto/packages/tools-plan/src/tools/plan-read-tool.test.ts b/dexto/packages/tools-plan/src/tools/plan-read-tool.test.ts new file mode 100644 index 00000000..ed05af8e --- /dev/null +++ b/dexto/packages/tools-plan/src/tools/plan-read-tool.test.ts @@ -0,0 +1,114 @@ +/** + * Plan Read Tool Tests + * + * Tests for the plan_read tool. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import { createPlanReadTool } from './plan-read-tool.js'; +import { PlanService } from '../plan-service.js'; +import { PlanErrorCode } from '../errors.js'; +import { DextoRuntimeError } from '@dexto/core'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +describe('plan_read tool', () => { + let mockLogger: ReturnType; + let tempDir: string; + let planService: PlanService; + + beforeEach(async () => { + mockLogger = createMockLogger(); + + const rawTempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-plan-read-test-')); + tempDir = await fs.realpath(rawTempDir); + + planService = new PlanService({ basePath: tempDir }, mockLogger as any); + + vi.clearAllMocks(); + }); + + afterEach(async () => { + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('execute', () => { + it('should return exists: false when no plan exists', async () => { + const tool = createPlanReadTool(planService); + const sessionId = 'test-session'; + + const result = (await tool.execute({}, { sessionId })) as { + exists: boolean; + message: string; + }; + + expect(result.exists).toBe(false); + expect(result.message).toContain('No plan found'); + }); + + it('should return plan content and metadata when plan exists', async () => { + const tool = createPlanReadTool(planService); + const sessionId = 'test-session'; + const content = '# My Plan\n\nSome content'; + const title = 'My Plan Title'; + + await planService.create(sessionId, content, { title }); + + const result = (await tool.execute({}, { sessionId })) as { + exists: boolean; + content: string; + status: string; + title: string; + path: string; + }; + + expect(result.exists).toBe(true); + expect(result.content).toBe(content); + expect(result.status).toBe('draft'); + expect(result.title).toBe(title); + expect(result.path).toBe(`.dexto/plans/${sessionId}/plan.md`); + }); + + it('should return ISO timestamps', async () => { + const tool = createPlanReadTool(planService); + const sessionId = 'test-session'; + + await planService.create(sessionId, '# Plan'); + + const result = (await tool.execute({}, { sessionId })) as { + createdAt: string; + updatedAt: string; + }; + + // Should be ISO format + expect(result.createdAt).toMatch(/^\d{4}-\d{2}-\d{2}T/); + expect(result.updatedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/); + }); + + it('should throw error when sessionId is missing', async () => { + const tool = createPlanReadTool(planService); + + try { + await tool.execute({}, {}); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.SESSION_ID_REQUIRED); + } + }); + }); +}); diff --git a/dexto/packages/tools-plan/src/tools/plan-read-tool.ts b/dexto/packages/tools-plan/src/tools/plan-read-tool.ts new file mode 100644 index 00000000..95c5e51e --- /dev/null +++ b/dexto/packages/tools-plan/src/tools/plan-read-tool.ts @@ -0,0 +1,50 @@ +/** + * Plan Read Tool + * + * Reads the current implementation plan for the session. + * No approval needed - read-only operation. + */ + +import { z } from 'zod'; +import type { InternalTool, ToolExecutionContext } from '@dexto/core'; +import type { PlanService } from '../plan-service.js'; +import { PlanError } from '../errors.js'; + +const PlanReadInputSchema = z.object({}).strict(); + +/** + * Creates the plan_read tool + */ +export function createPlanReadTool(planService: PlanService): InternalTool { + return { + id: 'plan_read', + description: + 'Read the current implementation plan for this session. Returns the plan content and metadata including status. Use markdown checkboxes (- [ ] and - [x]) in the content to track progress.', + inputSchema: PlanReadInputSchema, + + execute: async (_input: unknown, context?: ToolExecutionContext) => { + if (!context?.sessionId) { + throw PlanError.sessionIdRequired(); + } + + const plan = await planService.read(context.sessionId); + + if (!plan) { + return { + exists: false, + message: `No plan found for this session. Use plan_create to create one.`, + }; + } + + return { + exists: true, + path: `.dexto/plans/${context.sessionId}/plan.md`, + content: plan.content, + status: plan.meta.status, + title: plan.meta.title, + createdAt: new Date(plan.meta.createdAt).toISOString(), + updatedAt: new Date(plan.meta.updatedAt).toISOString(), + }; + }, + }; +} diff --git a/dexto/packages/tools-plan/src/tools/plan-review-tool.ts b/dexto/packages/tools-plan/src/tools/plan-review-tool.ts new file mode 100644 index 00000000..18b82f56 --- /dev/null +++ b/dexto/packages/tools-plan/src/tools/plan-review-tool.ts @@ -0,0 +1,104 @@ +/** + * Plan Review Tool + * + * Requests user review of the current plan. + * Shows the plan content for review with approval options: + * - Approve: Proceed with implementation + * - Approve + Accept Edits: Proceed and auto-approve file edits + * - Request Changes: Provide feedback for iteration + * - Reject: Reject the plan entirely + * + * Uses the tool confirmation pattern (not elicitation) so the user + * can see the full plan content before deciding. + */ + +import { z } from 'zod'; +import type { InternalTool, ToolExecutionContext, FileDisplayData } from '@dexto/core'; +import type { PlanService } from '../plan-service.js'; +import { PlanError } from '../errors.js'; + +const PlanReviewInputSchema = z + .object({ + summary: z + .string() + .optional() + .describe('Brief summary of the plan for context (shown above the plan content)'), + }) + .strict(); + +type PlanReviewInput = z.input; + +/** + * Creates the plan_review tool + * + * @param planService - Service for plan operations + */ +export function createPlanReviewTool(planService: PlanService): InternalTool { + return { + id: 'plan_review', + description: + 'Request user review of the current plan. Shows the full plan content for review with options to approve, request changes, or reject. Use after creating or updating a plan to get user approval before implementation.', + inputSchema: PlanReviewInputSchema, + + /** + * Generate preview showing the plan content for review. + * The ApprovalPrompt component detects plan_review and shows custom options. + */ + generatePreview: async ( + input: unknown, + context?: ToolExecutionContext + ): Promise => { + const { summary } = input as PlanReviewInput; + + if (!context?.sessionId) { + throw PlanError.sessionIdRequired(); + } + + // Read the current plan + const plan = await planService.read(context.sessionId); + if (!plan) { + throw PlanError.planNotFound(context.sessionId); + } + + // Build content with optional summary header + let displayContent = plan.content; + if (summary) { + displayContent = `## Summary\n${summary}\n\n---\n\n${plan.content}`; + } + + const lineCount = displayContent.split('\n').length; + const planPath = planService.getPlanPath(context.sessionId); + return { + type: 'file', + path: planPath, + operation: 'read', // 'read' indicates this is for viewing, not creating/modifying + content: displayContent, + size: Buffer.byteLength(displayContent, 'utf8'), + lineCount, + }; + }, + + execute: async (_input: unknown, context?: ToolExecutionContext) => { + // Tool execution means user approved the plan (selected Approve or Approve + Accept Edits) + // Request Changes and Reject are handled as denials in the approval flow + if (!context?.sessionId) { + throw PlanError.sessionIdRequired(); + } + + // Read plan to verify it still exists + const plan = await planService.read(context.sessionId); + if (!plan) { + throw PlanError.planNotFound(context.sessionId); + } + + // Update plan status to approved + await planService.updateMeta(context.sessionId, { status: 'approved' }); + + return { + approved: true, + message: 'Plan approved. You may now proceed with implementation.', + planStatus: 'approved', + }; + }, + }; +} diff --git a/dexto/packages/tools-plan/src/tools/plan-update-tool.test.ts b/dexto/packages/tools-plan/src/tools/plan-update-tool.test.ts new file mode 100644 index 00000000..73c09704 --- /dev/null +++ b/dexto/packages/tools-plan/src/tools/plan-update-tool.test.ts @@ -0,0 +1,196 @@ +/** + * Plan Update Tool Tests + * + * Tests for the plan_update tool including diff preview generation. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import { createPlanUpdateTool } from './plan-update-tool.js'; +import { PlanService } from '../plan-service.js'; +import { PlanErrorCode } from '../errors.js'; +import { DextoRuntimeError } from '@dexto/core'; +import type { DiffDisplayData } from '@dexto/core'; + +// Create mock logger +const createMockLogger = () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + createChild: vi.fn().mockReturnThis(), +}); + +describe('plan_update tool', () => { + let mockLogger: ReturnType; + let tempDir: string; + let planService: PlanService; + + beforeEach(async () => { + mockLogger = createMockLogger(); + + const rawTempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dexto-plan-update-test-')); + tempDir = await fs.realpath(rawTempDir); + + planService = new PlanService({ basePath: tempDir }, mockLogger as any); + + vi.clearAllMocks(); + }); + + afterEach(async () => { + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('generatePreview', () => { + it('should return DiffDisplayData with unified diff', async () => { + const tool = createPlanUpdateTool(planService); + const sessionId = 'test-session'; + const originalContent = '# Plan\n\n## Steps\n1. First step'; + const newContent = '# Plan\n\n## Steps\n1. First step\n2. Second step'; + + await planService.create(sessionId, originalContent); + + const preview = (await tool.generatePreview!( + { content: newContent }, + { sessionId } + )) as DiffDisplayData; + + expect(preview.type).toBe('diff'); + // Path is now absolute, check it ends with the expected suffix + expect(preview.filename).toContain(sessionId); + expect(preview.filename).toMatch(/plan\.md$/); + expect(preview.unified).toContain('-1. First step'); + expect(preview.unified).toContain('+1. First step'); + expect(preview.unified).toContain('+2. Second step'); + expect(preview.additions).toBeGreaterThan(0); + }); + + it('should throw error when plan does not exist', async () => { + const tool = createPlanUpdateTool(planService); + const sessionId = 'test-session'; + + try { + await tool.generatePreview!({ content: '# New Content' }, { sessionId }); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.PLAN_NOT_FOUND); + } + }); + + it('should throw error when sessionId is missing', async () => { + const tool = createPlanUpdateTool(planService); + + try { + await tool.generatePreview!({ content: '# Content' }, {}); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.SESSION_ID_REQUIRED); + } + }); + + it('should show deletions in diff', async () => { + const tool = createPlanUpdateTool(planService); + const sessionId = 'test-session'; + const originalContent = '# Plan\n\nLine to remove\nKeep this'; + const newContent = '# Plan\n\nKeep this'; + + await planService.create(sessionId, originalContent); + + const preview = (await tool.generatePreview!( + { content: newContent }, + { sessionId } + )) as DiffDisplayData; + + expect(preview.deletions).toBeGreaterThan(0); + expect(preview.unified).toContain('-Line to remove'); + }); + }); + + describe('execute', () => { + it('should update plan content and return success', async () => { + const tool = createPlanUpdateTool(planService); + const sessionId = 'test-session'; + const originalContent = '# Original Plan'; + const newContent = '# Updated Plan'; + + await planService.create(sessionId, originalContent); + + const result = (await tool.execute({ content: newContent }, { sessionId })) as { + success: boolean; + path: string; + status: string; + }; + + expect(result.success).toBe(true); + // Path is now absolute, check it ends with the expected suffix + expect(result.path).toContain(sessionId); + expect(result.path).toMatch(/plan\.md$/); + + // Verify content was updated + const plan = await planService.read(sessionId); + expect(plan!.content).toBe(newContent); + }); + + it('should include _display data with diff', async () => { + const tool = createPlanUpdateTool(planService); + const sessionId = 'test-session'; + + await planService.create(sessionId, '# Original'); + + const result = (await tool.execute({ content: '# Updated' }, { sessionId })) as { + _display: DiffDisplayData; + }; + + expect(result._display).toBeDefined(); + expect(result._display.type).toBe('diff'); + expect(result._display.unified).toContain('-# Original'); + expect(result._display.unified).toContain('+# Updated'); + }); + + it('should throw error when plan does not exist', async () => { + const tool = createPlanUpdateTool(planService); + const sessionId = 'non-existent'; + + try { + await tool.execute({ content: '# Content' }, { sessionId }); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.PLAN_NOT_FOUND); + } + }); + + it('should throw error when sessionId is missing', async () => { + const tool = createPlanUpdateTool(planService); + + try { + await tool.execute({ content: '# Content' }, {}); + expect.fail('Should have thrown an error'); + } catch (error) { + expect(error).toBeInstanceOf(DextoRuntimeError); + expect((error as DextoRuntimeError).code).toBe(PlanErrorCode.SESSION_ID_REQUIRED); + } + }); + + it('should preserve plan status after update', async () => { + const tool = createPlanUpdateTool(planService); + const sessionId = 'test-session'; + + await planService.create(sessionId, '# Plan'); + await planService.updateMeta(sessionId, { status: 'approved' }); + + await tool.execute({ content: '# Updated Plan' }, { sessionId }); + + const plan = await planService.read(sessionId); + expect(plan!.meta.status).toBe('approved'); + }); + }); +}); diff --git a/dexto/packages/tools-plan/src/tools/plan-update-tool.ts b/dexto/packages/tools-plan/src/tools/plan-update-tool.ts new file mode 100644 index 00000000..c4ecd350 --- /dev/null +++ b/dexto/packages/tools-plan/src/tools/plan-update-tool.ts @@ -0,0 +1,97 @@ +/** + * Plan Update Tool + * + * Updates the implementation plan for the current session. + * Shows a diff preview for approval before saving. + */ + +import { z } from 'zod'; +import { createPatch } from 'diff'; +import type { InternalTool, ToolExecutionContext, DiffDisplayData } from '@dexto/core'; +import type { PlanService } from '../plan-service.js'; +import { PlanError } from '../errors.js'; + +const PlanUpdateInputSchema = z + .object({ + content: z.string().describe('Updated plan content in markdown format'), + }) + .strict(); + +type PlanUpdateInput = z.input; + +/** + * Generate diff preview for plan update + */ +function generateDiffPreview( + filePath: string, + originalContent: string, + newContent: string +): DiffDisplayData { + const unified = createPatch(filePath, originalContent, newContent, 'before', 'after', { + context: 3, + }); + const additions = (unified.match(/^\+[^+]/gm) || []).length; + const deletions = (unified.match(/^-[^-]/gm) || []).length; + + return { + type: 'diff', + unified, + filename: filePath, + additions, + deletions, + }; +} + +/** + * Creates the plan_update tool + */ +export function createPlanUpdateTool(planService: PlanService): InternalTool { + return { + id: 'plan_update', + description: + 'Update the existing implementation plan for this session. Shows a diff preview for approval before saving. The plan must already exist (use plan_create first).', + inputSchema: PlanUpdateInputSchema, + + /** + * Generate diff preview for approval UI + */ + generatePreview: async ( + input: unknown, + context?: ToolExecutionContext + ): Promise => { + const { content: newContent } = input as PlanUpdateInput; + + if (!context?.sessionId) { + throw PlanError.sessionIdRequired(); + } + + // Read existing plan + const existing = await planService.read(context.sessionId); + if (!existing) { + throw PlanError.planNotFound(context.sessionId); + } + + // Generate diff preview + const planPath = planService.getPlanPath(context.sessionId); + return generateDiffPreview(planPath, existing.content, newContent); + }, + + execute: async (input: unknown, context?: ToolExecutionContext) => { + const { content } = input as PlanUpdateInput; + + if (!context?.sessionId) { + throw PlanError.sessionIdRequired(); + } + + const result = await planService.update(context.sessionId, content); + const planPath = planService.getPlanPath(context.sessionId); + + return { + success: true, + path: planPath, + status: result.meta.status, + _display: generateDiffPreview(planPath, result.oldContent, result.newContent), + }; + }, + }; +} diff --git a/dexto/packages/tools-plan/src/types.ts b/dexto/packages/tools-plan/src/types.ts new file mode 100644 index 00000000..0bacb55c --- /dev/null +++ b/dexto/packages/tools-plan/src/types.ts @@ -0,0 +1,58 @@ +/** + * Plan Types and Schemas + * + * Defines the structure of plans and their metadata. + */ + +import { z } from 'zod'; + +/** + * Plan status values + */ +export const PlanStatusSchema = z.enum([ + 'draft', + 'approved', + 'in_progress', + 'completed', + 'abandoned', +]); + +export type PlanStatus = z.infer; + +/** + * Plan metadata stored alongside the plan content + */ +export const PlanMetaSchema = z.object({ + sessionId: z.string().describe('Session ID this plan belongs to'), + status: PlanStatusSchema.default('draft').describe('Current plan status'), + title: z.string().optional().describe('Plan title'), + createdAt: z.number().describe('Unix timestamp when plan was created'), + updatedAt: z.number().describe('Unix timestamp when plan was last updated'), +}); + +export type PlanMeta = z.infer; + +/** + * Complete plan with content and metadata + */ +export interface Plan { + content: string; + meta: PlanMeta; +} + +/** + * Options for the plan service + */ +export interface PlanServiceOptions { + /** Base directory for plan storage */ + basePath: string; +} + +/** + * Result of a plan update operation + */ +export interface PlanUpdateResult { + oldContent: string; + newContent: string; + meta: PlanMeta; +} diff --git a/dexto/packages/tools-plan/tsconfig.json b/dexto/packages/tools-plan/tsconfig.json new file mode 100644 index 00000000..01aae715 --- /dev/null +++ b/dexto/packages/tools-plan/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true, + "declarationMap": true, + "skipLibCheck": true + }, + "include": ["src/**/*"], + "exclude": ["dist", "node_modules"] +} diff --git a/dexto/packages/tools-plan/vitest.setup.ts b/dexto/packages/tools-plan/vitest.setup.ts new file mode 100644 index 00000000..e2252e5e --- /dev/null +++ b/dexto/packages/tools-plan/vitest.setup.ts @@ -0,0 +1,5 @@ +import { config } from 'dotenv'; + +// Load .env file for integration tests +// This ensures environment variables are available during test execution +config(); diff --git a/dexto/packages/tools-process/CHANGELOG.md b/dexto/packages/tools-process/CHANGELOG.md new file mode 100644 index 00000000..5dab0ecf --- /dev/null +++ b/dexto/packages/tools-process/CHANGELOG.md @@ -0,0 +1,94 @@ +# @dexto/tools-process + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +- Updated dependencies [042f4f0] + - @dexto/core@1.5.6 + +## 1.5.5 + +### Patch Changes + +- 6df3ca9: Updated readme. Removed stale filesystem and process tool from dexto/core. +- Updated dependencies [63fa083] +- Updated dependencies [6df3ca9] + - @dexto/core@1.5.5 + +## 1.5.4 + +### Patch Changes + +- aa2c9a0: - new --dev flag for using dev mode with the CLI (for maintainers) (sets DEXTO_DEV_MODE=true and ensures local files are used) + - improved bash tool descriptions + - fixed explore agent task description getting truncated + - fixed some alignment issues + - fix search/find tools not asking approval for working outside directory + - add sound feature (sounds when approval reqd, when loop done) + - configurable in `preferences.yml` (on by default) and in `~/.dexto/sounds`, instructions in comment in `~/.dexto/preferences.yml` + - add new `env` system prompt contributor that includes info about os, working directory, git status. useful for coding agent to get enough context to improve cmd construction without unnecessary directory shifts + - support for loading `.claude/commands` and `.cursor/commands` global and local commands in addition to `.dexto/commands` +- Updated dependencies [0016cd3] +- Updated dependencies [499b890] +- Updated dependencies [aa2c9a0] + - @dexto/core@1.5.4 + +## 1.5.3 + +### Patch Changes + +- Updated dependencies [4f00295] +- Updated dependencies [69c944c] + - @dexto/core@1.5.3 + +## 1.5.2 + +### Patch Changes + +- Updated dependencies [8a85ea4] +- Updated dependencies [527f3f9] + - @dexto/core@1.5.2 + +## 1.5.1 + +### Patch Changes + +- Updated dependencies [bfcc7b1] +- Updated dependencies [4aabdb7] + - @dexto/core@1.5.1 + +## 1.5.0 + +### Minor Changes + +- e7722e5: Minor version bump for new release with bundler, custom tool pkgs, etc. + +### Patch Changes + +- 1e7e974: Added image bundler, @dexto/image-local and moved tool services outside core. Added registry providers to select core services. +- Updated dependencies [ee12727] +- Updated dependencies [1e7e974] +- Updated dependencies [4c05310] +- Updated dependencies [5fa79fa] +- Updated dependencies [ef40e60] +- Updated dependencies [e714418] +- Updated dependencies [e7722e5] +- Updated dependencies [7d5ab19] +- Updated dependencies [436a900] + - @dexto/core@1.5.0 diff --git a/dexto/packages/tools-process/package.json b/dexto/packages/tools-process/package.json new file mode 100644 index 00000000..f136d812 --- /dev/null +++ b/dexto/packages/tools-process/package.json @@ -0,0 +1,38 @@ +{ + "name": "@dexto/tools-process", + "version": "1.5.6", + "description": "Process tools provider for Dexto agents", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsup", + "typecheck": "tsc --noEmit", + "clean": "rm -rf dist" + }, + "keywords": [ + "dexto", + "tools", + "process", + "bash", + "shell" + ], + "dependencies": { + "@dexto/core": "workspace:*", + "zod": "^3.25.0" + }, + "devDependencies": { + "tsup": "^8.0.0", + "typescript": "^5.3.3" + }, + "files": [ + "dist", + "README.md" + ] +} diff --git a/dexto/packages/tools-process/src/bash-exec-tool.ts b/dexto/packages/tools-process/src/bash-exec-tool.ts new file mode 100644 index 00000000..ada3194a --- /dev/null +++ b/dexto/packages/tools-process/src/bash-exec-tool.ts @@ -0,0 +1,192 @@ +/** + * Bash Execute Tool + * + * Internal tool for executing shell commands. + * Approval is handled at the ToolManager level with pattern-based approval. + */ + +import * as path from 'node:path'; +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext } from '@dexto/core'; +import { ProcessService } from './process-service.js'; +import { ProcessError } from './errors.js'; +import type { ShellDisplayData } from '@dexto/core'; + +const BashExecInputSchema = z + .object({ + command: z.string().describe('Shell command to execute'), + description: z + .string() + .optional() + .describe('Human-readable description of what the command does (5-10 words)'), + timeout: z + .number() + .int() + .positive() + .max(600000) + .optional() + .default(120000) + .describe( + 'Timeout in milliseconds (max: 600000 = 10 minutes, default: 120000 = 2 minutes)' + ), + run_in_background: z + .boolean() + .optional() + .default(false) + .describe('Execute command in background (default: false)'), + cwd: z.string().optional().describe('Working directory for command execution (optional)'), + }) + .strict(); + +type BashExecInput = z.input; + +/** + * Create the bash_exec internal tool + */ +export function createBashExecTool(processService: ProcessService): InternalTool { + return { + id: 'bash_exec', + description: `Execute a shell command in the project root directory. + +IMPORTANT: This tool is for terminal operations like git, npm, docker, etc. Do NOT use it for file operations - use the specialized tools instead: +- File search: Use glob_files (NOT find or ls) +- Content search: Use grep_content (NOT grep or rg) +- Read files: Use read_file (NOT cat/head/tail) +- Edit files: Use edit_file (NOT sed/awk) +- Write files: Use write_file (NOT echo/cat with heredoc) + +Before executing the command, follow these steps: + +1. Directory Verification: + - If the command will create new directories or files, first use ls to verify the parent directory exists and is the correct location + - For example, before running "mkdir foo/bar", first use ls foo to check that "foo" exists + +2. Command Execution: + - Always quote file paths that contain spaces with double quotes + - Examples of proper quoting: + - cd "/Users/name/My Documents" (correct) + - cd /Users/name/My Documents (incorrect - will fail) + - python "/path/with spaces/script.py" (correct) + - python /path/with spaces/script.py (incorrect - will fail) + +Usage notes: +- The command argument is required. +- You can specify an optional timeout in milliseconds (max 600000ms / 10 minutes). Default is 120000ms (2 minutes). +- The description parameter should be a clear, concise summary of what the command does (5-10 words for simple commands, more context for complex commands). +- If the output exceeds 1MB, it will be truncated. +- You can use run_in_background=true to run the command in the background. Use this when you don't need the result immediately. You do not need to check the output right away - use bash_output to retrieve results later. Commands ending with & are blocked; use run_in_background instead. + +When issuing multiple commands: +- If the commands are independent and can run in parallel, make multiple bash_exec calls in a single response. +- If the commands depend on each other and must run sequentially, use a single call with && to chain them (e.g., git add . && git commit -m "msg" && git push). +- Use ; only when you need to run commands sequentially but don't care if earlier commands fail. +- Do NOT use newlines to separate commands (newlines are ok in quoted strings). + +Try to maintain your working directory throughout the session by using absolute paths and avoiding usage of cd. You may use cd if the user explicitly requests it. +- GOOD: pnpm test +- GOOD: pytest /absolute/path/to/tests +- BAD: cd /project && pnpm test +- BAD: cd /some/path && command + +Each command runs in a fresh shell, so cd does not persist between calls. + +Security: Dangerous commands are blocked. Injection attempts are detected. Requires approval with pattern-based session memory.`, + inputSchema: BashExecInputSchema, + + /** + * Generate preview for approval UI - shows the command to be executed + */ + generatePreview: async (input: unknown, _context?: ToolExecutionContext) => { + const { command, run_in_background } = input as BashExecInput; + + const preview: ShellDisplayData = { + type: 'shell', + command, + exitCode: 0, // Placeholder - not executed yet + duration: 0, // Placeholder - not executed yet + ...(run_in_background !== undefined && { isBackground: run_in_background }), + }; + return preview; + }, + + execute: async (input: unknown, context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { command, description, timeout, run_in_background, cwd } = + input as BashExecInput; + + // Validate cwd to prevent path traversal + let validatedCwd: string | undefined = cwd; + if (cwd) { + const baseDir = processService.getConfig().workingDirectory || process.cwd(); + + // Resolve cwd to absolute path + const candidatePath = path.isAbsolute(cwd) + ? path.resolve(cwd) + : path.resolve(baseDir, cwd); + + // Check if cwd is within the base directory + const relativePath = path.relative(baseDir, candidatePath); + const isOutsideBase = + relativePath.startsWith('..') || path.isAbsolute(relativePath); + + if (isOutsideBase) { + throw ProcessError.invalidWorkingDirectory( + cwd, + `Working directory must be within ${baseDir}` + ); + } + + validatedCwd = candidatePath; + } + + // Execute command using ProcessService + // Note: Approval is handled at ToolManager level with pattern-based approval + const result = await processService.executeCommand(command, { + description, + timeout, + runInBackground: run_in_background, + cwd: validatedCwd, + // Pass abort signal for cancellation support + abortSignal: context?.abortSignal, + }); + + // Type guard: if result has 'stdout', it's a ProcessResult (foreground) + // Otherwise it's a ProcessHandle (background) + if ('stdout' in result) { + // Foreground execution result + const _display: ShellDisplayData = { + type: 'shell', + command, + exitCode: result.exitCode, + duration: result.duration, + isBackground: false, + stdout: result.stdout, + stderr: result.stderr, + }; + + return { + stdout: result.stdout, + stderr: result.stderr, + exit_code: result.exitCode, + duration: result.duration, + _display, + }; + } else { + // Background execution handle + const _display: ShellDisplayData = { + type: 'shell', + command, + exitCode: 0, // Background process hasn't exited yet + duration: 0, // Still running + isBackground: true, + }; + + return { + process_id: result.processId, + message: `Command started in background with ID: ${result.processId}. Use bash_output to retrieve output.`, + _display, + }; + } + }, + }; +} diff --git a/dexto/packages/tools-process/src/bash-output-tool.ts b/dexto/packages/tools-process/src/bash-output-tool.ts new file mode 100644 index 00000000..05a7c453 --- /dev/null +++ b/dexto/packages/tools-process/src/bash-output-tool.ts @@ -0,0 +1,44 @@ +/** + * Bash Output Tool + * + * Internal tool for retrieving output from background processes + */ + +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext } from '@dexto/core'; +import { ProcessService } from './process-service.js'; + +const BashOutputInputSchema = z + .object({ + process_id: z.string().describe('Process ID from bash_exec (when run_in_background=true)'), + }) + .strict(); + +type BashOutputInput = z.input; + +/** + * Create the bash_output internal tool + */ +export function createBashOutputTool(processService: ProcessService): InternalTool { + return { + id: 'bash_output', + description: + 'Retrieve output from a background process started with bash_exec. Returns stdout, stderr, status (running/completed/failed), exit code, and duration. Each call returns only new output since last read. The output buffer is cleared after reading. Use this tool to monitor long-running commands.', + inputSchema: BashOutputInputSchema, + execute: async (input: unknown, _context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { process_id } = input as BashOutputInput; + + // Get output from ProcessService + const result = await processService.getProcessOutput(process_id); + + return { + stdout: result.stdout, + stderr: result.stderr, + status: result.status, + ...(result.exitCode !== undefined && { exit_code: result.exitCode }), + ...(result.duration !== undefined && { duration: result.duration }), + }; + }, + }; +} diff --git a/dexto/packages/tools-process/src/command-validator.ts b/dexto/packages/tools-process/src/command-validator.ts new file mode 100644 index 00000000..b1c43fa0 --- /dev/null +++ b/dexto/packages/tools-process/src/command-validator.ts @@ -0,0 +1,472 @@ +/** + * Command Validator + * + * Security-focused command validation for process execution + */ + +import { ProcessConfig, CommandValidation } from './types.js'; +import type { IDextoLogger } from '@dexto/core'; + +const MAX_COMMAND_LENGTH = 10000; // 10K characters + +// Dangerous command patterns that should be blocked +// Validated against common security vulnerabilities and dangerous command patterns +const DANGEROUS_PATTERNS = [ + // File system destruction + /rm\s+-rf\s+\//, // rm -rf / + /rm\s+-rf\s+\/\s*$/, // rm -rf / (end of line) + /rm\s+-rf\s+\/\s*2/, // rm -rf / 2>/dev/null (with error suppression) + + // Fork bomb variations + /:\(\)\{\s*:\|:&\s*\};:/, // Classic fork bomb + /:\(\)\{\s*:\|:&\s*\};/, // Fork bomb without final colon + /:\(\)\{\s*:\|:&\s*\}/, // Fork bomb without semicolon + + // Disk operations + /dd\s+if=.*of=\/dev\//, // dd to disk devices + /dd\s+if=\/dev\/zero.*of=\/dev\//, // dd zero to disk + /dd\s+if=\/dev\/urandom.*of=\/dev\//, // dd random to disk + />\s*\/dev\/sd[a-z]/, // Write to disk devices + />>\s*\/dev\/sd[a-z]/, // Append to disk devices + + // Filesystem operations + /mkfs\./, // Format filesystem + /mkfs\s+/, // Format filesystem with space + /fdisk\s+\/dev\/sd[a-z]/, // Partition disk + /parted\s+\/dev\/sd[a-z]/, // Partition disk with parted + + // Download and execute patterns + /wget.*\|\s*sh/, // wget | sh + /wget.*\|\s*bash/, // wget | bash + /curl.*\|\s*sh/, // curl | sh + /curl.*\|\s*bash/, // curl | bash + /wget.*\|\s*python/, // wget | python + /curl.*\|\s*python/, // curl | python + + // Shell execution + /\|\s*bash/, // Pipe to bash + /\|\s*sh/, // Pipe to sh + /\|\s*zsh/, // Pipe to zsh + /\|\s*fish/, // Pipe to fish + + // Command evaluation + /eval\s+\$\(/, // eval $() + /eval\s+`/, // eval backticks + /eval\s+"/, // eval double quotes + /eval\s+'/, // eval single quotes + + // Permission changes + /chmod\s+777\s+\//, // chmod 777 / + /chmod\s+777\s+\/\s*$/, // chmod 777 / (end of line) + /chmod\s+-R\s+777\s+\//, // chmod -R 777 / + /chown\s+-R\s+root\s+\//, // chown -R root / + + // Network operations + /nc\s+-l\s+-p\s+\d+/, // netcat listener + /ncat\s+-l\s+-p\s+\d+/, // ncat listener + /socat\s+.*LISTEN/, // socat listener + + // Process manipulation + /killall\s+-9/, // killall -9 + /pkill\s+-9/, // pkill -9 + /kill\s+-9\s+-1/, // kill -9 -1 (kill all processes) + + // System shutdown/reboot + /shutdown\s+now/, // shutdown now + /reboot/, // reboot + /halt/, // halt + /poweroff/, // poweroff + + // Memory operations + /echo\s+3\s*>\s*\/proc\/sys\/vm\/drop_caches/, // Clear page cache + /sync\s*;\s*echo\s+3\s*>\s*\/proc\/sys\/vm\/drop_caches/, // Sync and clear cache + + // Network interface manipulation + /ifconfig\s+.*down/, // Bring interface down + /ip\s+link\s+set\s+.*down/, // Bring interface down with ip + + // Package manager operations + /apt\s+remove\s+--purge\s+.*/, // Remove packages + /yum\s+remove\s+.*/, // Remove packages + /dnf\s+remove\s+.*/, // Remove packages + /pacman\s+-R\s+.*/, // Remove packages +]; + +// Command injection patterns +// Note: We don't block compound commands with && here, as they're handled by +// the compound command detection logic in determineApprovalRequirement() +const INJECTION_PATTERNS = [ + // Command chaining with dangerous commands using semicolon (more suspicious) + /;\s*rm\s+-rf/, // ; rm -rf + /;\s*chmod\s+777/, // ; chmod 777 + /;\s*chown\s+root/, // ; chown root + + // Command substitution with dangerous commands + /`.*rm.*`/, // backticks with rm + /\$\(.*rm.*\)/, // $() with rm + /`.*chmod.*`/, // backticks with chmod + /\$\(.*chmod.*\)/, // $() with chmod + /`.*chown.*`/, // backticks with chown + /\$\(.*chown.*\)/, // $() with chown + + // Multiple command separators + /;\s*;\s*/, // Multiple semicolons + /&&\s*&&\s*/, // Multiple && operators + /\|\|\s*\|\|\s*/, // Multiple || operators + + // Redirection with dangerous commands + /rm\s+.*>\s*\/dev\/null/, // rm with output redirection + /chmod\s+.*>\s*\/dev\/null/, // chmod with output redirection + /chown\s+.*>\s*\/dev\/null/, // chown with output redirection + + // Environment variable manipulation + /\$[A-Z_]+\s*=\s*.*rm/, // Environment variable with rm + /\$[A-Z_]+\s*=\s*.*chmod/, // Environment variable with chmod + /\$[A-Z_]+\s*=\s*.*chown/, // Environment variable with chown +]; + +// Commands that require approval +const REQUIRES_APPROVAL_PATTERNS = [ + // File operations + /^rm\s+/, // rm (removal) + /^mv\s+/, // move files + /^cp\s+/, // copy files + /^chmod\s+/, // chmod + /^chown\s+/, // chown + /^chgrp\s+/, // chgrp + /^ln\s+/, // create links + /^unlink\s+/, // unlink files + + // Git operations + /^git\s+push/, // git push + /^git\s+commit/, // git commit + /^git\s+reset/, // git reset + /^git\s+rebase/, // git rebase + /^git\s+merge/, // git merge + /^git\s+checkout/, // git checkout + /^git\s+branch/, // git branch + /^git\s+tag/, // git tag + + // Package management + /^npm\s+publish/, // npm publish + /^npm\s+uninstall/, // npm uninstall + /^yarn\s+publish/, // yarn publish + /^yarn\s+remove/, // yarn remove + /^pip\s+install/, // pip install + /^pip\s+uninstall/, // pip uninstall + /^apt\s+install/, // apt install + /^apt\s+remove/, // apt remove + /^yum\s+install/, // yum install + /^yum\s+remove/, // yum remove + /^dnf\s+install/, // dnf install + /^dnf\s+remove/, // dnf remove + /^pacman\s+-S/, // pacman install + /^pacman\s+-R/, // pacman remove + + // Container operations + /^docker\s+/, // docker commands + /^podman\s+/, // podman commands + /^kubectl\s+/, // kubectl commands + + // System operations + /^sudo\s+/, // sudo commands + /^su\s+/, // su commands + /^systemctl\s+/, // systemctl commands + /^service\s+/, // service commands + /^mount\s+/, // mount commands + /^umount\s+/, // umount commands + /^fdisk\s+/, // fdisk commands + /^parted\s+/, // parted commands + /^mkfs\s+/, // mkfs commands + /^fsck\s+/, // fsck commands + + // Network operations + /^iptables\s+/, // iptables commands + /^ufw\s+/, // ufw commands + /^firewall-cmd\s+/, // firewall-cmd commands + /^sshd\s+/, // sshd commands + /^ssh\s+/, // ssh commands + /^scp\s+/, // scp commands + /^rsync\s+/, // rsync commands + + // Process management + /^kill\s+/, // kill commands + /^killall\s+/, // killall commands + /^pkill\s+/, // pkill commands + /^nohup\s+/, // nohup commands + /^screen\s+/, // screen commands + /^tmux\s+/, // tmux commands + + // Database operations + /^mysql\s+/, // mysql commands + /^psql\s+/, // psql commands + /^sqlite3\s+/, // sqlite3 commands + /^mongodb\s+/, // mongodb commands + /^redis-cli\s+/, // redis-cli commands +]; + +// Safe command patterns for strict mode +const SAFE_PATTERNS = [ + // Directory navigation with commands + /^cd\s+.*&&\s+\w+/, // cd && command + /^cd\s+.*;\s+\w+/, // cd ; command + + // Safe pipe operations + /\|\s*grep/, // | grep + /\|\s*head/, // | head + /\|\s*tail/, // | tail + /\|\s*sort/, // | sort + /\|\s*uniq/, // | uniq + /\|\s*wc/, // | wc + /\|\s*cat/, // | cat + /\|\s*less/, // | less + /\|\s*more/, // | more + /\|\s*awk/, // | awk + /\|\s*sed/, // | sed + /\|\s*cut/, // | cut + /\|\s*tr/, // | tr + /\|\s*xargs/, // | xargs + + // Safe redirection + /^ls\s+.*>/, // ls with output redirection + /^find\s+.*>/, // find with output redirection + /^grep\s+.*>/, // grep with output redirection + /^cat\s+.*>/, // cat with output redirection +]; + +// Write operation patterns for moderate mode +const WRITE_PATTERNS = [ + // Output redirection + />/, // output redirection + />>/, // append redirection + /2>/, // error redirection + /2>>/, // error append redirection + /&>/, // both output and error redirection + /&>>/, // both output and error append redirection + + // File operations + /tee\s+/, // tee command + /touch\s+/, // touch command + /mkdir\s+/, // mkdir command + /rmdir\s+/, // rmdir command + + // Text editors + /vim\s+/, // vim command + /nano\s+/, // nano command + /emacs\s+/, // emacs command + /code\s+/, // code command (VS Code) + + // File copying and moving + /cp\s+/, // cp command + /mv\s+/, // mv command + /scp\s+/, // scp command + /rsync\s+/, // rsync command +]; + +/** + * CommandValidator - Validates commands for security and policy compliance + * + * Security checks: + * 1. Command length limits + * 2. Dangerous command patterns + * 3. Command injection detection + * 4. Allowed/blocked command lists + * 5. Shell metacharacter analysis + * TODO: Add tests for this class + */ +export class CommandValidator { + private config: ProcessConfig; + private logger: IDextoLogger; + + constructor(config: ProcessConfig, logger: IDextoLogger) { + this.config = config; + this.logger = logger; + this.logger.debug( + `CommandValidator initialized with security level: ${config.securityLevel}` + ); + } + + /** + * Validate a command for security and policy compliance + */ + validateCommand(command: string): CommandValidation { + // 1. Check for empty command + if (!command || command.trim() === '') { + return { + isValid: false, + error: 'Command cannot be empty', + }; + } + + const trimmedCommand = command.trim(); + + // 2. Check for shell backgrounding (trailing &) + // This bypasses timeout and creates orphaned processes that can't be controlled + if (/&\s*$/.test(trimmedCommand)) { + return { + isValid: false, + error: 'Commands ending with & (shell backgrounding) are not allowed. Use run_in_background parameter instead for proper process management.', + }; + } + + // 3. Check command length + if (trimmedCommand.length > MAX_COMMAND_LENGTH) { + return { + isValid: false, + error: `Command too long: ${trimmedCommand.length} characters. Maximum: ${MAX_COMMAND_LENGTH}`, + }; + } + + // 4. Check against dangerous patterns (strict and moderate) + if (this.config.securityLevel !== 'permissive') { + for (const pattern of DANGEROUS_PATTERNS) { + if (pattern.test(trimmedCommand)) { + return { + isValid: false, + error: `Command matches dangerous pattern: ${pattern.source}`, + }; + } + } + } + + // 5. Check for command injection attempts (all security levels) + const injectionResult = this.detectInjection(trimmedCommand); + if (!injectionResult.isValid) { + return injectionResult; + } + + // 6. Check against blocked commands list + for (const blockedPattern of this.config.blockedCommands) { + if (trimmedCommand.includes(blockedPattern)) { + return { + isValid: false, + error: `Command is blocked: matches "${blockedPattern}"`, + }; + } + } + + // 7. Check against allowed commands list (if not empty) + if (this.config.allowedCommands.length > 0) { + const isAllowed = this.config.allowedCommands.some((allowedCmd) => + trimmedCommand.startsWith(allowedCmd) + ); + + if (!isAllowed) { + return { + isValid: false, + error: `Command not in allowed list. Allowed: ${this.config.allowedCommands.join(', ')}`, + }; + } + } + + // 8. Determine if approval is required based on security level + const requiresApproval = this.determineApprovalRequirement(trimmedCommand); + + return { + isValid: true, + normalizedCommand: trimmedCommand, + requiresApproval, + }; + } + + /** + * Detect command injection attempts + */ + private detectInjection(command: string): CommandValidation { + // Check for obvious injection patterns + for (const pattern of INJECTION_PATTERNS) { + if (pattern.test(command)) { + return { + isValid: false, + error: `Potential command injection detected: ${pattern.source}`, + }; + } + } + + // In strict mode, be more aggressive + if (this.config.securityLevel === 'strict') { + // Check for multiple commands chained together (except safe ones) + const hasMultipleCommands = /;|\|{1,2}|&&/.test(command); + if (hasMultipleCommands) { + // Allow safe patterns like "cd dir && ls" or "command | grep pattern" + const isSafe = SAFE_PATTERNS.some((pattern) => pattern.test(command)); + if (!isSafe) { + return { + isValid: false, + error: 'Multiple commands detected in strict mode. Use moderate or permissive mode if this is intentional.', + }; + } + } + } + + return { + isValid: true, + }; + } + + /** + * Determine if a command requires approval + * Handles compound commands (with &&, ||, ;) by checking each sub-command + */ + private determineApprovalRequirement(command: string): boolean { + // Split compound commands by &&, ||, or ; to check each part independently + // This ensures dangerous operations in the middle of compound commands are detected + const subCommands = command.split(/\s*(?:&&|\|\||;)\s*/).map((cmd) => cmd.trim()); + + // Check if ANY sub-command requires approval + for (const subCmd of subCommands) { + if (!subCmd) continue; // Skip empty parts + + // Strip leading shell keywords and braces to get the actual command + // This prevents bypassing approval checks via control-flow wrapping + const normalizedSubCmd = subCmd + .replace(/^(?:then|do|else)\b\s*/, '') + .replace(/^\{\s*/, '') + .trim(); + if (!normalizedSubCmd) continue; + + // Commands that modify system state always require approval + for (const pattern of REQUIRES_APPROVAL_PATTERNS) { + if (pattern.test(normalizedSubCmd)) { + return true; + } + } + + // In strict mode, all commands require approval + if (this.config.securityLevel === 'strict') { + return true; + } + + // In moderate mode, write operations require approval + if (this.config.securityLevel === 'moderate') { + if (WRITE_PATTERNS.some((pattern) => pattern.test(normalizedSubCmd))) { + return true; + } + } + } + + // Permissive mode - no additional approval required + return false; + } + + /** + * Get list of blocked commands + */ + getBlockedCommands(): string[] { + return [...this.config.blockedCommands]; + } + + /** + * Get list of allowed commands + */ + getAllowedCommands(): string[] { + return [...this.config.allowedCommands]; + } + + /** + * Get security level + */ + getSecurityLevel(): string { + return this.config.securityLevel; + } +} diff --git a/dexto/packages/tools-process/src/error-codes.ts b/dexto/packages/tools-process/src/error-codes.ts new file mode 100644 index 00000000..f1dc6166 --- /dev/null +++ b/dexto/packages/tools-process/src/error-codes.ts @@ -0,0 +1,32 @@ +/** + * Process Service Error Codes + * + * Standardized error codes for process execution and management + */ + +export enum ProcessErrorCode { + // Command validation errors + INVALID_COMMAND = 'PROCESS_INVALID_COMMAND', + COMMAND_BLOCKED = 'PROCESS_COMMAND_BLOCKED', + COMMAND_TOO_LONG = 'PROCESS_COMMAND_TOO_LONG', + INJECTION_DETECTED = 'PROCESS_INJECTION_DETECTED', + APPROVAL_REQUIRED = 'PROCESS_APPROVAL_REQUIRED', + APPROVAL_DENIED = 'PROCESS_APPROVAL_DENIED', + + // Execution errors + EXECUTION_FAILED = 'PROCESS_EXECUTION_FAILED', + TIMEOUT = 'PROCESS_TIMEOUT', + PERMISSION_DENIED = 'PROCESS_PERMISSION_DENIED', + COMMAND_NOT_FOUND = 'PROCESS_COMMAND_NOT_FOUND', + WORKING_DIRECTORY_INVALID = 'PROCESS_WORKING_DIRECTORY_INVALID', + + // Process management errors + PROCESS_NOT_FOUND = 'PROCESS_NOT_FOUND', + TOO_MANY_PROCESSES = 'PROCESS_TOO_MANY_PROCESSES', + KILL_FAILED = 'PROCESS_KILL_FAILED', + OUTPUT_BUFFER_FULL = 'PROCESS_OUTPUT_BUFFER_FULL', + + // Configuration errors + INVALID_CONFIG = 'PROCESS_INVALID_CONFIG', + SERVICE_NOT_INITIALIZED = 'PROCESS_SERVICE_NOT_INITIALIZED', +} diff --git a/dexto/packages/tools-process/src/errors.ts b/dexto/packages/tools-process/src/errors.ts new file mode 100644 index 00000000..cde2ecf9 --- /dev/null +++ b/dexto/packages/tools-process/src/errors.ts @@ -0,0 +1,254 @@ +/** + * Process Service Errors + * + * Error classes for process execution and management + */ + +import { DextoRuntimeError, ErrorType } from '@dexto/core'; + +/** Error scope for process operations */ +const PROCESS_SCOPE = 'process'; +import { ProcessErrorCode } from './error-codes.js'; + +export interface ProcessErrorContext { + command?: string; + processId?: string; + timeout?: number; + [key: string]: unknown; +} + +/** + * Factory class for creating Process-related errors + */ +export class ProcessError { + private constructor() { + // Private constructor prevents instantiation + } + + /** + * Invalid command error + */ + static invalidCommand(command: string, reason: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.INVALID_COMMAND, + PROCESS_SCOPE, + ErrorType.USER, + `Invalid command: ${command}. ${reason}`, + { command, reason } + ); + } + + /** + * Command blocked error + */ + static commandBlocked(command: string, reason: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.COMMAND_BLOCKED, + PROCESS_SCOPE, + ErrorType.FORBIDDEN, + `Command is blocked: ${command}. ${reason}`, + { command, reason } + ); + } + + /** + * Command too long error + */ + static commandTooLong(length: number, maxLength: number): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.COMMAND_TOO_LONG, + PROCESS_SCOPE, + ErrorType.USER, + `Command too long: ${length} characters. Maximum allowed: ${maxLength}`, + { length, maxLength } + ); + } + + /** + * Command injection detected error + */ + static commandInjection(command: string, pattern: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.INJECTION_DETECTED, + PROCESS_SCOPE, + ErrorType.FORBIDDEN, + `Potential command injection detected in: ${command}. Pattern: ${pattern}`, + { command, pattern } + ); + } + + /** + * Command approval required error + */ + static approvalRequired(command: string, reason?: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.APPROVAL_REQUIRED, + PROCESS_SCOPE, + ErrorType.FORBIDDEN, + `Command requires approval: ${command}${reason ? `. ${reason}` : ''}`, + { command, reason }, + 'Provide an approval function to execute dangerous commands' + ); + } + + /** + * Command approval denied error + */ + static approvalDenied(command: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.APPROVAL_DENIED, + PROCESS_SCOPE, + ErrorType.FORBIDDEN, + `Command approval denied by user: ${command}`, + { command } + ); + } + + /** + * Command execution failed error + */ + static executionFailed(command: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.EXECUTION_FAILED, + PROCESS_SCOPE, + ErrorType.SYSTEM, + `Command execution failed: ${command}. ${cause}`, + { command, cause } + ); + } + + /** + * Command timeout error + */ + static timeout(command: string, timeout: number): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.TIMEOUT, + PROCESS_SCOPE, + ErrorType.TIMEOUT, + `Command timed out after ${timeout}ms: ${command}`, + { command, timeout }, + 'Increase timeout or optimize the command' + ); + } + + /** + * Permission denied error + */ + static permissionDenied(command: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.PERMISSION_DENIED, + PROCESS_SCOPE, + ErrorType.FORBIDDEN, + `Permission denied: ${command}`, + { command } + ); + } + + /** + * Command not found error + */ + static commandNotFound(command: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.COMMAND_NOT_FOUND, + PROCESS_SCOPE, + ErrorType.NOT_FOUND, + `Command not found: ${command}`, + { command }, + 'Ensure the command is installed and available in PATH' + ); + } + + /** + * Invalid working directory error + */ + static invalidWorkingDirectory(path: string, reason: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.WORKING_DIRECTORY_INVALID, + PROCESS_SCOPE, + ErrorType.USER, + `Invalid working directory: ${path}. ${reason}`, + { path, reason } + ); + } + + /** + * Process not found error + */ + static processNotFound(processId: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.PROCESS_NOT_FOUND, + PROCESS_SCOPE, + ErrorType.NOT_FOUND, + `Process not found: ${processId}`, + { processId } + ); + } + + /** + * Too many concurrent processes error + */ + static tooManyProcesses(current: number, max: number): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.TOO_MANY_PROCESSES, + PROCESS_SCOPE, + ErrorType.USER, + `Too many concurrent processes: ${current}. Maximum allowed: ${max}`, + { current, max }, + 'Wait for running processes to complete or increase the limit' + ); + } + + /** + * Kill process failed error + */ + static killFailed(processId: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.KILL_FAILED, + PROCESS_SCOPE, + ErrorType.SYSTEM, + `Failed to kill process ${processId}: ${cause}`, + { processId, cause } + ); + } + + /** + * Output buffer full error + */ + static outputBufferFull(processId: string, size: number, maxSize: number): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.OUTPUT_BUFFER_FULL, + PROCESS_SCOPE, + ErrorType.SYSTEM, + `Output buffer full for process ${processId}: ${size} bytes. Maximum: ${maxSize}`, + { processId, size, maxSize }, + 'Process output exceeded buffer limit' + ); + } + + /** + * Invalid configuration error + */ + static invalidConfig(reason: string): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.INVALID_CONFIG, + PROCESS_SCOPE, + ErrorType.USER, + `Invalid Process configuration: ${reason}`, + { reason } + ); + } + + /** + * Service not initialized error + */ + static notInitialized(): DextoRuntimeError { + return new DextoRuntimeError( + ProcessErrorCode.SERVICE_NOT_INITIALIZED, + PROCESS_SCOPE, + ErrorType.SYSTEM, + 'ProcessService has not been initialized', + {}, + 'Initialize the ProcessService before using it' + ); + } +} diff --git a/dexto/packages/tools-process/src/index.ts b/dexto/packages/tools-process/src/index.ts new file mode 100644 index 00000000..3602e590 --- /dev/null +++ b/dexto/packages/tools-process/src/index.ts @@ -0,0 +1,32 @@ +/** + * @dexto/tools-process + * + * Process tools provider for Dexto agents. + * Provides process operation tools: bash exec, output, kill. + */ + +// Main provider export +export { processToolsProvider } from './tool-provider.js'; + +// Service and utilities (for advanced use cases) +export { ProcessService } from './process-service.js'; +export { CommandValidator } from './command-validator.js'; +export { ProcessError } from './errors.js'; +export { ProcessErrorCode } from './error-codes.js'; + +// Types +export type { + ProcessConfig, + ExecuteOptions, + ProcessResult, + ProcessHandle, + ProcessOutput, + ProcessInfo, + CommandValidation, + OutputBuffer, +} from './types.js'; + +// Tool implementations (for custom integrations) +export { createBashExecTool } from './bash-exec-tool.js'; +export { createBashOutputTool } from './bash-output-tool.js'; +export { createKillProcessTool } from './kill-process-tool.js'; diff --git a/dexto/packages/tools-process/src/kill-process-tool.ts b/dexto/packages/tools-process/src/kill-process-tool.ts new file mode 100644 index 00000000..4c596416 --- /dev/null +++ b/dexto/packages/tools-process/src/kill-process-tool.ts @@ -0,0 +1,43 @@ +/** + * Kill Process Tool + * + * Internal tool for terminating background processes + */ + +import { z } from 'zod'; +import { InternalTool, ToolExecutionContext } from '@dexto/core'; +import { ProcessService } from './process-service.js'; + +const KillProcessInputSchema = z + .object({ + process_id: z.string().describe('Process ID of the background process to terminate'), + }) + .strict(); + +type KillProcessInput = z.input; + +/** + * Create the kill_process internal tool + */ +export function createKillProcessTool(processService: ProcessService): InternalTool { + return { + id: 'kill_process', + description: + "Terminate a background process started with bash_exec. Sends SIGTERM signal first, then SIGKILL if process doesn't terminate within 5 seconds. Only works on processes started by this agent. Returns success status and whether the process was running. Does not require additional approval (process was already approved when started).", + inputSchema: KillProcessInputSchema, + execute: async (input: unknown, _context?: ToolExecutionContext) => { + // Input is validated by provider before reaching here + const { process_id } = input as KillProcessInput; + + // Kill process using ProcessService + await processService.killProcess(process_id); + + // Note: killProcess returns void and doesn't throw if process already stopped + return { + success: true, + process_id, + message: `Termination signal sent to process ${process_id}`, + }; + }, + }; +} diff --git a/dexto/packages/tools-process/src/process-service.ts b/dexto/packages/tools-process/src/process-service.ts new file mode 100644 index 00000000..9f8ff280 --- /dev/null +++ b/dexto/packages/tools-process/src/process-service.ts @@ -0,0 +1,688 @@ +/** + * Process Service + * + * Secure command execution and process management for Dexto internal tools + */ + +import { spawn, ChildProcess } from 'node:child_process'; +import * as crypto from 'node:crypto'; +import * as path from 'node:path'; +import { + ProcessConfig, + ExecuteOptions, + ProcessResult, + ProcessHandle, + ProcessOutput, + ProcessInfo, + OutputBuffer, +} from './types.js'; +import { CommandValidator } from './command-validator.js'; +import { ProcessError } from './errors.js'; +import type { IDextoLogger } from '@dexto/core'; +import { DextoLogComponent } from '@dexto/core'; + +const DEFAULT_TIMEOUT = 120000; // 2 minutes + +/** + * Background process tracking + */ +interface BackgroundProcess { + processId: string; + command: string; + child: ChildProcess; + startedAt: Date; + completedAt?: Date | undefined; + status: 'running' | 'completed' | 'failed'; + exitCode?: number | undefined; + outputBuffer: OutputBuffer; + description?: string | undefined; +} + +/** + * ProcessService - Handles command execution and process management + * + * This service receives fully-validated configuration from the Process Tools Provider. + * All defaults have been applied by the provider's schema, so the service trusts the config + * and uses it as-is without any fallback logic. + * + * TODO: Add tests for this class + */ +export class ProcessService { + private config: ProcessConfig; + private commandValidator: CommandValidator; + private initialized: boolean = false; + private initPromise: Promise | null = null; + private backgroundProcesses: Map = new Map(); + private logger: IDextoLogger; + + /** + * Create a new ProcessService with validated configuration. + * + * @param config - Fully-validated configuration from provider schema. + * All required fields have values, defaults already applied. + * @param logger - Logger instance for this service + */ + constructor(config: ProcessConfig, logger: IDextoLogger) { + // Config is already fully validated with defaults applied - just use it + this.config = config; + + this.logger = logger.createChild(DextoLogComponent.PROCESS); + this.commandValidator = new CommandValidator(this.config, this.logger); + } + + /** + * Initialize the service. + * Safe to call multiple times - subsequent calls return the same promise. + */ + initialize(): Promise { + if (this.initPromise) { + return this.initPromise; + } + + this.initPromise = this.doInitialize(); + return this.initPromise; + } + + /** + * Internal initialization logic. + */ + private async doInitialize(): Promise { + if (this.initialized) { + this.logger.debug('ProcessService already initialized'); + return; + } + + // Clean up any stale processes on startup + this.backgroundProcesses.clear(); + + this.initialized = true; + this.logger.info('ProcessService initialized successfully'); + } + + /** + * Ensure the service is initialized before use. + * Tools should call this at the start of their execute methods. + * Safe to call multiple times - will await the same initialization promise. + */ + async ensureInitialized(): Promise { + if (this.initialized) { + return; + } + await this.initialize(); + } + + /** + * Execute a command + */ + async executeCommand( + command: string, + options: ExecuteOptions = {} + ): Promise { + await this.ensureInitialized(); + + // Validate command + const validation = this.commandValidator.validateCommand(command); + if (!validation.isValid || !validation.normalizedCommand) { + throw ProcessError.invalidCommand(command, validation.error || 'Unknown error'); + } + + const normalizedCommand = validation.normalizedCommand; + + // Note: Command-level approval removed - approval is now handled at the tool level + // in ToolManager with pattern-based approval for bash commands. + // CommandValidator still validates for dangerous patterns (blocks truly dangerous commands) + // but no longer triggers a second approval prompt. + + // Handle timeout - clamp to valid range to prevent negative/NaN/invalid values + const rawTimeout = + options.timeout !== undefined && Number.isFinite(options.timeout) + ? options.timeout + : DEFAULT_TIMEOUT; + const timeout = Math.max(1, Math.min(rawTimeout, this.config.maxTimeout)); + + // Setup working directory + const cwd: string = this.resolveSafeCwd(options.cwd); + + // Setup environment - filter out undefined values + const env: Record = {}; + for (const [key, value] of Object.entries({ + ...process.env, + ...this.config.environment, + ...options.env, + })) { + if (value !== undefined) { + env[key] = value; + } + } + + // If running in background, return the process handle directly + if (options.runInBackground) { + return await this.executeInBackground(normalizedCommand, options); + } + + // Execute command in foreground + return await this.executeForeground(normalizedCommand, { + cwd, + timeout, + env, + ...(options.description !== undefined && { description: options.description }), + ...(options.abortSignal !== undefined && { abortSignal: options.abortSignal }), + }); + } + + private static readonly SIGKILL_TIMEOUT_MS = 200; + + /** + * Kill a process tree (process group on Unix, taskkill on Windows) + */ + private async killProcessTree(pid: number, child: ChildProcess): Promise { + if (process.platform === 'win32') { + // Windows: use taskkill with /t flag to kill process tree + await new Promise((resolve) => { + const killer = spawn('taskkill', ['/pid', String(pid), '/f', '/t'], { + stdio: 'ignore', + }); + killer.once('exit', () => resolve()); + killer.once('error', () => resolve()); + }); + } else { + // Unix: kill process group using negative PID + try { + process.kill(-pid, 'SIGTERM'); + await new Promise((res) => setTimeout(res, ProcessService.SIGKILL_TIMEOUT_MS)); + if (child.exitCode === null) { + process.kill(-pid, 'SIGKILL'); + } + } catch { + // Fallback to killing just the process if group kill fails + child.kill('SIGTERM'); + await new Promise((res) => setTimeout(res, ProcessService.SIGKILL_TIMEOUT_MS)); + if (child.exitCode === null) { + child.kill('SIGKILL'); + } + } + } + } + + /** + * Execute command in foreground with timeout and abort support + */ + private executeForeground( + command: string, + options: { + cwd: string; + timeout: number; + env: Record; + description?: string; + abortSignal?: AbortSignal; + } + ): Promise { + return new Promise((resolve, reject) => { + const startTime = Date.now(); + let stdout = ''; + let stderr = ''; + let stdoutBytes = 0; + let stderrBytes = 0; + let outputTruncated = false; + let killed = false; + let aborted = false; + let closed = false; + const maxBuffer = this.config.maxOutputBuffer; + + // Check if already aborted before starting + if (options.abortSignal?.aborted) { + this.logger.debug(`Command cancelled before execution: ${command}`); + resolve({ + stdout: '', + stderr: '(Command was cancelled)', + exitCode: 130, // Standard exit code for SIGINT + duration: 0, + }); + return; + } + + this.logger.debug(`Executing command: ${command}`); + + // Spawn process with shell and detached for process group support (Unix) + const child = spawn(command, { + cwd: options.cwd, + env: options.env, + shell: true, + detached: process.platform !== 'win32', // Create process group on Unix + }); + + // Setup timeout + const timeoutHandle = setTimeout(() => { + killed = true; + if (child.pid) { + void this.killProcessTree(child.pid, child); + } else { + child.kill('SIGTERM'); + } + }, options.timeout); + + // Setup abort handler + const abortHandler = () => { + if (closed) return; + aborted = true; + this.logger.debug(`Command cancelled by user: ${command}`); + clearTimeout(timeoutHandle); + if (child.pid) { + void this.killProcessTree(child.pid, child); + } else { + child.kill('SIGTERM'); + } + }; + + options.abortSignal?.addEventListener('abort', abortHandler, { once: true }); + + // Collect stdout with buffer limit + child.stdout?.on('data', (data) => { + if (outputTruncated) return; // Ignore further data after truncation + + const chunk = data.toString(); + const chunkBytes = Buffer.byteLength(chunk, 'utf8'); + + if (stdoutBytes + stderrBytes + chunkBytes <= maxBuffer) { + stdout += chunk; + stdoutBytes += chunkBytes; + } else { + // Add remaining bytes up to limit, then truncate + const remaining = maxBuffer - stdoutBytes - stderrBytes; + if (remaining > 0) { + stdout += chunk.slice(0, remaining); + stdoutBytes += remaining; + } + stdout += '\n...[truncated]'; + outputTruncated = true; + this.logger.warn(`Output buffer full for command: ${command}`); + } + }); + + // Collect stderr with buffer limit + child.stderr?.on('data', (data) => { + if (outputTruncated) return; // Ignore further data after truncation + + const chunk = data.toString(); + const chunkBytes = Buffer.byteLength(chunk, 'utf8'); + + if (stdoutBytes + stderrBytes + chunkBytes <= maxBuffer) { + stderr += chunk; + stderrBytes += chunkBytes; + } else { + // Add remaining bytes up to limit, then truncate + const remaining = maxBuffer - stdoutBytes - stderrBytes; + if (remaining > 0) { + stderr += chunk.slice(0, remaining); + stderrBytes += remaining; + } + stderr += '\n...[truncated]'; + outputTruncated = true; + this.logger.warn(`Output buffer full for command: ${command}`); + } + }); + + // Handle completion + child.on('close', (code, signal) => { + closed = true; + clearTimeout(timeoutHandle); + options.abortSignal?.removeEventListener('abort', abortHandler); + const duration = Date.now() - startTime; + + // Handle abort - return result instead of rejecting + if (aborted) { + stdout += '\n\n(Command was cancelled)'; + this.logger.debug(`Command cancelled after ${duration}ms: ${command}`); + resolve({ + stdout, + stderr, + exitCode: 130, // Standard exit code for SIGINT + duration, + }); + return; + } + + if (killed) { + reject(ProcessError.timeout(command, options.timeout)); + return; + } + + let exitCode = typeof code === 'number' ? code : 1; + if (code === null) { + stderr += `\nProcess terminated by signal ${signal ?? 'UNKNOWN'}`; + } + + this.logger.debug( + `Command completed with exit code ${exitCode} in ${duration}ms: ${command}` + ); + + resolve({ + stdout, + stderr, + exitCode, + duration, + }); + }); + + // Handle errors + child.on('error', (error) => { + clearTimeout(timeoutHandle); + options.abortSignal?.removeEventListener('abort', abortHandler); + + // Check for specific error types + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + reject(ProcessError.commandNotFound(command)); + } else if ((error as NodeJS.ErrnoException).code === 'EACCES') { + reject(ProcessError.permissionDenied(command)); + } else { + reject(ProcessError.executionFailed(command, error.message)); + } + }); + }); + } + + /** + * Execute command in background + */ + private async executeInBackground( + command: string, + options: ExecuteOptions + ): Promise { + // Check concurrent process limit + const runningCount = Array.from(this.backgroundProcesses.values()).filter( + (p) => p.status === 'running' + ).length; + + if (runningCount >= this.config.maxConcurrentProcesses) { + throw ProcessError.tooManyProcesses(runningCount, this.config.maxConcurrentProcesses); + } + + // Generate unique process ID + const processId = crypto.randomBytes(4).toString('hex'); + + // Setup working directory + const cwd: string = this.resolveSafeCwd(options.cwd); + + // Setup environment - filter out undefined values + const env: Record = {}; + for (const [key, value] of Object.entries({ + ...process.env, + ...this.config.environment, + ...options.env, + })) { + if (value !== undefined) { + env[key] = value; + } + } + + this.logger.debug(`Starting background process ${processId}: ${command}`); + + // Spawn process + const child = spawn(command, { + cwd, + env, + shell: true, + detached: false, + }); + + // Create output buffer + const outputBuffer: OutputBuffer = { + stdout: [], + stderr: [], + complete: false, + lastRead: Date.now(), + bytesUsed: 0, + truncated: false, + }; + + // Track background process + const bgProcess: BackgroundProcess = { + processId, + command, + child, + startedAt: new Date(), + status: 'running', + outputBuffer, + description: options.description, + }; + + this.backgroundProcesses.set(processId, bgProcess); + + // Enforce background timeout + const bgTimeout = Math.max( + 1, + Math.min(options.timeout || DEFAULT_TIMEOUT, this.config.maxTimeout) + ); + let killEscalationTimer: ReturnType | null = null; + const killTimer = setTimeout(() => { + if (bgProcess.status === 'running') { + this.logger.warn( + `Background process ${processId} timed out after ${bgTimeout}ms, sending SIGTERM` + ); + child.kill('SIGTERM'); + // Escalate to SIGKILL if process doesn't terminate within 5s + killEscalationTimer = setTimeout(() => { + if (bgProcess.status === 'running') { + this.logger.warn( + `Background process ${processId} did not respond to SIGTERM, sending SIGKILL` + ); + child.kill('SIGKILL'); + } + }, 5000); + } + }, bgTimeout); + + // bytesUsed is kept on outputBuffer for correct accounting across reads + + // Setup output collection with buffer limit + child.stdout?.on('data', (data) => { + const chunk = data.toString(); + const chunkBytes = Buffer.byteLength(chunk, 'utf8'); + + if (outputBuffer.bytesUsed + chunkBytes <= this.config.maxOutputBuffer) { + outputBuffer.stdout.push(chunk); + outputBuffer.bytesUsed += chunkBytes; + } else { + if (!outputBuffer.truncated) { + outputBuffer.truncated = true; + this.logger.warn(`Output buffer full for process ${processId}`); + } + } + }); + + child.stderr?.on('data', (data) => { + const chunk = data.toString(); + const chunkBytes = Buffer.byteLength(chunk, 'utf8'); + + if (outputBuffer.bytesUsed + chunkBytes <= this.config.maxOutputBuffer) { + outputBuffer.stderr.push(chunk); + outputBuffer.bytesUsed += chunkBytes; + } else { + if (!outputBuffer.truncated) { + outputBuffer.truncated = true; + this.logger.warn(`Error buffer full for process ${processId}`); + } + } + }); + + // Handle completion + child.on('close', (code) => { + clearTimeout(killTimer); + if (killEscalationTimer) clearTimeout(killEscalationTimer); + bgProcess.status = code === 0 ? 'completed' : 'failed'; + bgProcess.exitCode = code ?? undefined; + bgProcess.completedAt = new Date(); + bgProcess.outputBuffer.complete = true; + + this.logger.debug(`Background process ${processId} completed with exit code ${code}`); + }); + + // Handle errors + child.on('error', (error) => { + clearTimeout(killTimer); + if (killEscalationTimer) clearTimeout(killEscalationTimer); + bgProcess.status = 'failed'; + bgProcess.completedAt = new Date(); + bgProcess.outputBuffer.complete = true; + const chunk = `Error: ${error.message}`; + const chunkBytes = Buffer.byteLength(chunk, 'utf8'); + if (bgProcess.outputBuffer.bytesUsed + chunkBytes <= this.config.maxOutputBuffer) { + bgProcess.outputBuffer.stderr.push(chunk); + bgProcess.outputBuffer.bytesUsed += chunkBytes; + } else { + if (!bgProcess.outputBuffer.truncated) { + bgProcess.outputBuffer.truncated = true; + this.logger.warn(`Error buffer full for process ${processId}`); + } + } + + this.logger.error(`Background process ${processId} failed: ${error.message}`); + }); + + return { + processId, + command, + pid: child.pid, + startedAt: bgProcess.startedAt, + description: options.description, + }; + } + + /** + * Get output from a background process + */ + async getProcessOutput(processId: string): Promise { + await this.ensureInitialized(); + + const bgProcess = this.backgroundProcesses.get(processId); + if (!bgProcess) { + throw ProcessError.processNotFound(processId); + } + + // Get new output since last read + const stdout = bgProcess.outputBuffer.stdout.join(''); + const stderr = bgProcess.outputBuffer.stderr.join(''); + + // Clear the buffer (data has been read) and reset byte counter + bgProcess.outputBuffer.stdout = []; + bgProcess.outputBuffer.stderr = []; + bgProcess.outputBuffer.lastRead = Date.now(); + bgProcess.outputBuffer.bytesUsed = 0; + + return { + stdout, + stderr, + status: bgProcess.status, + exitCode: bgProcess.exitCode, + duration: bgProcess.completedAt + ? bgProcess.completedAt.getTime() - bgProcess.startedAt.getTime() + : undefined, + }; + } + + /** + * Kill a background process + */ + async killProcess(processId: string): Promise { + await this.ensureInitialized(); + + const bgProcess = this.backgroundProcesses.get(processId); + if (!bgProcess) { + throw ProcessError.processNotFound(processId); + } + + if (bgProcess.status !== 'running') { + this.logger.debug(`Process ${processId} is not running (status: ${bgProcess.status})`); + return; // Already completed + } + + try { + bgProcess.child.kill('SIGTERM'); + + // Force kill after timeout + setTimeout(() => { + // Escalate based on actual process state, not our status flag + if (bgProcess.child.exitCode === null) { + bgProcess.child.kill('SIGKILL'); + } + }, 5000); + + this.logger.debug(`Process ${processId} sent SIGTERM`); + } catch (error) { + throw ProcessError.killFailed( + processId, + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * List all background processes + */ + async listProcesses(): Promise { + await this.ensureInitialized(); + + return Array.from(this.backgroundProcesses.values()).map((bgProcess) => ({ + processId: bgProcess.processId, + command: bgProcess.command, + pid: bgProcess.child.pid, + status: bgProcess.status, + startedAt: bgProcess.startedAt, + completedAt: bgProcess.completedAt, + exitCode: bgProcess.exitCode, + description: bgProcess.description, + })); + } + + /** + * Get buffer size in bytes + */ + private getBufferSize(buffer: OutputBuffer): number { + const stdoutSize = buffer.stdout.reduce((sum, line) => sum + line.length, 0); + const stderrSize = buffer.stderr.reduce((sum, line) => sum + line.length, 0); + return stdoutSize + stderrSize; + } + + /** + * Get service configuration + */ + getConfig(): Readonly { + return { ...this.config }; + } + + /** + * Resolve and confine cwd to the configured working directory + */ + private resolveSafeCwd(cwd?: string): string { + const baseDir = this.config.workingDirectory || process.cwd(); + if (!cwd) return baseDir; + const candidate = path.isAbsolute(cwd) ? path.resolve(cwd) : path.resolve(baseDir, cwd); + const rel = path.relative(baseDir, candidate); + const outside = rel.startsWith('..') || path.isAbsolute(rel); + if (outside) { + throw ProcessError.invalidWorkingDirectory( + cwd, + `Working directory must be within ${baseDir}` + ); + } + return candidate; + } + + /** + * Cleanup completed processes + */ + async cleanup(): Promise { + const now = Date.now(); + const CLEANUP_AGE = 3600000; // 1 hour + + for (const [processId, bgProcess] of this.backgroundProcesses.entries()) { + if (bgProcess.status !== 'running' && bgProcess.completedAt) { + const age = now - bgProcess.completedAt.getTime(); + if (age > CLEANUP_AGE) { + this.backgroundProcesses.delete(processId); + this.logger.debug(`Cleaned up old process ${processId}`); + } + } + } + } +} diff --git a/dexto/packages/tools-process/src/tool-provider.ts b/dexto/packages/tools-process/src/tool-provider.ts new file mode 100644 index 00000000..2a1c4542 --- /dev/null +++ b/dexto/packages/tools-process/src/tool-provider.ts @@ -0,0 +1,161 @@ +/** + * Process Tools Provider + * + * Provides process execution and management tools by wrapping ProcessService. + * When registered, the provider initializes ProcessService and creates tools + * for command execution and process management. + */ + +import { z } from 'zod'; +import type { CustomToolProvider, ToolCreationContext } from '@dexto/core'; +import type { InternalTool } from '@dexto/core'; +import { ProcessService } from './process-service.js'; +import { createBashExecTool } from './bash-exec-tool.js'; +import { createBashOutputTool } from './bash-output-tool.js'; +import { createKillProcessTool } from './kill-process-tool.js'; + +/** + * Default configuration constants for Process tools. + * These are the SINGLE SOURCE OF TRUTH for all default values. + */ +const DEFAULT_SECURITY_LEVEL = 'moderate'; +const DEFAULT_MAX_TIMEOUT = 600000; // 10 minutes +const DEFAULT_MAX_CONCURRENT_PROCESSES = 5; +const DEFAULT_MAX_OUTPUT_BUFFER = 1 * 1024 * 1024; // 1MB +const DEFAULT_ALLOWED_COMMANDS: string[] = []; +const DEFAULT_BLOCKED_COMMANDS: string[] = []; +const DEFAULT_ENVIRONMENT: Record = {}; + +/** + * Configuration schema for Process tools provider. + * + * This is the SINGLE SOURCE OF TRUTH for all configuration: + * - Validation rules + * - Default values (using constants above) + * - Documentation + * - Type definitions + * + * Services receive fully-validated config from this schema and use it as-is, + * with no additional defaults or fallbacks needed. + */ +const ProcessToolsConfigSchema = z + .object({ + type: z.literal('process-tools'), + securityLevel: z + .enum(['strict', 'moderate', 'permissive']) + .default(DEFAULT_SECURITY_LEVEL) + .describe('Security level for command execution validation'), + maxTimeout: z + .number() + .int() + .positive() + .max(DEFAULT_MAX_TIMEOUT) + .default(DEFAULT_MAX_TIMEOUT) + .describe( + `Maximum timeout for commands in milliseconds (max: ${DEFAULT_MAX_TIMEOUT / 1000 / 60} minutes)` + ), + maxConcurrentProcesses: z + .number() + .int() + .positive() + .default(DEFAULT_MAX_CONCURRENT_PROCESSES) + .describe( + `Maximum number of concurrent background processes (default: ${DEFAULT_MAX_CONCURRENT_PROCESSES})` + ), + maxOutputBuffer: z + .number() + .int() + .positive() + .default(DEFAULT_MAX_OUTPUT_BUFFER) + .describe( + `Maximum output buffer size in bytes (default: ${DEFAULT_MAX_OUTPUT_BUFFER / 1024 / 1024}MB)` + ), + workingDirectory: z + .string() + .optional() + .describe('Working directory for process execution (defaults to process.cwd())'), + allowedCommands: z + .array(z.string()) + .default(DEFAULT_ALLOWED_COMMANDS) + .describe( + 'Explicitly allowed commands (empty = all allowed with approval, strict mode only)' + ), + blockedCommands: z + .array(z.string()) + .default(DEFAULT_BLOCKED_COMMANDS) + .describe('Blocked command patterns (applies to all security levels)'), + environment: z + .record(z.string()) + .default(DEFAULT_ENVIRONMENT) + .describe('Custom environment variables to set for command execution'), + timeout: z + .number() + .int() + .positive() + .max(DEFAULT_MAX_TIMEOUT) + .optional() + .describe( + `Default timeout in milliseconds (max: ${DEFAULT_MAX_TIMEOUT / 1000 / 60} minutes)` + ), + }) + .strict(); + +type ProcessToolsConfig = z.output; + +/** + * Process tools provider. + * + * Wraps ProcessService and provides process operation tools: + * - bash_exec: Execute bash commands (foreground or background) + * - bash_output: Retrieve output from background processes + * - kill_process: Terminate background processes + * + * When registered via customToolRegistry, ProcessService is automatically + * initialized and process operation tools become available to the agent. + */ +export const processToolsProvider: CustomToolProvider<'process-tools', ProcessToolsConfig> = { + type: 'process-tools', + configSchema: ProcessToolsConfigSchema, + + create: (config: ProcessToolsConfig, context: ToolCreationContext): InternalTool[] => { + const { logger } = context; + + logger.debug('Creating ProcessService for process tools'); + + // Create ProcessService with validated config + const processService = new ProcessService( + { + securityLevel: config.securityLevel, + maxTimeout: config.maxTimeout, + maxConcurrentProcesses: config.maxConcurrentProcesses, + maxOutputBuffer: config.maxOutputBuffer, + workingDirectory: config.workingDirectory || process.cwd(), + allowedCommands: config.allowedCommands, + blockedCommands: config.blockedCommands, + environment: config.environment, + }, + logger + ); + + // Start initialization in background - service methods use ensureInitialized() for lazy init + // This means tools will wait for initialization to complete before executing + processService.initialize().catch((error) => { + logger.error(`Failed to initialize ProcessService: ${error.message}`); + }); + + logger.debug('ProcessService created - initialization will complete on first tool use'); + + // Create and return all process operation tools + return [ + createBashExecTool(processService), + createBashOutputTool(processService), + createKillProcessTool(processService), + ]; + }, + + metadata: { + displayName: 'Process Tools', + description: 'Process execution and management (bash, output, kill)', + category: 'process', + }, +}; diff --git a/dexto/packages/tools-process/src/types.ts b/dexto/packages/tools-process/src/types.ts new file mode 100644 index 00000000..ee2d8ade --- /dev/null +++ b/dexto/packages/tools-process/src/types.ts @@ -0,0 +1,114 @@ +/** + * Process Service Types + * + * Types and interfaces for command execution and process management + */ + +/** + * Process execution options + */ +export interface ExecuteOptions { + /** Working directory */ + cwd?: string | undefined; + /** Timeout in milliseconds (max: 600000) */ + timeout?: number | undefined; + /** Run command in background */ + runInBackground?: boolean | undefined; + /** Environment variables */ + env?: Record | undefined; + /** Description of what the command does (5-10 words) */ + description?: string | undefined; + /** Abort signal for cancellation support */ + abortSignal?: AbortSignal | undefined; +} + +/** + * Process execution result (foreground execution only) + * For background execution, see ProcessHandle + */ +export interface ProcessResult { + stdout: string; + stderr: string; + exitCode: number; + duration: number; +} + +/** + * Background process handle + */ +export interface ProcessHandle { + processId: string; + command: string; + pid?: number | undefined; // System process ID + startedAt: Date; + description?: string | undefined; +} + +/** + * Process output (for retrieving from background processes) + */ +export interface ProcessOutput { + stdout: string; + stderr: string; + status: 'running' | 'completed' | 'failed'; + exitCode?: number | undefined; + duration?: number | undefined; +} + +/** + * Process information + */ +export interface ProcessInfo { + processId: string; + command: string; + pid?: number | undefined; + status: 'running' | 'completed' | 'failed'; + startedAt: Date; + completedAt?: Date | undefined; + exitCode?: number | undefined; + description?: string | undefined; +} + +/** + * Command validation result + */ +export interface CommandValidation { + isValid: boolean; + error?: string; + normalizedCommand?: string; + requiresApproval?: boolean; +} + +/** + * Process service configuration + */ +export interface ProcessConfig { + /** Security level for command execution */ + securityLevel: 'strict' | 'moderate' | 'permissive'; + /** Maximum timeout for commands in milliseconds */ + maxTimeout: number; + /** Maximum concurrent background processes */ + maxConcurrentProcesses: number; + /** Maximum output buffer size in bytes */ + maxOutputBuffer: number; + /** Explicitly allowed commands (empty = all allowed with approval) */ + allowedCommands: string[]; + /** Blocked command patterns */ + blockedCommands: string[]; + /** Custom environment variables */ + environment: Record; + /** Working directory (defaults to process.cwd()) */ + workingDirectory?: string | undefined; +} + +/** + * Output buffer management + */ +export interface OutputBuffer { + stdout: string[]; + stderr: string[]; + complete: boolean; + lastRead: number; // Timestamp of last read + bytesUsed: number; // Running byte count for O(1) limit checks + truncated?: boolean; // True if content was dropped due to limits +} diff --git a/dexto/packages/tools-process/tsconfig.json b/dexto/packages/tools-process/tsconfig.json new file mode 100644 index 00000000..7f2de0e5 --- /dev/null +++ b/dexto/packages/tools-process/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true, + "declarationMap": true + }, + "include": ["src/**/*"], + "exclude": ["dist", "node_modules"] +} diff --git a/dexto/packages/tools-process/tsup.config.ts b/dexto/packages/tools-process/tsup.config.ts new file mode 100644 index 00000000..45c414d6 --- /dev/null +++ b/dexto/packages/tools-process/tsup.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig([ + { + entry: ['src/**/*.ts'], + format: ['cjs', 'esm'], + outDir: 'dist', + dts: true, + platform: 'node', + bundle: false, + clean: true, + tsconfig: './tsconfig.json', + esbuildOptions(options) { + options.logOverride = { + ...(options.logOverride ?? {}), + 'empty-import-meta': 'silent', + }; + }, + }, +]); diff --git a/dexto/packages/tools-todo/CHANGELOG.md b/dexto/packages/tools-todo/CHANGELOG.md new file mode 100644 index 00000000..d6ed8930 --- /dev/null +++ b/dexto/packages/tools-todo/CHANGELOG.md @@ -0,0 +1,32 @@ +# @dexto/tools-todo + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +- Updated dependencies [042f4f0] + - @dexto/core@1.5.6 + +## 0.1.1 + +### Patch Changes + +- 9ab3eac: Added todo tools. +- Updated dependencies [63fa083] +- Updated dependencies [6df3ca9] + - @dexto/core@1.5.5 diff --git a/dexto/packages/tools-todo/package.json b/dexto/packages/tools-todo/package.json new file mode 100644 index 00000000..383bb397 --- /dev/null +++ b/dexto/packages/tools-todo/package.json @@ -0,0 +1,39 @@ +{ + "name": "@dexto/tools-todo", + "version": "1.5.6", + "description": "Todo/task tracking tools provider for Dexto agents", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsup", + "typecheck": "tsc --noEmit", + "clean": "rm -rf dist" + }, + "keywords": [ + "dexto", + "tools", + "todo", + "task-tracking" + ], + "dependencies": { + "@dexto/core": "workspace:*", + "nanoid": "^5.0.9", + "zod": "^3.25.0" + }, + "devDependencies": { + "tsup": "^8.0.0", + "typescript": "^5.3.3", + "vitest": "^2.1.8" + }, + "files": [ + "dist", + "README.md" + ] +} diff --git a/dexto/packages/tools-todo/src/error-codes.ts b/dexto/packages/tools-todo/src/error-codes.ts new file mode 100644 index 00000000..15bda76f --- /dev/null +++ b/dexto/packages/tools-todo/src/error-codes.ts @@ -0,0 +1,15 @@ +/** + * Todo Service Error Codes + */ + +export enum TodoErrorCode { + // Service lifecycle errors + SERVICE_NOT_INITIALIZED = 'TODO_SERVICE_NOT_INITIALIZED', + + // Todo management errors + TODO_LIMIT_EXCEEDED = 'TODO_LIMIT_EXCEEDED', + INVALID_TODO_STATUS = 'TODO_INVALID_TODO_STATUS', + + // Database errors + DATABASE_ERROR = 'TODO_DATABASE_ERROR', +} diff --git a/dexto/packages/tools-todo/src/errors.ts b/dexto/packages/tools-todo/src/errors.ts new file mode 100644 index 00000000..cbc184b7 --- /dev/null +++ b/dexto/packages/tools-todo/src/errors.ts @@ -0,0 +1,77 @@ +/** + * Todo Service Errors + * + * Error factory for todo list management operations + */ + +import { DextoRuntimeError, ErrorScope, ErrorType } from '@dexto/core'; +import { TodoErrorCode } from './error-codes.js'; + +/** + * Error scope for todo-related errors. + * Uses a custom string since todo is a custom tool, not part of core. + */ +const TODO_ERROR_SCOPE = 'todo'; + +/** + * Factory class for creating Todo-related errors + */ +export class TodoError { + private constructor() { + // Private constructor prevents instantiation + } + + /** + * Service not initialized error + */ + static notInitialized(): DextoRuntimeError { + return new DextoRuntimeError( + TodoErrorCode.SERVICE_NOT_INITIALIZED, + TODO_ERROR_SCOPE, + ErrorType.SYSTEM, + 'TodoService has not been initialized', + {}, + 'Initialize the TodoService before using it' + ); + } + + /** + * Todo limit exceeded error + */ + static todoLimitExceeded(current: number, max: number): DextoRuntimeError { + return new DextoRuntimeError( + TodoErrorCode.TODO_LIMIT_EXCEEDED, + TODO_ERROR_SCOPE, + ErrorType.USER, + `Todo limit exceeded: ${current} todos. Maximum allowed: ${max}`, + { current, max }, + 'Complete or delete existing todos before adding new ones' + ); + } + + /** + * Invalid todo status error + */ + static invalidStatus(status: string): DextoRuntimeError { + return new DextoRuntimeError( + TodoErrorCode.INVALID_TODO_STATUS, + TODO_ERROR_SCOPE, + ErrorType.USER, + `Invalid todo status: ${status}. Must be 'pending', 'in_progress', or 'completed'`, + { status } + ); + } + + /** + * Database error + */ + static databaseError(operation: string, cause: string): DextoRuntimeError { + return new DextoRuntimeError( + TodoErrorCode.DATABASE_ERROR, + TODO_ERROR_SCOPE, + ErrorType.SYSTEM, + `Database error during ${operation}: ${cause}`, + { operation, cause } + ); + } +} diff --git a/dexto/packages/tools-todo/src/index.ts b/dexto/packages/tools-todo/src/index.ts new file mode 100644 index 00000000..09442dfe --- /dev/null +++ b/dexto/packages/tools-todo/src/index.ts @@ -0,0 +1,21 @@ +/** + * @dexto/tools-todo + * + * Todo/task tracking tools provider for Dexto agents. + * Provides the todo_write tool for managing task lists. + */ + +// Main provider export +export { todoToolsProvider } from './tool-provider.js'; + +// Service and utilities (for advanced use cases) +export { TodoService } from './todo-service.js'; +export { TodoError } from './errors.js'; +export { TodoErrorCode } from './error-codes.js'; + +// Types +export type { Todo, TodoInput, TodoStatus, TodoUpdateResult, TodoConfig } from './types.js'; +export { TODO_STATUS_VALUES } from './types.js'; + +// Tool implementations (for custom integrations) +export { createTodoWriteTool } from './todo-write-tool.js'; diff --git a/dexto/packages/tools-todo/src/todo-service.test.ts b/dexto/packages/tools-todo/src/todo-service.test.ts new file mode 100644 index 00000000..f4ce67af --- /dev/null +++ b/dexto/packages/tools-todo/src/todo-service.test.ts @@ -0,0 +1,226 @@ +/** + * TodoService Unit Tests + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { TodoService } from './todo-service.js'; +import type { Database, AgentEventBus, IDextoLogger } from '@dexto/core'; +import type { TodoInput } from './types.js'; + +// Mock database +function createMockDatabase(): Database { + const store = new Map(); + return { + get: vi.fn().mockImplementation(async (key: string) => store.get(key)), + set: vi.fn().mockImplementation(async (key: string, value: unknown) => { + store.set(key, value); + }), + delete: vi.fn().mockImplementation(async (key: string) => { + store.delete(key); + }), + list: vi.fn().mockResolvedValue([]), + append: vi.fn().mockResolvedValue(undefined), + getRange: vi.fn().mockResolvedValue([]), + connect: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + isConnected: vi.fn().mockReturnValue(true), + getStoreType: vi.fn().mockReturnValue('mock'), + } as Database; +} + +// Mock event bus +function createMockEventBus(): AgentEventBus { + return { + emit: vi.fn(), + on: vi.fn().mockReturnThis(), + once: vi.fn().mockReturnThis(), + off: vi.fn().mockReturnThis(), + } as unknown as AgentEventBus; +} + +// Mock logger +function createMockLogger(): IDextoLogger { + return { + debug: vi.fn(), + silly: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trackException: vi.fn(), + createChild: vi.fn(() => createMockLogger()), + setLevel: vi.fn(), + getLevel: vi.fn().mockReturnValue('info'), + getLogFilePath: vi.fn().mockReturnValue(null), + destroy: vi.fn().mockResolvedValue(undefined), + } as unknown as IDextoLogger; +} + +describe('TodoService', () => { + let service: TodoService; + let mockDb: Database; + let mockEventBus: AgentEventBus; + let mockLogger: IDextoLogger; + + beforeEach(async () => { + mockDb = createMockDatabase(); + mockEventBus = createMockEventBus(); + mockLogger = createMockLogger(); + + service = new TodoService(mockDb, mockEventBus, mockLogger); + await service.initialize(); + }); + + describe('initialize', () => { + it('should initialize successfully', async () => { + const newService = new TodoService(mockDb, mockEventBus, mockLogger); + await expect(newService.initialize()).resolves.not.toThrow(); + }); + + it('should be idempotent', async () => { + await expect(service.initialize()).resolves.not.toThrow(); + }); + }); + + describe('updateTodos', () => { + const sessionId = 'test-session'; + + it('should create new todos', async () => { + const todoInputs: TodoInput[] = [ + { content: 'Task 1', activeForm: 'Working on Task 1', status: 'pending' }, + { content: 'Task 2', activeForm: 'Working on Task 2', status: 'in_progress' }, + ]; + + const result = await service.updateTodos(sessionId, todoInputs); + + expect(result.todos).toHaveLength(2); + expect(result.created).toBe(2); + expect(result.updated).toBe(0); + expect(result.deleted).toBe(0); + expect(result.todos[0]?.content).toBe('Task 1'); + expect(result.todos[1]?.content).toBe('Task 2'); + }); + + it('should preserve existing todo IDs when updating', async () => { + // Create initial todos + const initialInputs: TodoInput[] = [ + { content: 'Task 1', activeForm: 'Working on Task 1', status: 'pending' }, + ]; + const initialResult = await service.updateTodos(sessionId, initialInputs); + const originalId = initialResult.todos[0]?.id; + + // Update with same content but different status + const updatedInputs: TodoInput[] = [ + { content: 'Task 1', activeForm: 'Working on Task 1', status: 'completed' }, + ]; + const updatedResult = await service.updateTodos(sessionId, updatedInputs); + + expect(updatedResult.todos[0]?.id).toBe(originalId); + expect(updatedResult.todos[0]?.status).toBe('completed'); + expect(updatedResult.updated).toBe(1); + expect(updatedResult.created).toBe(0); + }); + + it('should track deleted todos', async () => { + // Create initial todos + const initialInputs: TodoInput[] = [ + { content: 'Task 1', activeForm: 'Working on Task 1', status: 'pending' }, + { content: 'Task 2', activeForm: 'Working on Task 2', status: 'pending' }, + ]; + await service.updateTodos(sessionId, initialInputs); + + // Update with only one todo + const updatedInputs: TodoInput[] = [ + { content: 'Task 1', activeForm: 'Working on Task 1', status: 'completed' }, + ]; + const result = await service.updateTodos(sessionId, updatedInputs); + + expect(result.todos).toHaveLength(1); + expect(result.deleted).toBe(1); + }); + + it('should emit service:event when todos are updated', async () => { + const todoInputs: TodoInput[] = [ + { content: 'Task 1', activeForm: 'Working on Task 1', status: 'pending' }, + ]; + + await service.updateTodos(sessionId, todoInputs); + + expect(mockEventBus.emit).toHaveBeenCalledWith('service:event', { + service: 'todo', + event: 'updated', + sessionId, + data: expect.objectContaining({ + todos: expect.any(Array), + stats: expect.objectContaining({ + created: 1, + updated: 0, + deleted: 0, + }), + }), + }); + }); + + it('should respect maxTodosPerSession limit', async () => { + const limitedService = new TodoService(mockDb, mockEventBus, mockLogger, { + maxTodosPerSession: 2, + }); + await limitedService.initialize(); + + const todoInputs: TodoInput[] = [ + { content: 'Task 1', activeForm: 'Working on Task 1', status: 'pending' }, + { content: 'Task 2', activeForm: 'Working on Task 2', status: 'pending' }, + { content: 'Task 3', activeForm: 'Working on Task 3', status: 'pending' }, + ]; + + await expect(limitedService.updateTodos(sessionId, todoInputs)).rejects.toThrow( + /Todo limit exceeded/ + ); + }); + + it('should not emit events when enableEvents is false', async () => { + const silentService = new TodoService(mockDb, mockEventBus, mockLogger, { + enableEvents: false, + }); + await silentService.initialize(); + + const todoInputs: TodoInput[] = [ + { content: 'Task 1', activeForm: 'Working on Task 1', status: 'pending' }, + ]; + + await silentService.updateTodos(sessionId, todoInputs); + + expect(mockEventBus.emit).not.toHaveBeenCalled(); + }); + }); + + describe('getTodos', () => { + const sessionId = 'test-session'; + + it('should return empty array for new session', async () => { + const todos = await service.getTodos(sessionId); + expect(todos).toEqual([]); + }); + + it('should return todos after update', async () => { + const todoInputs: TodoInput[] = [ + { content: 'Task 1', activeForm: 'Working on Task 1', status: 'pending' }, + ]; + await service.updateTodos(sessionId, todoInputs); + + const todos = await service.getTodos(sessionId); + expect(todos).toHaveLength(1); + expect(todos[0]?.content).toBe('Task 1'); + }); + }); + + describe('error handling', () => { + it('should throw if not initialized', async () => { + const uninitService = new TodoService(mockDb, mockEventBus, mockLogger); + + await expect(uninitService.getTodos('session')).rejects.toThrow(/not been initialized/); + await expect(uninitService.updateTodos('session', [])).rejects.toThrow( + /not been initialized/ + ); + }); + }); +}); diff --git a/dexto/packages/tools-todo/src/todo-service.ts b/dexto/packages/tools-todo/src/todo-service.ts new file mode 100644 index 00000000..d36dff3e --- /dev/null +++ b/dexto/packages/tools-todo/src/todo-service.ts @@ -0,0 +1,209 @@ +/** + * Todo Service + * + * Manages todo lists for tracking agent workflow and task progress. + * Emits events through the AgentEventBus using the service:event pattern. + */ + +import { nanoid } from 'nanoid'; +import type { Database, AgentEventBus, IDextoLogger } from '@dexto/core'; +import { DextoRuntimeError } from '@dexto/core'; +import { TodoError } from './errors.js'; +import type { Todo, TodoInput, TodoUpdateResult, TodoConfig, TodoStatus } from './types.js'; +import { TODO_STATUS_VALUES } from './types.js'; + +const DEFAULT_MAX_TODOS = 100; +const TODOS_KEY_PREFIX = 'todos:'; + +/** + * TodoService - Manages todo lists for agent workflow tracking + */ +export class TodoService { + private database: Database; + private eventBus: AgentEventBus; + private logger: IDextoLogger; + private config: Required; + private initialized: boolean = false; + + constructor( + database: Database, + eventBus: AgentEventBus, + logger: IDextoLogger, + config: TodoConfig = {} + ) { + this.database = database; + this.eventBus = eventBus; + this.logger = logger; + this.config = { + maxTodosPerSession: config.maxTodosPerSession ?? DEFAULT_MAX_TODOS, + enableEvents: config.enableEvents ?? true, + }; + } + + /** + * Initialize the service + */ + async initialize(): Promise { + if (this.initialized) { + this.logger.debug('TodoService already initialized'); + return; + } + + this.initialized = true; + this.logger.info('TodoService initialized successfully'); + } + + /** + * Update todos for a session (replaces entire list) + */ + async updateTodos(sessionId: string, todoInputs: TodoInput[]): Promise { + if (!this.initialized) { + throw TodoError.notInitialized(); + } + + // Validate todo count + if (todoInputs.length > this.config.maxTodosPerSession) { + throw TodoError.todoLimitExceeded(todoInputs.length, this.config.maxTodosPerSession); + } + + try { + // Get existing todos + const existing = await this.getTodos(sessionId); + const existingMap = new Map(existing.map((t) => [this.getTodoKey(t), t])); + + // Create new todos with IDs + const now = new Date(); + const newTodos: Todo[] = []; + const stats = { created: 0, updated: 0, deleted: 0 }; + + for (let i = 0; i < todoInputs.length; i++) { + const input = todoInputs[i]!; + this.validateTodoStatus(input.status); + + // Generate consistent key for matching + const todoKey = this.getTodoKeyFromInput(input); + const existingTodo = existingMap.get(todoKey); + + if (existingTodo) { + // Update existing todo + const updated: Todo = { + ...existingTodo, + status: input.status, + updatedAt: now, + position: i, + }; + newTodos.push(updated); + stats.updated++; + existingMap.delete(todoKey); + } else { + // Create new todo + const created: Todo = { + id: nanoid(), + sessionId, + content: input.content, + activeForm: input.activeForm, + status: input.status, + position: i, + createdAt: now, + updatedAt: now, + }; + newTodos.push(created); + stats.created++; + } + } + + // Remaining items in existingMap are deleted + stats.deleted = existingMap.size; + + // Save to database + const key = this.getTodosDatabaseKey(sessionId); + await this.database.set(key, newTodos); + + // Emit event using the service:event pattern + if (this.config.enableEvents) { + this.eventBus.emit('service:event', { + service: 'todo', + event: 'updated', + sessionId, + data: { + todos: newTodos, + stats, + }, + }); + } + + this.logger.debug( + `Updated todos for session ${sessionId}: ${stats.created} created, ${stats.updated} updated, ${stats.deleted} deleted` + ); + + return { + todos: newTodos, + sessionId, + ...stats, + }; + } catch (error) { + if (error instanceof DextoRuntimeError) { + throw error; + } + throw TodoError.databaseError( + 'updateTodos', + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Get todos for a session + */ + async getTodos(sessionId: string): Promise { + if (!this.initialized) { + throw TodoError.notInitialized(); + } + + try { + const key = this.getTodosDatabaseKey(sessionId); + const todos = await this.database.get(key); + return todos || []; + } catch (error) { + if (error instanceof DextoRuntimeError) { + throw error; + } + throw TodoError.databaseError( + 'getTodos', + error instanceof Error ? error.message : String(error) + ); + } + } + + /** + * Generate database key for session todos + */ + private getTodosDatabaseKey(sessionId: string): string { + return `${TODOS_KEY_PREFIX}${sessionId}`; + } + + /** + * Generate consistent key for todo matching (content + activeForm) + * Uses JSON encoding to prevent collisions when fields contain delimiters + */ + private getTodoKey(todo: Todo | TodoInput): string { + return JSON.stringify([todo.content, todo.activeForm]); + } + + /** + * Generate key from TodoInput + * Uses JSON encoding to prevent collisions when fields contain delimiters + */ + private getTodoKeyFromInput(input: TodoInput): string { + return JSON.stringify([input.content, input.activeForm]); + } + + /** + * Validate todo status + */ + private validateTodoStatus(status: TodoStatus): void { + if (!TODO_STATUS_VALUES.includes(status)) { + throw TodoError.invalidStatus(status); + } + } +} diff --git a/dexto/packages/tools-todo/src/todo-write-tool.ts b/dexto/packages/tools-todo/src/todo-write-tool.ts new file mode 100644 index 00000000..be2fa796 --- /dev/null +++ b/dexto/packages/tools-todo/src/todo-write-tool.ts @@ -0,0 +1,95 @@ +/** + * Todo Write Tool + * + * Manages task lists for tracking agent progress and workflow organization + */ + +import { z } from 'zod'; +import type { InternalTool, ToolExecutionContext } from '@dexto/core'; +import type { TodoService } from './todo-service.js'; +import { TODO_STATUS_VALUES } from './types.js'; + +/** + * Zod schema for todo item input + */ +const TodoItemSchema = z + .object({ + content: z + .string() + .min(1) + .describe('Task description in imperative form (e.g., "Fix authentication bug")'), + activeForm: z + .string() + .min(1) + .describe( + 'Present continuous form shown during execution (e.g., "Fixing authentication bug")' + ), + status: z + .enum(TODO_STATUS_VALUES) + .describe( + 'Task status: pending (not started), in_progress (currently working), completed (finished)' + ), + }) + .strict(); + +/** + * Zod schema for todo_write tool input + */ +const TodoWriteInputSchema = z + .object({ + todos: z + .array(TodoItemSchema) + .min(1) + .describe('Array of todo items representing the complete task list for the session'), + }) + .strict() + .superRefine((value, ctx) => { + const inProgressCount = value.todos.filter((todo) => todo.status === 'in_progress').length; + if (inProgressCount > 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Only one todo may be in_progress at a time.', + path: ['todos'], + }); + } + }) + .describe( + 'Manage task list for current session. Replaces the entire todo list with the provided tasks.' + ); + +/** + * Create todo_write internal tool + */ +export function createTodoWriteTool(todoService: TodoService): InternalTool { + return { + id: 'todo_write', + description: `Track progress on multi-step tasks. Use for: +- Implementation tasks with 3+ steps (features, refactors, bug fixes) +- Tasks where the user asks for a plan or breakdown +- Complex workflows where progress visibility helps + +Do NOT use for simple single-file edits, quick questions, or explanations. + +IMPORTANT: This replaces the entire todo list. Always include ALL tasks (pending, in_progress, completed). Only ONE task should be in_progress at a time. Update status as you work: pending → in_progress → completed.`, + inputSchema: TodoWriteInputSchema, + + execute: async (input: unknown, context?: ToolExecutionContext): Promise => { + // Validate input against schema + const validatedInput = TodoWriteInputSchema.parse(input); + + // Use session_id from context, otherwise default + const sessionId = context?.sessionId ?? 'default'; + + // Update todos in todo service + const result = await todoService.updateTodos(sessionId, validatedInput.todos); + + // Count by status for summary + const completed = result.todos.filter((t) => t.status === 'completed').length; + const inProgress = result.todos.filter((t) => t.status === 'in_progress').length; + const pending = result.todos.filter((t) => t.status === 'pending').length; + + // Return simple summary - TodoPanel shows full state + return `Updated tasks: ${completed}/${result.todos.length} completed${inProgress > 0 ? `, 1 in progress` : ''}${pending > 0 ? `, ${pending} pending` : ''}`; + }, + }; +} diff --git a/dexto/packages/tools-todo/src/tool-provider.ts b/dexto/packages/tools-todo/src/tool-provider.ts new file mode 100644 index 00000000..d6073f2e --- /dev/null +++ b/dexto/packages/tools-todo/src/tool-provider.ts @@ -0,0 +1,92 @@ +/** + * Todo Tools Provider + * + * Provides task tracking tools by wrapping TodoService. + * When registered, the provider initializes TodoService and creates the + * todo_write tool for managing task lists. + */ + +import { z } from 'zod'; +import type { CustomToolProvider, ToolCreationContext, InternalTool } from '@dexto/core'; +import { TodoService } from './todo-service.js'; +import { createTodoWriteTool } from './todo-write-tool.js'; + +/** + * Default configuration constants for Todo tools. + */ +const DEFAULT_MAX_TODOS_PER_SESSION = 100; +const DEFAULT_ENABLE_EVENTS = true; + +/** + * Configuration schema for Todo tools provider. + */ +const TodoToolsConfigSchema = z + .object({ + type: z.literal('todo-tools'), + maxTodosPerSession: z + .number() + .int() + .positive() + .default(DEFAULT_MAX_TODOS_PER_SESSION) + .describe(`Maximum todos per session (default: ${DEFAULT_MAX_TODOS_PER_SESSION})`), + enableEvents: z + .boolean() + .default(DEFAULT_ENABLE_EVENTS) + .describe('Enable real-time events for todo updates'), + }) + .strict(); + +type TodoToolsConfig = z.output; + +/** + * Todo tools provider. + * + * Wraps TodoService and provides the todo_write tool for managing task lists. + * + * When registered via customToolRegistry, TodoService is automatically + * initialized and the todo_write tool becomes available to the agent. + */ +export const todoToolsProvider: CustomToolProvider<'todo-tools', TodoToolsConfig> = { + type: 'todo-tools', + configSchema: TodoToolsConfigSchema, + + create: (config: TodoToolsConfig, context: ToolCreationContext): InternalTool[] => { + const { logger, agent, services } = context; + + logger.debug('Creating TodoService for todo tools'); + + // Get required services from context + const storageManager = services?.storageManager; + if (!storageManager) { + throw new Error( + 'TodoService requires storageManager service. Ensure it is available in ToolCreationContext.' + ); + } + + const database = storageManager.getDatabase(); + const eventBus = agent.agentEventBus; + + // Create TodoService with validated config + const todoService = new TodoService(database, eventBus, logger, { + maxTodosPerSession: config.maxTodosPerSession, + enableEvents: config.enableEvents, + }); + + // Start initialization in background + todoService.initialize().catch((error) => { + const message = error instanceof Error ? error.message : String(error); + logger.error(`TodoToolsProvider.create: Failed to initialize TodoService: ${message}`); + }); + + logger.debug('TodoService created - initialization will complete on first tool use'); + + // Create and return the todo_write tool + return [createTodoWriteTool(todoService)]; + }, + + metadata: { + displayName: 'Todo Tools', + description: 'Task tracking and workflow management (todo_write)', + category: 'workflow', + }, +}; diff --git a/dexto/packages/tools-todo/src/types.ts b/dexto/packages/tools-todo/src/types.ts new file mode 100644 index 00000000..c14121e8 --- /dev/null +++ b/dexto/packages/tools-todo/src/types.ts @@ -0,0 +1,60 @@ +/** + * Todo Service Types + * + * Types for todo list management and workflow tracking + */ + +/** + * Valid todo status values + * Centralized constant to prevent duplication across domains + */ +export const TODO_STATUS_VALUES = ['pending', 'in_progress', 'completed'] as const; + +/** + * Todo item status + */ +export type TodoStatus = (typeof TODO_STATUS_VALUES)[number]; + +/** + * Todo item with system metadata + */ +export interface Todo { + id: string; + sessionId: string; + content: string; + activeForm: string; + status: TodoStatus; + position: number; + createdAt: Date; + updatedAt: Date; +} + +/** + * Todo input from tool (without system metadata) + */ +export interface TodoInput { + content: string; + activeForm: string; + status: TodoStatus; +} + +/** + * Todo list update result + */ +export interface TodoUpdateResult { + todos: Todo[]; + sessionId: string; + created: number; + updated: number; + deleted: number; +} + +/** + * Configuration for TodoService + */ +export interface TodoConfig { + /** Maximum todos per session */ + maxTodosPerSession?: number; + /** Enable real-time events */ + enableEvents?: boolean; +} diff --git a/dexto/packages/tools-todo/tsconfig.json b/dexto/packages/tools-todo/tsconfig.json new file mode 100644 index 00000000..01aae715 --- /dev/null +++ b/dexto/packages/tools-todo/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "declaration": true, + "declarationMap": true, + "skipLibCheck": true + }, + "include": ["src/**/*"], + "exclude": ["dist", "node_modules"] +} diff --git a/dexto/packages/tools-todo/tsup.config.ts b/dexto/packages/tools-todo/tsup.config.ts new file mode 100644 index 00000000..a0a1473b --- /dev/null +++ b/dexto/packages/tools-todo/tsup.config.ts @@ -0,0 +1,24 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig([ + { + entry: ['src/**/*.ts', '!src/**/*.test.ts'], + format: ['cjs', 'esm'], + outDir: 'dist', + dts: { + compilerOptions: { + skipLibCheck: true, + }, + }, + platform: 'node', + bundle: false, + clean: true, + tsconfig: './tsconfig.json', + esbuildOptions(options) { + options.logOverride = { + ...(options.logOverride ?? {}), + 'empty-import-meta': 'silent', + }; + }, + }, +]); diff --git a/dexto/packages/webui/.gitignore b/dexto/packages/webui/.gitignore new file mode 100644 index 00000000..5ef6a520 --- /dev/null +++ b/dexto/packages/webui/.gitignore @@ -0,0 +1,41 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# env files (can opt-in for committing if needed) +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/dexto/packages/webui/AGENTS.md b/dexto/packages/webui/AGENTS.md new file mode 100644 index 00000000..8eb70eab --- /dev/null +++ b/dexto/packages/webui/AGENTS.md @@ -0,0 +1,349 @@ +# WebUI Development Guidelines for AI Agents + +Comprehensive guide for AI agents working on the Dexto Vite WebUI. + +## Core Philosophy + +**Server Schemas = Single Source of Truth** + +Server defines all API types using Zod schemas → Hono typed client extracts types → React Query infers automatically → Components get full type safety. + +**NO Type Casting. NO `any` Types. NO Explicit Type Parameters.** + +If you need to cast, it's a RED FLAG. Fix the server schema instead. + +## Architecture + +**Stack**: Vite + React 19 + TypeScript + TanStack Router + Hono Typed Client + TanStack Query + SSE + +**Key Files**: +- `lib/client.ts` - Hono client initialization +- `lib/queryKeys.ts` - React Query key factory +- `components/hooks/` - All API hooks +- `types.ts` - UI-specific types only (NOT API types) + +## Type Flow + +``` +Server Zod Schemas → Hono Routes → Typed Client → React Query → Components +``` + +All automatic. No manual type definitions. + +## React Query Hook Patterns + +### Query Hook (Standard Pattern) + +```typescript +// components/hooks/useServers.ts +import { useQuery } from '@tanstack/react-query'; +import { client } from '@/lib/client'; +import { queryKeys } from '@/lib/queryKeys'; + +// No explicit types! Let TypeScript infer. +export function useServers(enabled: boolean = true) { + return useQuery({ + queryKey: queryKeys.servers.all, + queryFn: async () => { + const res = await client.api.mcp.servers.$get(); + if (!res.ok) throw new Error('Failed to fetch servers'); + const data = await res.json(); + return data.servers; // Type inferred from server schema + }, + enabled, + }); +} + +// Export types using standard inference pattern +export type McpServer = NonNullable['data']>[number]; +``` + +**Type Inference Pattern Breakdown**: +- `ReturnType` - Hook's return type (UseQueryResult) +- `['data']` - Data property +- `NonNullable<...>` - Remove undefined (assumes loaded) +- `[number]` - Array element type (if array) +- `['field'][number]` - Nested array access + +### Mutation Hook (Standard Pattern) + +```typescript +export function useCreateMemory() { + const queryClient = useQueryClient(); + + return useMutation({ + // Simple payload: inline type + mutationFn: async (payload: { + content: string; + tags?: string[]; + }) => { + const response = await client.api.memory.$post({ json: payload }); + return await response.json(); + }, + // Always invalidate affected queries + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: queryKeys.memories.all }); + }, + }); +} + +// Complex payload: use Parameters utility +export function useSwitchAgent() { + return useMutation({ + mutationFn: async ( + payload: Parameters[0]['json'] + ) => { + const response = await client.api.agents.switch.$post({ json: payload }); + return await response.json(); + }, + }); +} +``` + +### Handling Multiple Response Codes + +Always check `response.ok` to narrow discriminated unions: + +```typescript +const response = await client.api['message-sync'].$post({...}); + +if (!response.ok) { + throw new Error(`Request failed: ${response.status}`); +} + +const data = await response.json(); // Now properly typed! +``` + +### SSE Streaming + +```typescript +import { createMessageStream } from '@dexto/client-sdk'; +import type { MessageStreamEvent } from '@dexto/client-sdk'; + +const responsePromise = client.api['message-stream'].$post({ json: { message, sessionId } }); +const iterator = createMessageStream(responsePromise, { signal: abortController.signal }); + +for await (const event of iterator) { + processEvent(event); // Fully typed as MessageStreamEvent +} +``` + +## Component Patterns + +### Importing and Using Hooks + +```typescript +// ✅ Import types from hooks (centralized) +import { useServers } from '@/hooks/useServers'; +import type { McpServer } from '@/hooks/useServers'; + +export function ServersList() { + const { data: servers, isLoading } = useServers(); + const deleteServer = useDeleteServer(); + + if (isLoading) return ; + if (!servers) return ; + + return ( +
+ {servers.map((server) => ( // server is fully typed + deleteServer.mutate(server.id)} + /> + ))} +
+ ); +} +``` + +### Mutation Success/Error Handling + +```typescript +const createMemory = useCreateMemory(); + +const handleSubmit = () => { + createMemory.mutate( + { content, tags }, + { + onSuccess: (data) => { + toast.success('Memory created'); + onClose(); + }, + onError: (error: Error) => { + setError(error.message); + }, + } + ); +}; +``` + +### Mutations in useCallback/useMemo Dependencies + +**CRITICAL:** useMutation objects are NOT stable and will cause infinite re-renders if added to dependency arrays. Instead, extract `mutate` or `mutateAsync` functions which ARE stable: + +```typescript +// ❌ WRONG - mutation object is unstable +const addServerMutation = useAddServer(); + +const handleClick = useCallback(() => { + addServerMutation.mutate({ name, config }); +}, [addServerMutation]); // ⚠️ Causes infinite loop! + +// ✅ CORRECT - extract stable function +const { mutate: addServer } = useAddServer(); + +const handleClick = useCallback(() => { + addServer({ name, config }); +}, [addServer]); // ✅ Safe - mutate function is stable + +// ✅ CORRECT - for async operations +const { mutateAsync: addServer } = useAddServer(); + +const handleClick = useCallback(async () => { + await addServer({ name, config }); + doSomethingElse(); +}, [addServer]); // ✅ Safe - mutateAsync function is stable +``` + +**Reference:** See `ToolConfirmationHandler.tsx` lines 36, 220 for the pattern in action. + +## State Management + +- **TanStack Query** - Server state, caching, API data +- **React Context** - App-wide UI state (theme, active session) +- **Zustand** - Persistent UI state (localStorage) + +## Common Patterns + +### Conditional Queries + +```typescript +export function useServerTools(serverId: string | null, enabled: boolean = true) { + return useQuery({ + queryKey: queryKeys.servers.tools(serverId || ''), + queryFn: async () => { + if (!serverId) return []; + // ... fetch tools + }, + enabled: enabled && !!serverId, // Only run if both true + }); +} +``` + +### Parameterized Hooks + +```typescript +export function useLLMCatalog(options?: { enabled?: boolean; mode?: 'grouped' | 'flat' }) { + const mode = options?.mode ?? 'grouped'; + return useQuery({ + queryKey: [...queryKeys.llm.catalog, mode], + queryFn: async () => { + const response = await client.api.llm.catalog.$get({ query: { mode } }); + return await response.json(); + }, + enabled: options?.enabled ?? true, + }); +} +``` + +## What NOT to Do + +### ❌ Don't Add Explicit Types + +```typescript +// ❌ WRONG +export function useServers() { + return useQuery({ ... }); +} + +// ✅ CORRECT +export function useServers() { + return useQuery({ ... }); // TypeScript infers +} +``` + +### ❌ Don't Cast API Response Types + +```typescript +// ❌ WRONG - RED FLAG! +const servers = data.servers as McpServer[]; +config: payload.config as McpServerConfig; + +// ✅ CORRECT - Fix server schema +``` + +### ❌ Don't Duplicate Types + +```typescript +// ❌ WRONG - in types.ts +export interface McpServer { id: string; name: string; } + +// ✅ CORRECT - export from hook +export type McpServer = NonNullable['data']>[number]; +``` + +### ❌ Don't Create Inline Types + +```typescript +// ❌ WRONG +function ServerCard({ server }: { server: { id: string; name: string } }) {} + +// ✅ CORRECT +import type { McpServer } from '@/hooks/useServers'; +function ServerCard({ server }: { server: McpServer }) {} +``` + +### ❌ Don't Skip Cache Invalidation + +```typescript +// ❌ WRONG +export function useDeleteServer() { + return useMutation({ + mutationFn: async (serverId: string) => { ... }, + // Missing onSuccess! + }); +} + +// ✅ CORRECT +export function useDeleteServer() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: async (serverId: string) => { ... }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: queryKeys.servers.all }); + }, + }); +} +``` + +## Migration Checklist + +When adding a new API endpoint: + +1. Define Zod schema in server route +2. Use `z.output` for inline server types +3. Create hook in `components/hooks/` (no explicit types) +4. Export inferred types using `NonNullable>` pattern +5. Add query key to `queryKeys.ts` +6. Handle cache invalidation in mutations +7. Import types from hook in components +8. Verify no type casts needed + +## Key Files Reference + +- **Server Routes**: `packages/server/src/hono/routes/` +- **Client SDK**: `packages/client-sdk/src/` +- **Core Types**: `packages/core/src/` +- **Query Keys**: `packages/webui/lib/queryKeys.ts` + +## Summary + +1. **Server schemas = source of truth** - Never duplicate types +2. **Let TypeScript infer everything** - No explicit type parameters +3. **Export types from hooks** - Centralized and consistent +4. **Type casting = red flag** - Fix at source +5. **Always invalidate cache** - Keep UI in sync + +**If you're fighting with types, you're doing it wrong. Fix the server schema.** diff --git a/dexto/packages/webui/CHANGELOG.md b/dexto/packages/webui/CHANGELOG.md new file mode 100644 index 00000000..4f01d645 --- /dev/null +++ b/dexto/packages/webui/CHANGELOG.md @@ -0,0 +1,494 @@ +# @dexto/webui + +## 1.5.6 + +### Patch Changes + +- 042f4f0: ### CLI Improvements + - Add `/export` command to export conversations as Markdown or JSON + - Add `Ctrl+T` toggle for task list visibility during processing + - Improve task list UI with collapsible view near the processing message + - Fix race condition causing duplicate rendering (mainly visible with explore tool) + - Don't truncate `pattern` and `question` args in tool output display + + ### Bug Fixes + - Fix build script to preserve `.dexto` storage (conversations, logs) during clean builds + - Fix `@dexto/tools-todo` versioning - add to fixed version group in changeset config + + ### Configuration Changes + - Remove approval timeout defaults - now waits indefinitely (better UX for CLI) + - Add package versioning guidelines to AGENTS.md + +- Updated dependencies [042f4f0] + - @dexto/client-sdk@1.5.6 + - @dexto/analytics@1.5.6 + - @dexto/registry@1.5.6 + - @dexto/core@1.5.6 + +## 1.5.5 + +### Patch Changes + +- 9ab3eac: Added todo tools. +- 63fa083: Session and context management fixes: + - Remove continuation session logic after compaction, now sticks to same session + - `/clear` continues same session and resets context (frees up AI context window) + - `/new` command creates new session with fresh context and clears screen + - Add context tokens remaining to footer, align context calculations everywhere + - Fix context calculation logic by including cache read tokens + + Other improvements: + - Fix code block syntax highlighting in terminal (uses cli-highlight) + - Make terminal the default mode during onboarding + - Reduce OTEL dependency bloat by replacing auto-instrumentation with specific packages (47 MB saved: 65 MB → 18 MB) + +- Updated dependencies [63fa083] +- Updated dependencies [6df3ca9] + - @dexto/core@1.5.5 + - @dexto/registry@1.5.5 + - @dexto/analytics@1.5.5 + - @dexto/client-sdk@1.5.5 + +## 1.5.4 + +### Patch Changes + +- 499b890: Fix model override persistence after compaction and improve context token tracking + + **Bug Fixes:** + - Fix model override resetting to config model after compaction (now respects session overrides) + + **Context Tracking Improvements:** + - New algorithm uses actual `input_tokens` and `output_tokens` from LLM responses as source of truth + - Self-correcting estimates: inaccuracies auto-correct when next LLM response arrives + - Handles pruning automatically (next response's input_tokens reflects pruned state) + - `/context` and compaction decisions now share common calculation logic + - Removed `outputBuffer` concept in favor of single configurable threshold + - Default compaction threshold lowered to 90% + + **New `/context` Command:** + - Interactive overlay with stacked token bar visualization + - Breakdown by component: system prompt, tools, messages, free space, auto-compact buffer + - Expandable per-tool token details + - Shows pruned tool count and compaction history + + **Observability:** + - Comparison logging between estimated vs actual tokens for calibration + - `dexto_llm_tokens_consumed` metric now includes estimated input tokens and accuracy metrics + +- Updated dependencies [0016cd3] +- Updated dependencies [499b890] +- Updated dependencies [aa2c9a0] + - @dexto/core@1.5.4 + - @dexto/analytics@1.5.4 + - @dexto/client-sdk@1.5.4 + - @dexto/registry@1.5.4 + +## 1.5.3 + +### Patch Changes + +- 4f00295: Added spawn-agent tools and explore agent. +- Updated dependencies [4f00295] +- Updated dependencies [69c944c] + - @dexto/core@1.5.3 + - @dexto/analytics@1.5.3 + - @dexto/client-sdk@1.5.3 + - @dexto/registry@1.5.3 + +## 1.5.2 + +### Patch Changes + +- Updated dependencies [8a85ea4] +- Updated dependencies [527f3f9] + - @dexto/core@1.5.2 + - @dexto/analytics@1.5.2 + - @dexto/client-sdk@1.5.2 + - @dexto/registry@1.5.2 + +## 1.5.1 + +### Patch Changes + +- 4aabdb7: Fix claude caching, added gpt-5.2 models and reasoning effort options in user flows. +- Updated dependencies [bfcc7b1] +- Updated dependencies [4aabdb7] + - @dexto/core@1.5.1 + - @dexto/analytics@1.5.1 + - @dexto/client-sdk@1.5.1 + - @dexto/registry@1.5.1 + +## 1.5.0 + +### Minor Changes + +- e7722e5: Minor version bump for new release with bundler, custom tool pkgs, etc. + +### Patch Changes + +- abfe5ce: Update and standardize analytics for CLI and web UI +- ee12727: Added support for node-llama (llama.cpp) for local GGUF models. Added Ollama as first-class provider. Updated onboarding/setup flow. +- 4c05310: Improve local model/GGUF model support, bash permission fixes in TUI, and add local/ollama switching/deleting support in web UI +- 5fa79fa: Renamed compression to compaction, added context-awareness to hono, updated cli tool display formatting and added integration test for image-local. +- ef40e60: Upgrades package versions and related changes to MCP SDK. CLI colors improved and token streaming added to status bar. + + Security: Resolve all Dependabot security vulnerabilities. Updated @modelcontextprotocol/sdk to 1.25.2, esbuild to 0.25.0, langchain to 0.3.37, and @langchain/core to 0.3.80. Added pnpm overrides for indirect vulnerabilities (preact@10.27.3, qs@6.14.1, jws@3.2.3, mdast-util-to-hast@13.2.1). Fixed type errors from MCP SDK breaking changes. + +- e714418: Added providers for db and cache storages. Expanded settings panel for API keys and other app preferences in WebUI along with other UI/UX enhancements. +- 7d5ab19: Updated WebUI design, event and state management and forms +- 436a900: Add support for openrouter, bedrock, glama, vertex ai, fix model switching issues and new model experience for each +- Updated dependencies [abfe5ce] +- Updated dependencies [ee12727] +- Updated dependencies [1e7e974] +- Updated dependencies [4c05310] +- Updated dependencies [5fa79fa] +- Updated dependencies [ef40e60] +- Updated dependencies [e714418] +- Updated dependencies [e7722e5] +- Updated dependencies [7d5ab19] +- Updated dependencies [436a900] + - @dexto/analytics@1.5.0 + - @dexto/core@1.5.0 + - @dexto/client-sdk@1.5.0 + - @dexto/registry@1.5.0 + +## 1.4.0 + +### Minor Changes + +- f73a519: Revamp CLI. Breaking change to DextoAgent.generate() and stream() apis and hono message APIs, so new minor version. Other fixes for logs, web UI related to message streaming/generating + +### Patch Changes + +- bd5c097: Add features check for internal tools, fix coding agent and logger agent elicitation +- 3cdce89: Revamp CLI for coding agent, add new events, improve mcp management, custom models, minor UI changes, prompt management +- d640e40: Remove LLM services, tokenizers, just stick with vercel, remove 'router' from schema and all types and docs +- 6e6a3e7: Fix message typings to use proper discriminated unions in core and webui +- c54760f: Revamp context management layer - add partial stream cancellation, message queueing, context compression with LLM, MCP UI support and gaming agent. New APIs and UI changes for these things +- ab47df8: Add approval metadata and ui badge +- Updated dependencies [bd5c097] +- Updated dependencies [3cdce89] +- Updated dependencies [d640e40] +- Updated dependencies [6f5627d] +- Updated dependencies [6e6a3e7] +- Updated dependencies [f73a519] +- Updated dependencies [c54760f] +- Updated dependencies [ab47df8] +- Updated dependencies [3b4b919] + - @dexto/core@1.4.0 + - @dexto/registry@1.4.0 + - @dexto/analytics@1.4.0 + - @dexto/client-sdk@1.4.0 + +## 1.3.0 + +### Minor Changes + +- eb266af: Migrate WebUI from next-js to vite. Fix any typing in web UI. Improve types in core. minor renames in event schemas + +### Patch Changes + +- 215ae5b: Add new chat, search chats buttons, add rename, copy session ids, update UI to not have overlapping buttons for different layouts with session panel open +- 338da3f: Update model switcher UI +- 66ce8c2: Added changeset for ink-cli upgrades and metadata patch in webui +- Updated dependencies [e2f770b] +- Updated dependencies [f843b62] +- Updated dependencies [eb266af] + - @dexto/core@1.3.0 + - @dexto/client-sdk@1.3.0 + - @dexto/analytics@1.3.0 + +## 1.2.6 + +### Patch Changes + +- 7feb030: Update memory and prompt configs, fix agent install bug +- Updated dependencies [7feb030] + - @dexto/core@1.2.6 + - @dexto/client-sdk@1.2.6 + - @dexto/analytics@1.2.6 + +## 1.2.5 + +### Patch Changes + +- c1e814f: ## Logger v2 & Config Enrichment + + ### New Features + - **Multi-transport logging system**: Configure console, file, and remote logging transports via `logger` field in agent.yml. Supports log levels (error, warn, info, debug, silly) and automatic log rotation for file transports. + - **Per-agent isolation**: CLI automatically creates per-agent log files at `~/.dexto/logs/.log`, database at `~/.dexto/database/.db`, and blob storage at `~/.dexto/blobs//` + - **Agent ID derivation**: Agent ID is now automatically derived from `agentCard.name` (sanitized) or config filename, enabling proper multi-agent isolation without manual configuration + + ### Breaking Changes + - **Storage blob default changed**: Default blob storage type changed from `local` to `in-memory`. Existing configs with explicit `blob: { type: 'local' }` are unaffected. CLI enrichment provides automatic paths for SQLite and local blob storage. + + ### Improvements + - **Config enrichment layer**: New `enrichAgentConfig()` in agent-management package adds per-agent paths before initialization, eliminating path resolution in core services + - **Logger error factory**: Added typed error factory pattern for logger errors following project conventions + - **Removed wildcard exports**: Logger module now uses explicit named exports for better tree-shaking + + ### Documentation + - Added complete logger configuration section to agent.yml documentation + - Documented agentId field and derivation rules + - Updated storage documentation with CLI auto-configuration notes + - Added logger v2 architecture notes to core README + +- 81598b5: Decoupled elicitation from tool confirmation. Added `DenialReason` enum and structured error messages to approval responses. + - Tool approvals and elicitation now independently configurable via `elicitation.enabled` config + - Approval errors include `reason` (user_denied, timeout, system_denied, etc.) and `message` fields + - Enables `auto-approve` for tools while preserving interactive elicitation + + Config files without the new `elicitation` section will use defaults. No legacy code paths. + +- 8f373cc: Migrate server API to Hono framework with feature flag + - Migrated Express server to Hono with OpenAPI schema generation + - Added DEXTO_USE_HONO environment variable flag (default: false for backward compatibility) + - Fixed WebSocket test isolation by adding sessionId filtering + - Fixed logger context to pass structured objects instead of stringified JSON + - Fixed CI workflow for OpenAPI docs synchronization + - Updated documentation links and fixed broken API references + +- f28ad7e: Migrate webUI to use client-sdk, add agents.md file to webui,improve types in apis for consumption +- 4dd4998: Add changeset for command approval enhancement and orphaned tool handling +- a35a256: Migrate from WebSocket to Server-Sent Events (SSE) for real-time streaming + - Replace WebSocket with SSE for message streaming via new `/api/message-stream` endpoint + - Refactor approval system from event-based providers to simpler handler pattern + - Add new APIs for session approval + - Move session title generation to a separate API + - Add `ApprovalCoordinator` for multi-client SSE routing with sessionId mapping + - Add stream and generate methods to DextoAgent and integ tests for itq= + +- a154ae0: UI refactor with TanStack Query, new agent management package, and Hono as default server + + **Server:** + - Make Hono the default API server (use `DEXTO_USE_EXPRESS=true` env var to use Express) + - Fix agentId propagation to Hono server for correct agent name display + - Fix circular reference crashes in error logging by using structured logger context + + **WebUI:** + - Integrate TanStack Query for server state management with automatic caching and invalidation + - Add centralized query key factory and API client with structured error handling + - Replace manual data fetching with TanStack Query hooks across all components + - Add Zustand for client-side persistent state (recent agents in localStorage) + - Add keyboard shortcuts support with react-hotkeys-hook + - Add optimistic updates for session management via WebSocket events + - Fix Dialog auto-close bug in CreateMemoryModal + - Add defensive null handling in MemoryPanel + - Standardize Prettier formatting (single quotes, 4-space indentation) + + **Agent Management:** + - Add `@dexto/agent-management` package for centralized agent configuration management + - Extract agent registry, preferences, and path utilities into dedicated package + + **Internal:** + - Improve build orchestration and fix dependency imports + - Add `@dexto/agent-management` to global CLI installation + +- ac649fd: Fix error handling and UI bugs, add gpt-5.1, gemini-3 +- Updated dependencies [c1e814f] +- Updated dependencies [f9bca72] +- Updated dependencies [c0a10cd] +- Updated dependencies [81598b5] +- Updated dependencies [4c90ffe] +- Updated dependencies [1a20506] +- Updated dependencies [8f373cc] +- Updated dependencies [f28ad7e] +- Updated dependencies [4dd4998] +- Updated dependencies [5e27806] +- Updated dependencies [a35a256] +- Updated dependencies [0fa6ef5] +- Updated dependencies [e2fb5f8] +- Updated dependencies [a154ae0] +- Updated dependencies [5a26bdf] +- Updated dependencies [ac649fd] + - @dexto/core@1.2.5 + - @dexto/client-sdk@1.2.5 + - @dexto/analytics@1.2.5 + +## 0.2.4 + +### Patch Changes + +- cd706e7: bump up version after fixing node-machine-id +- Updated dependencies [cd706e7] + - @dexto/analytics@1.2.4 + - @dexto/core@1.2.4 + +## 0.2.3 + +### Patch Changes + +- 5d6ae73: Bump up version to fix bugs +- Updated dependencies [5d6ae73] + - @dexto/analytics@1.2.3 + - @dexto/core@1.2.3 + +## 0.2.2 + +### Patch Changes + +- 4a3b1b5: Add changeset for new mcp servers +- 8b96b63: Add posthog analytics package and add to web ui +- Updated dependencies [8b96b63] + - @dexto/analytics@1.2.2 + - @dexto/core@1.2.2 + +## 0.2.1 + +### Patch Changes + +- 9a26447: Fix starter prompts + - @dexto/core@1.2.1 + +## 0.2.0 + +### Minor Changes + +- 1e25f91: Update web UI to be default, fix port bugs, update docs + +### Patch Changes + +- 72ef45c: Fix minor spinner bug +- 3a65cde: Update older LLMs to new LLMs, update docs +- 70a78ca: Update webUI to handle different sizes, mobile views, improve model picker UI +- 5ba5d38: **Features:** + - Agent switcher now supports file-based agents loaded via CLI (e.g., `dexto --agent path/to/agent.yml`) + - Agent selector UI remembers recent agents (up to 5) with localStorage persistence + - WebUI displays currently active file-based agent and recent agent history + - Dev server (`pnpm dev`) now auto-opens browser when WebUI is ready + - Added `/test-api` custom command for automated API test coverage analysis + + **Bug Fixes:** + - Fixed critical bug where Memory, A2A, and MCP API routes used stale agent references after switching + - Fixed telemetry shutdown blocking agent switches when observability infrastructure (Jaeger/OTLP) is unavailable + - Fixed dark mode styling issues when Chrome's Auto Dark Mode is enabled + - Fixed agent card not updating for A2A and MCP routes after agent switch + + **Improvements:** + - Refactored `Dexto.createAgent()` to static method, removing unnecessary singleton pattern + - Improved error handling for agent switching with typed errors (CONFLICT error type, `AgentError.switchInProgress()`) + - Telemetry now disabled by default (opt-in) in default agent configuration + - Added localStorage corruption recovery for recent agents list + +- c6594d9: Add changeset for updated readme and docs. +- 930a4ca: Fixes in UI, docs and agents +- 2940fbf: Add changeset for playground ui updates +- 930d75a: Add mcp server restart feature and button in webUI +- 3fc6716: Add changeset for new servers & webui fixes. +- Updated dependencies [b51e4d9] +- Updated dependencies [a27ddf0] +- Updated dependencies [155813c] +- Updated dependencies [1e25f91] +- Updated dependencies [3a65cde] +- Updated dependencies [5ba5d38] +- Updated dependencies [930a4ca] +- Updated dependencies [ecad345] +- Updated dependencies [930d75a] + - @dexto/core@1.2.0 + +## 0.1.8 + +### Patch Changes + +- c40b675: - Updated toolResult sanitization flow + - Added support for video rendering to WebUI +- 015100c: Added new memory manager for creating, storing and managing memories. + - FileContributor has a new memories contributor for loading memories into SystemPrompt. +- 2b81734: Updated WebUI for MCP connection flow and other minor updates +- 5cc6933: Fixes for prompts/resource management, UI improvements, custom slash command support, add support for embedded/linked resources, proper argument handling for prompts +- 40f89f5: Add New Agent buttons, form editor, new APIs, Dexto class +- 0558564: Several enhancement have been made to the WebUI to improve UI/UX + - Minor fixes to styling for AgentSelector and MCP Registry + - Sessions Panel updated UI & list ordering + - Sessions Panel updated page routing +- 01167a2: Refactors +- a53b87a: feat: Redesign agent registry system with improved agent switching + - **@dexto/core**: Enhanced agent registry with better ID-based resolution, improved error handling, and normalized registry entries + - **dexto**: Added agent switching capabilities via API with proper state management + - **@dexto/webui**: Updated agent selector UI with better UX for switching between agents + - Agent resolution now uses `agentId` instead of `agentName` throughout the system + - Registry entries now require explicit `id` field matching the registry key + +- 1da4398: Added support for video rendering in webui & updated thinking label +- 24e5093: Add customize agent capabilities +- c695e57: Add blob storage system for persistent binary data management: + - Implement blob storage backend with local filesystem support + - Add blob:// URI scheme for referencing stored blobs + - Integrate blob storage with resource system for seamless @resource references + - Add automatic blob expansion in chat history and message references + - Add real-time cache invalidation events for resources and prompts + - Fix prompt cache invalidation WebSocket event handling in WebUI + - Add robustness improvements: empty text validation after resource expansion and graceful blob expansion error handling + - Support image/file uploads with automatic blob storage + - Add WebUI components for blob resource display and autocomplete +- 0700f6f: Support for in-built and custom plugins +- 0a5636c: Added a new Approval System and support MCP Elicitations +- 35d48c5: Add chat summary generation +- Updated dependencies [c40b675] +- Updated dependencies [015100c] +- Updated dependencies [0760f8a] +- Updated dependencies [5cc6933] +- Updated dependencies [40f89f5] +- Updated dependencies [3a24d08] +- Updated dependencies [01167a2] +- Updated dependencies [a53b87a] +- Updated dependencies [24e5093] +- Updated dependencies [c695e57] +- Updated dependencies [0700f6f] +- Updated dependencies [0a5636c] +- Updated dependencies [35d48c5] + - @dexto/core@1.1.11 + +## 0.1.7 + +### Patch Changes + +- @dexto/core@1.1.10 + +## 0.1.6 + +### Patch Changes + +- 27778ba: Add claude 4.5 sonnet and make it default +- Updated dependencies [27778ba] + - @dexto/core@1.1.9 + +## 0.1.5 + +### Patch Changes + +- Updated dependencies [d79d358] + - @dexto/core@1.1.8 + +## 0.1.4 + +### Patch Changes + +- @dexto/core@1.1.7 + +## 0.1.3 + +### Patch Changes + +- e6d029c: Update logos and fix build + - @dexto/core@1.1.6 + +## 0.1.2 + +### Patch Changes + +- e523d86: Add keyboard shortcut to delete session +- Updated dependencies [e2bd0ce] +- Updated dependencies [11cbec0] +- Updated dependencies [795c7f1] +- Updated dependencies [9d7541c] + - @dexto/core@1.1.5 + +## 0.1.1 + +### Patch Changes + +- 2fccffd: Migrating to monorepo +- Updated dependencies [2fccffd] + - @dexto/core@1.1.4 diff --git a/dexto/packages/webui/CLAUDE.md b/dexto/packages/webui/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/dexto/packages/webui/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/dexto/packages/webui/GEMINI.md b/dexto/packages/webui/GEMINI.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/dexto/packages/webui/GEMINI.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/dexto/packages/webui/README.md b/dexto/packages/webui/README.md new file mode 100644 index 00000000..0a18664f --- /dev/null +++ b/dexto/packages/webui/README.md @@ -0,0 +1,46 @@ +# Dexto Playground + +This project is an interactive playground for testing MCP tools/servers tools and building your own AI agents. + +[MCP - Model Context Protocol] + +## Features + +- **Tool Testing Playground**: Connect and test MCP servers and their tools interactively +- **Simple Chat Interface**: Clean, focused conversation with AI agents +- **Server Management**: Easy connection and management of MCP servers +- **Tool Discovery**: Explore available tools and their capabilities +- **Configuration Export**: Export your tool setup for use with Claude Desktop or other MCP clients + +## What is MCP? + +The Model Context Protocol (MCP) allows AI models to securely connect to external tools and data sources. Dexto provides a simple interface to: + +- Connect to MCP servers +- Test tool functionality +- Chat with AI agents that have access to your tools +- Export configurations for other clients + +## Quick Start + +1. Connect a MCP server using the "Tools" panel +2. Test individual tools in the playground (`/playground`) +3. Chat with AI agents that can use your connected tools +4. Export your configuration when ready + +This project is built with [Vite](https://vitejs.dev) and [TanStack Router](https://tanstack.com/router) for a smooth development experience. + +## Developer guide + +Clear out ports 3000 (linux): +```bash +lsof -ti:3000-3001 | xargs kill -9 +``` + +Go to repository root and run the server in dev mode: +```bash +pnpm run dev +``` + +Open [http://localhost:3000](http://localhost:3000) to start testing your tools. + diff --git a/dexto/packages/webui/components.json b/dexto/packages/webui/components.json new file mode 100644 index 00000000..9bcfb145 --- /dev/null +++ b/dexto/packages/webui/components.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": false, + "tsx": true, + "tailwind": { + "config": "tailwind.config.js", + "css": "src/styles/globals.css", + "baseColor": "neutral", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" + }, + "iconLibrary": "lucide" +} \ No newline at end of file diff --git a/dexto/packages/webui/components/AddCustomServerModal.tsx b/dexto/packages/webui/components/AddCustomServerModal.tsx new file mode 100644 index 00000000..2b43b5c7 --- /dev/null +++ b/dexto/packages/webui/components/AddCustomServerModal.tsx @@ -0,0 +1,598 @@ +import React, { useState } from 'react'; +import type { ServerRegistryEntry } from '@dexto/registry'; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from './ui/dialog'; +import { Button } from './ui/button'; +import { Input } from './ui/input'; +import { Label } from './ui/label'; +import { Textarea } from './ui/textarea'; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from './ui/select'; +import { Alert, AlertDescription } from './ui/alert'; +import { Plus, Save } from 'lucide-react'; +import { KeyValueEditor } from './ui/key-value-editor'; + +interface HeaderPair { + key: string; + value: string; + id: string; +} + +interface AddCustomServerModalProps { + isOpen: boolean; + onClose: () => void; + onAddServer: ( + entry: Omit + ) => Promise; +} + +export default function AddCustomServerModal({ + isOpen, + onClose, + onAddServer, +}: AddCustomServerModalProps) { + const [formData, setFormData] = useState<{ + name: string; + description: string; + category: + | 'productivity' + | 'development' + | 'research' + | 'creative' + | 'data' + | 'communication' + | 'custom'; + icon: string; + version: string; + author: string; + homepage: string; + config: { + type: 'stdio' | 'sse' | 'http'; + command: string; + args: string[]; + url: string; + env: Record; + headers: Record; + timeout: number; + }; + tags: string[]; + isInstalled: boolean; + requirements: { + platform: 'win32' | 'darwin' | 'linux' | 'all'; + node: string; + python: string; + dependencies: string[]; + }; + }>({ + name: '', + description: '', + category: 'custom', + icon: '', + version: '', + author: '', + homepage: '', + config: { + type: 'stdio', + command: '', + args: [], + url: '', + env: {}, + headers: {}, + timeout: 30000, + }, + tags: [], + isInstalled: false, + requirements: { + platform: 'all', + node: '', + python: '', + dependencies: [], + }, + }); + + const [argsInput, setArgsInput] = useState(''); + const [tagsInput, setTagsInput] = useState(''); + const [envInput, setEnvInput] = useState(''); + const [headerPairs, setHeaderPairs] = useState([]); + const [dependenciesInput, setDependenciesInput] = useState(''); + const [error, setError] = useState(null); + const [isSubmitting, setIsSubmitting] = useState(false); + + const categories = [ + { value: 'productivity', label: 'Productivity' }, + { value: 'development', label: 'Development' }, + { value: 'research', label: 'Research' }, + { value: 'creative', label: 'Creative' }, + { value: 'data', label: 'Data' }, + { value: 'communication', label: 'Communication' }, + { value: 'custom', label: 'Custom' }, + ]; + + const platforms = [ + { value: 'all', label: 'All Platforms' }, + { value: 'win32', label: 'Windows' }, + { value: 'darwin', label: 'macOS' }, + { value: 'linux', label: 'Linux' }, + ]; + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + setError(null); + setIsSubmitting(true); + + try { + // Parse inputs + const args = argsInput + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + const tags = tagsInput + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + const dependencies = dependenciesInput + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + + // Parse environment variables + const env: Record = {}; + if (envInput.trim()) { + const envLines = envInput.split('\n'); + for (const line of envLines) { + // Skip empty lines + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + // Split only at the first '=' character + const equalIndex = trimmedLine.indexOf('='); + if (equalIndex > 0) { + // Key must exist (equalIndex > 0, not >= 0) + const key = trimmedLine.substring(0, equalIndex).trim(); + const value = trimmedLine.substring(equalIndex + 1).trim(); + + // Only add if key is not empty + if (key) { + env[key] = value; // Value can be empty string + } + } + } + } + + // Convert header pairs to record + const headers: Record = {}; + headerPairs.forEach((pair) => { + if (pair.key.trim() && pair.value.trim()) { + headers[pair.key.trim()] = pair.value.trim(); + } + }); + + // Validate required fields + if (!formData.name.trim()) { + throw new Error('Server name is required'); + } + if (!formData.description.trim()) { + throw new Error('Description is required'); + } + if (formData.config.type === 'stdio' && !formData.config.command.trim()) { + throw new Error('Command is required for stdio servers'); + } + if (formData.config.type === 'sse' && !formData.config.url.trim()) { + throw new Error('URL is required for SSE servers'); + } + if (formData.config.type === 'http' && !formData.config.url.trim()) { + throw new Error('URL is required for HTTP servers'); + } + + const entry: Omit = { + ...formData, + config: { + ...formData.config, + args, + env, + headers, + }, + tags, + requirements: { + ...formData.requirements, + dependencies, + }, + }; + + await onAddServer(entry); + onClose(); + + // Reset form + setFormData({ + name: '', + description: '', + category: 'custom', + icon: '', + version: '', + author: '', + homepage: '', + config: { + type: 'stdio', + command: '', + args: [], + url: '', + env: {}, + headers: {}, + timeout: 30000, + }, + tags: [], + isInstalled: false, + requirements: { + platform: 'all', + node: '', + python: '', + dependencies: [], + }, + }); + setArgsInput(''); + setTagsInput(''); + setEnvInput(''); + setHeaderPairs([]); + setDependenciesInput(''); + } catch (err: any) { + setError(err.message || 'Failed to add custom server'); + } finally { + setIsSubmitting(false); + } + }; + + const handleConfigChange = (e: React.ChangeEvent) => { + const { name, value } = e.target; + setFormData((prev) => ({ + ...prev, + config: { + ...prev.config, + [name]: value, + }, + })); + }; + + return ( + !open && onClose()}> + + + + + Add Custom Server to Registry + + + Add your own custom MCP server configuration to the registry for easy reuse. + + + +
+ {error && ( + + {error} + + )} + + {/* Basic Information */} +
+
+ + + setFormData((prev) => ({ ...prev, name: e.target.value })) + } + placeholder="My Custom Server" + required + /> +
+
+ + +
+
+ +
+ +