/** * Quality Gate * * Validates delegation results before returning to users. * Implements confidence thresholds and automatic escalation. */ import { QualityCheck, QualityCheckResult, QualityIssue, QualityGateConfig, DelegationResult, DelegationContext } from '../core/types'; // ============================================================================ // Default Configuration // ============================================================================ const DEFAULT_CONFIG: QualityGateConfig = { enabled: true, minConfidence: 0.7, checks: [], escalationThreshold: 0.5, autoEscalate: true }; // ============================================================================ // Built-in Quality Checks // ============================================================================ const BUILT_IN_CHECKS: QualityCheck[] = [ { id: 'confidence-check', name: 'Confidence Threshold', description: 'Ensures result meets minimum confidence threshold', severity: 'high', autoFix: false, check: async (result: DelegationResult) => { const passed = result.confidence >= 0.7; return { passed, score: result.confidence, issues: passed ? [] : [{ severity: 'high' as const, message: `Confidence ${result.confidence.toFixed(2)} below threshold 0.70` }], recommendations: passed ? [] : ['Consider escalating to main agent'] }; } }, { id: 'output-validity', name: 'Output Validity', description: 'Checks that output is non-empty and meaningful', severity: 'critical', autoFix: false, check: async (result: DelegationResult) => { const hasOutput = result.output && result.output.trim().length > 0; const isNotError = !result.output?.toLowerCase().includes('error'); return { passed: hasOutput && isNotError, score: hasOutput ? (isNotError ? 1 : 0.5) : 0, issues: [ ...(hasOutput ? [] : [{ severity: 'critical' as const, message: 'Empty output' }]), ...(isNotError ? [] : [{ severity: 'high' as const, message: 'Output contains error message' }]) ], recommendations: hasOutput ? [] : ['Regenerate response', 'Escalate to main agent'] }; } }, { id: 'response-time', name: 'Response Time Check', description: 'Validates response was generated in reasonable time', severity: 'low', autoFix: false, check: async (result: DelegationResult) => { const maxTime = 120000; // 2 minutes const passed = result.duration <= maxTime; return { passed, score: Math.max(0, 1 - result.duration / maxTime), issues: passed ? [] : [{ severity: 'low' as const, message: `Response took ${result.duration}ms (max: ${maxTime}ms)` }], recommendations: passed ? [] : ['Consider optimizing agent performance'] }; } }, { id: 'success-verification', name: 'Success Verification', description: 'Confirms the delegation completed successfully', severity: 'critical', autoFix: false, check: async (result: DelegationResult) => { return { passed: result.success, score: result.success ? 1 : 0, issues: result.success ? [] : [{ severity: 'critical' as const, message: result.escalationReason || 'Delegation failed' }], recommendations: result.success ? [] : ['Retry with different agent', 'Escalate to main agent'] }; } } ]; // ============================================================================ // Quality Gate Class // ============================================================================ export class QualityGate { private config: QualityGateConfig; private checks: Map = new Map(); private results: Map = new Map(); constructor(config: Partial = {}) { this.config = { ...DEFAULT_CONFIG, ...config }; // Register built-in checks BUILT_IN_CHECKS.forEach(check => this.registerCheck(check)); // Register custom checks this.config.checks.forEach(check => this.registerCheck(check)); } // ============================================================================ // Check Registration // ============================================================================ /** * Register a quality check */ registerCheck(check: QualityCheck): void { this.checks.set(check.id, check); } /** * Remove a quality check */ removeCheck(checkId: string): void { this.checks.delete(checkId); } /** * Get all registered checks */ getChecks(): QualityCheck[] { return Array.from(this.checks.values()); } // ============================================================================ // Result Validation // ============================================================================ /** * Validate a delegation result */ async validate( result: DelegationResult, context: DelegationContext ): Promise<{ passed: boolean; overallScore: number; checkResults: Map; shouldEscalate: boolean; issues: QualityIssue[]; }> { if (!this.config.enabled) { return { passed: result.success, overallScore: result.confidence, checkResults: new Map(), shouldEscalate: false, issues: [] }; } const checkResults = new Map(); const allIssues: QualityIssue[] = []; let totalScore = 0; let criticalFailures = 0; // Run all checks for (const [id, check] of this.checks) { try { const checkResult = await check.check(result, context); checkResults.set(id, checkResult); totalScore += checkResult.score; // Collect issues checkResult.issues.forEach(issue => { allIssues.push({ ...issue, location: issue.location || check.name }); }); // Count critical failures if (!checkResult.passed && check.severity === 'critical') { criticalFailures++; } } catch (error) { console.error(`Quality check ${id} failed:`, error); checkResults.set(id, { passed: false, score: 0, issues: [{ severity: 'medium', message: `Check failed: ${error instanceof Error ? error.message : 'Unknown error'}` }], recommendations: [] }); } } // Calculate overall score const overallScore = totalScore / this.checks.size; // Determine if passed const passed = criticalFailures === 0 && overallScore >= this.config.minConfidence; // Determine if should escalate const shouldEscalate = this.config.autoEscalate && ( overallScore < this.config.escalationThreshold || criticalFailures > 0 || allIssues.some(i => i.severity === 'critical') ); // Store results if (context.requestId) { this.results.set(context.requestId, Array.from(checkResults.values())); } return { passed, overallScore, checkResults, shouldEscalate, issues: allIssues }; } /** * Quick validation for fast path */ quickValidate(result: DelegationResult): { passed: boolean; shouldEscalate: boolean; } { if (!this.config.enabled) { return { passed: result.success, shouldEscalate: false }; } const passed = result.success && result.confidence >= this.config.minConfidence && !!result.output; const shouldEscalate = this.config.autoEscalate && ( !result.success || result.confidence < this.config.escalationThreshold ); return { passed, shouldEscalate }; } // ============================================================================ // Escalation Decision // ============================================================================ /** * Decide if result should be escalated to main agent */ shouldEscalate( result: DelegationResult, validation: { overallScore: number; issues: QualityIssue[] } ): { escalate: boolean; reason: string; recommendedAction: string; } { if (!this.config.autoEscalate) { return { escalate: false, reason: 'Auto-escalation disabled', recommendedAction: 'Return result as-is' }; } // Check confidence if (result.confidence < this.config.escalationThreshold) { return { escalate: true, reason: `Confidence ${result.confidence.toFixed(2)} below threshold ${this.config.escalationThreshold}`, recommendedAction: 'Forward to main agent for review' }; } // Check for critical issues const criticalIssues = validation.issues.filter(i => i.severity === 'critical'); if (criticalIssues.length > 0) { return { escalate: true, reason: `Critical issues: ${criticalIssues.map(i => i.message).join(', ')}`, recommendedAction: 'Escalate with context for resolution' }; } // Check for high severity issues const highIssues = validation.issues.filter(i => i.severity === 'high'); if (highIssues.length > 2) { return { escalate: true, reason: `Multiple high-severity issues: ${highIssues.length}`, recommendedAction: 'Consider escalation for quality assurance' }; } return { escalate: false, reason: 'Result meets quality standards', recommendedAction: 'Return result to user' }; } // ============================================================================ // Statistics // ============================================================================ getStats(): { totalChecks: number; validationHistory: number; averageScore: number; escalationRate: number; } { const allResults = Array.from(this.results.values()).flat(); const totalChecks = this.checks.size; const averageScore = allResults.length > 0 ? allResults.reduce((sum, r) => sum + r.score, 0) / allResults.length : 0; return { totalChecks, validationHistory: this.results.size, averageScore, escalationRate: 0 // Would need to track escalations }; } /** * Clear validation history */ clearHistory(): void { this.results.clear(); } } // ============================================================================ // Factory Function // ============================================================================ export function createQualityGate(config?: Partial): QualityGate { return new QualityGate(config); } // ============================================================================ // Export // ============================================================================ export default QualityGate;