Add Delegation System with 3rd Party AI Tool Integration
NEW: Delegation System (v1.2.0) - Request Classifier for fast request analysis (<50ms) - Agent Pool Manager with auto-scaling (8 agent types) - Delegation Engine with 4 strategies (full, parallel, hierarchical, hybrid) - Progress Streamer for real-time updates - Context Handoff Protocol for inter-agent communication - Quality Gate with confidence thresholds and auto-escalation NEW: 3rd Party Integration Adapters - OpenClaw adapter with parallel execution support - Claude Code CLI adapter with tool registration - Generic adapter for custom integrations - Standardized IntegrationAdapter interface Agent Types Added: - fast-responder (quick answers < 2s) - explorer (code navigation) - researcher (deep analysis) - coder (implementation) - reviewer (quality checks) - planner (architecture) - executor (commands) - analyzer (debugging) Tests: All 6 tests passing This project was 100% autonomously built by Z.AI GLM-5
This commit is contained in:
admin
378
delegation-system/quality/quality-gate.ts
Normal file
378
delegation-system/quality/quality-gate.ts
Normal file
@@ -0,0 +1,378 @@
|
||||
/**
|
||||
* Quality Gate
|
||||
*
|
||||
* Validates delegation results before returning to users.
|
||||
* Implements confidence thresholds and automatic escalation.
|
||||
*/
|
||||
|
||||
import {
|
||||
QualityCheck,
|
||||
QualityCheckResult,
|
||||
QualityIssue,
|
||||
QualityGateConfig,
|
||||
DelegationResult,
|
||||
DelegationContext
|
||||
} from '../core/types';
|
||||
|
||||
// ============================================================================
|
||||
// Default Configuration
|
||||
// ============================================================================
|
||||
|
||||
const DEFAULT_CONFIG: QualityGateConfig = {
|
||||
enabled: true,
|
||||
minConfidence: 0.7,
|
||||
checks: [],
|
||||
escalationThreshold: 0.5,
|
||||
autoEscalate: true
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Built-in Quality Checks
|
||||
// ============================================================================
|
||||
|
||||
const BUILT_IN_CHECKS: QualityCheck[] = [
|
||||
{
|
||||
id: 'confidence-check',
|
||||
name: 'Confidence Threshold',
|
||||
description: 'Ensures result meets minimum confidence threshold',
|
||||
severity: 'high',
|
||||
autoFix: false,
|
||||
check: async (result: DelegationResult) => {
|
||||
const passed = result.confidence >= 0.7;
|
||||
return {
|
||||
passed,
|
||||
score: result.confidence,
|
||||
issues: passed ? [] : [{
|
||||
severity: 'high' as const,
|
||||
message: `Confidence ${result.confidence.toFixed(2)} below threshold 0.70`
|
||||
}],
|
||||
recommendations: passed ? [] : ['Consider escalating to main agent']
|
||||
};
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'output-validity',
|
||||
name: 'Output Validity',
|
||||
description: 'Checks that output is non-empty and meaningful',
|
||||
severity: 'critical',
|
||||
autoFix: false,
|
||||
check: async (result: DelegationResult) => {
|
||||
const hasOutput = result.output && result.output.trim().length > 0;
|
||||
const isNotError = !result.output?.toLowerCase().includes('error');
|
||||
|
||||
return {
|
||||
passed: hasOutput && isNotError,
|
||||
score: hasOutput ? (isNotError ? 1 : 0.5) : 0,
|
||||
issues: [
|
||||
...(hasOutput ? [] : [{ severity: 'critical' as const, message: 'Empty output' }]),
|
||||
...(isNotError ? [] : [{ severity: 'high' as const, message: 'Output contains error message' }])
|
||||
],
|
||||
recommendations: hasOutput ? [] : ['Regenerate response', 'Escalate to main agent']
|
||||
};
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'response-time',
|
||||
name: 'Response Time Check',
|
||||
description: 'Validates response was generated in reasonable time',
|
||||
severity: 'low',
|
||||
autoFix: false,
|
||||
check: async (result: DelegationResult) => {
|
||||
const maxTime = 120000; // 2 minutes
|
||||
const passed = result.duration <= maxTime;
|
||||
|
||||
return {
|
||||
passed,
|
||||
score: Math.max(0, 1 - result.duration / maxTime),
|
||||
issues: passed ? [] : [{
|
||||
severity: 'low' as const,
|
||||
message: `Response took ${result.duration}ms (max: ${maxTime}ms)`
|
||||
}],
|
||||
recommendations: passed ? [] : ['Consider optimizing agent performance']
|
||||
};
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'success-verification',
|
||||
name: 'Success Verification',
|
||||
description: 'Confirms the delegation completed successfully',
|
||||
severity: 'critical',
|
||||
autoFix: false,
|
||||
check: async (result: DelegationResult) => {
|
||||
return {
|
||||
passed: result.success,
|
||||
score: result.success ? 1 : 0,
|
||||
issues: result.success ? [] : [{
|
||||
severity: 'critical' as const,
|
||||
message: result.escalationReason || 'Delegation failed'
|
||||
}],
|
||||
recommendations: result.success ? [] : ['Retry with different agent', 'Escalate to main agent']
|
||||
};
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
// ============================================================================
|
||||
// Quality Gate Class
|
||||
// ============================================================================
|
||||
|
||||
export class QualityGate {
|
||||
private config: QualityGateConfig;
|
||||
private checks: Map<string, QualityCheck> = new Map();
|
||||
private results: Map<string, QualityCheckResult[]> = new Map();
|
||||
|
||||
constructor(config: Partial<QualityGateConfig> = {}) {
|
||||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||
|
||||
// Register built-in checks
|
||||
BUILT_IN_CHECKS.forEach(check => this.registerCheck(check));
|
||||
|
||||
// Register custom checks
|
||||
this.config.checks.forEach(check => this.registerCheck(check));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Check Registration
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Register a quality check
|
||||
*/
|
||||
registerCheck(check: QualityCheck): void {
|
||||
this.checks.set(check.id, check);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a quality check
|
||||
*/
|
||||
removeCheck(checkId: string): void {
|
||||
this.checks.delete(checkId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered checks
|
||||
*/
|
||||
getChecks(): QualityCheck[] {
|
||||
return Array.from(this.checks.values());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Result Validation
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Validate a delegation result
|
||||
*/
|
||||
async validate(
|
||||
result: DelegationResult,
|
||||
context: DelegationContext
|
||||
): Promise<{
|
||||
passed: boolean;
|
||||
overallScore: number;
|
||||
checkResults: Map<string, QualityCheckResult>;
|
||||
shouldEscalate: boolean;
|
||||
issues: QualityIssue[];
|
||||
}> {
|
||||
if (!this.config.enabled) {
|
||||
return {
|
||||
passed: result.success,
|
||||
overallScore: result.confidence,
|
||||
checkResults: new Map(),
|
||||
shouldEscalate: false,
|
||||
issues: []
|
||||
};
|
||||
}
|
||||
|
||||
const checkResults = new Map<string, QualityCheckResult>();
|
||||
const allIssues: QualityIssue[] = [];
|
||||
let totalScore = 0;
|
||||
let criticalFailures = 0;
|
||||
|
||||
// Run all checks
|
||||
for (const [id, check] of this.checks) {
|
||||
try {
|
||||
const checkResult = await check.check(result, context);
|
||||
checkResults.set(id, checkResult);
|
||||
totalScore += checkResult.score;
|
||||
|
||||
// Collect issues
|
||||
checkResult.issues.forEach(issue => {
|
||||
allIssues.push({
|
||||
...issue,
|
||||
location: issue.location || check.name
|
||||
});
|
||||
});
|
||||
|
||||
// Count critical failures
|
||||
if (!checkResult.passed && check.severity === 'critical') {
|
||||
criticalFailures++;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Quality check ${id} failed:`, error);
|
||||
checkResults.set(id, {
|
||||
passed: false,
|
||||
score: 0,
|
||||
issues: [{
|
||||
severity: 'medium',
|
||||
message: `Check failed: ${error instanceof Error ? error.message : 'Unknown error'}`
|
||||
}],
|
||||
recommendations: []
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate overall score
|
||||
const overallScore = totalScore / this.checks.size;
|
||||
|
||||
// Determine if passed
|
||||
const passed = criticalFailures === 0 && overallScore >= this.config.minConfidence;
|
||||
|
||||
// Determine if should escalate
|
||||
const shouldEscalate = this.config.autoEscalate && (
|
||||
overallScore < this.config.escalationThreshold ||
|
||||
criticalFailures > 0 ||
|
||||
allIssues.some(i => i.severity === 'critical')
|
||||
);
|
||||
|
||||
// Store results
|
||||
if (context.requestId) {
|
||||
this.results.set(context.requestId, Array.from(checkResults.values()));
|
||||
}
|
||||
|
||||
return {
|
||||
passed,
|
||||
overallScore,
|
||||
checkResults,
|
||||
shouldEscalate,
|
||||
issues: allIssues
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Quick validation for fast path
|
||||
*/
|
||||
quickValidate(result: DelegationResult): {
|
||||
passed: boolean;
|
||||
shouldEscalate: boolean;
|
||||
} {
|
||||
if (!this.config.enabled) {
|
||||
return { passed: result.success, shouldEscalate: false };
|
||||
}
|
||||
|
||||
const passed = result.success &&
|
||||
result.confidence >= this.config.minConfidence &&
|
||||
!!result.output;
|
||||
|
||||
const shouldEscalate = this.config.autoEscalate && (
|
||||
!result.success ||
|
||||
result.confidence < this.config.escalationThreshold
|
||||
);
|
||||
|
||||
return { passed, shouldEscalate };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Escalation Decision
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Decide if result should be escalated to main agent
|
||||
*/
|
||||
shouldEscalate(
|
||||
result: DelegationResult,
|
||||
validation: { overallScore: number; issues: QualityIssue[] }
|
||||
): {
|
||||
escalate: boolean;
|
||||
reason: string;
|
||||
recommendedAction: string;
|
||||
} {
|
||||
if (!this.config.autoEscalate) {
|
||||
return {
|
||||
escalate: false,
|
||||
reason: 'Auto-escalation disabled',
|
||||
recommendedAction: 'Return result as-is'
|
||||
};
|
||||
}
|
||||
|
||||
// Check confidence
|
||||
if (result.confidence < this.config.escalationThreshold) {
|
||||
return {
|
||||
escalate: true,
|
||||
reason: `Confidence ${result.confidence.toFixed(2)} below threshold ${this.config.escalationThreshold}`,
|
||||
recommendedAction: 'Forward to main agent for review'
|
||||
};
|
||||
}
|
||||
|
||||
// Check for critical issues
|
||||
const criticalIssues = validation.issues.filter(i => i.severity === 'critical');
|
||||
if (criticalIssues.length > 0) {
|
||||
return {
|
||||
escalate: true,
|
||||
reason: `Critical issues: ${criticalIssues.map(i => i.message).join(', ')}`,
|
||||
recommendedAction: 'Escalate with context for resolution'
|
||||
};
|
||||
}
|
||||
|
||||
// Check for high severity issues
|
||||
const highIssues = validation.issues.filter(i => i.severity === 'high');
|
||||
if (highIssues.length > 2) {
|
||||
return {
|
||||
escalate: true,
|
||||
reason: `Multiple high-severity issues: ${highIssues.length}`,
|
||||
recommendedAction: 'Consider escalation for quality assurance'
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
escalate: false,
|
||||
reason: 'Result meets quality standards',
|
||||
recommendedAction: 'Return result to user'
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Statistics
|
||||
// ============================================================================
|
||||
|
||||
getStats(): {
|
||||
totalChecks: number;
|
||||
validationHistory: number;
|
||||
averageScore: number;
|
||||
escalationRate: number;
|
||||
} {
|
||||
const allResults = Array.from(this.results.values()).flat();
|
||||
const totalChecks = this.checks.size;
|
||||
const averageScore = allResults.length > 0
|
||||
? allResults.reduce((sum, r) => sum + r.score, 0) / allResults.length
|
||||
: 0;
|
||||
|
||||
return {
|
||||
totalChecks,
|
||||
validationHistory: this.results.size,
|
||||
averageScore,
|
||||
escalationRate: 0 // Would need to track escalations
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear validation history
|
||||
*/
|
||||
clearHistory(): void {
|
||||
this.results.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Factory Function
|
||||
// ============================================================================
|
||||
|
||||
export function createQualityGate(config?: Partial<QualityGateConfig>): QualityGate {
|
||||
return new QualityGate(config);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Export
|
||||
// ============================================================================
|
||||
|
||||
export default QualityGate;
|
||||
Reference in New Issue
Block a user