diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a8f390e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,100 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.1.0] - 2025-03-03 + +### Added +- **Claude Code Integration** (`agent-system/integrations/claude-code.ts`) + - Full context compaction support for Claude Code CLI/IDE + - Subagent spawning with 6 predefined types + - Parallel subagent execution + - Persistent memory (remember/recall) + - Session management + - Tool registration for Claude API + - Auto-compaction on threshold + +- **OpenClaw Integration** (`agent-system/integrations/openclaw.ts`) + - Deterministic pipeline creation + - Parallel agent execution (4 projects × 3 roles pattern) + - Workspace isolation with permissions and quotas + - Lobster YAML workflow parser + - Hook system for events + - Context save/restore across sessions + +- **Examples** (`examples/`) + - Basic context compaction example + - Claude Code integration example + - OpenClaw integration example + - State machine pipeline example + +- **Project Files** + - `package.json` for npm/bun compatibility + - `tsconfig.json` for TypeScript configuration + - `LICENSE` (MIT) + +### Changed +- Updated `agent-system/index.ts` to export new integrations +- Updated README with comprehensive documentation and hero section + +## [1.0.0] - 2025-03-03 + +### Added +- **Agent System** (`agent-system/`) + - `core/token-counter.ts` - Token counting and budget management + - `core/summarizer.ts` - LLM-powered conversation summarization + - `core/context-manager.ts` - Context compaction with 4 strategies + - `core/orchestrator.ts` - Agent lifecycle management + - `core/subagent-spawner.ts` - Subagent creation and execution + - `agents/base-agent.ts` - Base agent class + - `agents/task-agent.ts` - Task-specific agent + - `storage/memory-store.ts` - Persistent storage + - `utils/helpers.ts` - Utility functions + +- **Pipeline System** (`pipeline-system/`) + - `core/state-machine.ts` - Deterministic state machine + - `engine/parallel-executor.ts` - Parallel execution engine + - `events/event-bus.ts` - Event-driven coordination + - `workspace/agent-workspace.ts` - Workspace isolation + - `workflows/yaml-workflow.ts` - YAML workflow parser + - `integrations/claude-code.ts` - Claude Code integration layer + +- **Downloads** (`downloads/`) + - `agent-system.zip` - Agent system package + - `pipeline-system.zip` - Pipeline system package + - `complete-agent-pipeline-system.zip` - Full system package + +- **Documentation** + - Comprehensive README with examples + - API reference tables + - Architecture diagrams + +--- + +## Release Notes + +### v1.1.0 - Integration Release + +This release adds full integration support for both Claude Code and OpenClaw, making it easy to incorporate context compaction and deterministic pipeline orchestration into existing AI development workflows. + +**Key Highlights:** +- One-line integration with Claude Code CLI +- OpenClaw-compatible pipeline definitions +- Support for the 4×3 parallel agent pattern (4 projects × 3 roles) +- Persistent memory across sessions + +### v1.0.0 - Initial Release + +Initial release of the Agentic Compaction & Pipeline System, providing: +- Token-aware context management +- 4 compaction strategies +- Deterministic state machine +- Parallel execution engine +- Event-driven coordination + +--- + +*All releases are 100% autonomously built by Z.AI GLM-5* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a79967b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Z.AI GLM-5 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/examples/01-basic-compaction.ts b/examples/01-basic-compaction.ts new file mode 100644 index 0000000..fb52388 --- /dev/null +++ b/examples/01-basic-compaction.ts @@ -0,0 +1,69 @@ +/** + * Example: Basic Context Compaction + * + * Demonstrates how to use context compaction to manage + * conversation history within token limits. + */ + +import { + ContextManager, + TokenCounter, + Summarizer +} from '../agent-system'; + +async function basicCompaction() { + // Initialize components with a 128k token budget + const tokenCounter = new TokenCounter(128000); + const summarizer = new Summarizer(); + + const contextManager = new ContextManager(tokenCounter, summarizer, { + maxTokens: 100000, // Max tokens before compaction triggers + compactionStrategy: 'hybrid', // Use all strategies combined + slidingWindowSize: 50, // Keep last 50 messages + preserveRecentCount: 10, // Keep last 10 messages verbatim + priorityKeywords: [ // Always retain messages with these keywords + 'error', 'important', 'decision', 'critical', 'remember' + ], + reserveTokens: 20000 // Reserve for response generation + }); + + // Simulate a long conversation + const messages = [ + { role: 'user' as const, content: 'I need to build a REST API for my application.' }, + { role: 'assistant' as const, content: 'I can help you design and implement a REST API. What features do you need?' }, + { role: 'user' as const, content: 'I need CRUD operations for users, products, and orders.' }, + { role: 'assistant' as const, content: 'Great! Let\'s design the API endpoints...' }, + // ... many more messages ... + ]; + + // Add messages to context + for (const msg of messages) { + contextManager.addMessage(msg); + } + + // Check if compaction is needed + if (contextManager.needsCompaction()) { + console.log('Context is full, compacting...'); + + const result = await contextManager.compact(); + + console.log('Compaction result:'); + console.log(` - Tokens saved: ${result.tokensSaved}`); + console.log(` - Messages removed: ${result.messagesRemoved}`); + if (result.summary) { + console.log(` - Summary: ${result.summary.substring(0, 100)}...`); + } + } + + // Get active context for API call + const activeContext = contextManager.getActiveContext(); + console.log(`Active context: ${activeContext.messages.length} messages`); + console.log(`Token usage: ${tokenCounter.getUsagePercentage() * 100}%`); + + // Get token stats + const stats = tokenCounter.getStats(); + console.log(`Token stats: ${stats.used}/${stats.total} (${stats.percentage}%)`); +} + +// Run the example +basicCompaction().catch(console.error); diff --git a/examples/02-claude-code-integration.ts b/examples/02-claude-code-integration.ts new file mode 100644 index 0000000..5b4d8cd --- /dev/null +++ b/examples/02-claude-code-integration.ts @@ -0,0 +1,100 @@ +/** + * Example: Claude Code Integration + * + * Demonstrates how to use the Claude Code integration + * for context-aware AI applications. + */ + +import { ClaudeCodeIntegration } from '../agent-system/integrations/claude-code'; + +async function claudeCodeExample() { + // Initialize Claude Code integration + const claude = new ClaudeCodeIntegration({ + maxContextTokens: 200000, // Claude's 200k context window + reserveTokens: 40000, // Reserve for response + compactionStrategy: 'hybrid', + autoCompact: true, // Auto-compact when threshold reached + compactionThreshold: 0.75, // Compact at 75% capacity + enableSubagents: true, // Enable subagent spawning + maxSubagents: 6, + persistentMemory: true, + memoryStorePath: '.claude-code/memory' + }); + + console.log('Session ID:', claude.getSessionId()); + + // Add user message + claude.addMessage({ + role: 'user', + content: 'Analyze this codebase and identify potential security issues.', + metadata: { + priority: 1, + fileReferences: ['src/auth.ts', 'src/middleware.ts'] + } + }); + + // Check token stats + const stats = claude.getTokenStats(); + console.log(`Token usage: ${stats.percentage}% (${stats.used}/${stats.total})`); + + // Get context for Claude API + const { messages, systemPrompt } = claude.getContextForAPI(); + console.log(`Prepared ${messages.length} messages for API`); + if (systemPrompt) { + console.log('System prompt includes context summary'); + } + + // Spawn a researcher subagent + console.log('\n--- Spawning Researcher Subagent ---'); + const researcherResult = await claude.spawnSubagent({ + type: 'researcher', + prompt: 'Research best practices for JWT authentication in TypeScript', + priority: 'high' + }); + + console.log('Researcher result:', { + success: researcherResult.success, + tokens: researcherResult.tokens, + duration: `${researcherResult.duration}ms` + }); + + // Spawn multiple subagents in parallel (OpenClaw pattern: 4×3) + console.log('\n--- Parallel Subagent Execution ---'); + const parallelResults = await claude.executeParallelSubagents([ + { type: 'explorer', prompt: 'Find all authentication endpoints' }, + { type: 'explorer', prompt: 'Find all database queries' }, + { type: 'reviewer', prompt: 'Review input validation logic' } + ]); + + console.log(`Completed ${parallelResults.length} subagent tasks`); + + // Use persistent memory + await claude.remember('projectContext', { + name: 'MyApp', + type: 'REST API', + securityScan: 'in-progress' + }); + + const savedContext = await claude.recall('projectContext'); + console.log('Recalled context:', savedContext); + + // Save context for later restoration + await claude.saveContext('security-audit-milestone'); + console.log('Context saved for future sessions'); + + // Get session info + const session = claude.getSessionInfo(); + console.log('\nSession Info:', { + id: session.id, + messageCount: session.messageCount, + tokenUsage: session.tokenUsage, + status: session.status + }); + + // Get registered tools + const tools = claude.getTools(); + console.log(`\nRegistered tools: ${tools.map(t => t.name).join(', ')}`); +} + +// Run the example +claudeCodeExample().catch(console.error); diff --git a/examples/03-openclaw-integration.ts b/examples/03-openclaw-integration.ts new file mode 100644 index 0000000..aeeb32c --- /dev/null +++ b/examples/03-openclaw-integration.ts @@ -0,0 +1,164 @@ +/** + * Example: OpenClaw Integration + * + * Demonstrates how to use OpenClaw integration for + * deterministic multi-agent pipeline orchestration. + */ + +import { OpenClawIntegration } from '../agent-system/integrations/openclaw'; + +async function openclawExample() { + // Initialize OpenClaw integration + const openclaw = new OpenClawIntegration({ + maxContextTokens: 200000, + compactionStrategy: 'hybrid', + workspaceIsolation: true, + enableLobsterWorkflows: true, + enableParallelExecution: true, + maxParallelAgents: 12, // 4 projects × 3 roles + hooks: { + onCompactionStart: (ctx) => { + console.log(`[Hook] Compaction started for context ${ctx.id}`); + }, + onCompactionEnd: (result) => { + console.log(`[Hook] Compaction complete: saved ${result.tokensSaved} tokens`); + }, + onStateTransition: (from, to, ctx) => { + console.log(`[Hook] State transition: ${from} → ${to}`); + } + } + }); + + console.log('Context ID:', openclaw.getContext().id); + + // Add message with OpenClaw context + openclaw.addMessage({ + role: 'user', + content: 'Implement user authentication with JWT tokens', + tags: ['feature', 'auth', 'security'], + references: { + files: ['src/auth.ts', 'src/middleware.ts'], + functions: ['login', 'verifyToken', 'refreshToken'] + } + }); + + // Create a deterministic pipeline + console.log('\n--- Creating Pipeline ---'); + const pipeline = openclaw.createPipeline({ + name: 'feature-development', + description: 'Complete feature development workflow with deterministic state machine', + states: [ + { + name: 'analyze', + type: 'parallel', + agents: ['explorer', 'researcher'], + transitions: [ + { target: 'design', event: 'analysis_complete' } + ] + }, + { + name: 'design', + type: 'sequential', + agents: ['planner'], + onEnter: 'agent:planner', + transitions: [ + { target: 'implement', event: 'design_approved' } + ] + }, + { + name: 'implement', + type: 'parallel', + agents: ['coder'], + transitions: [ + { target: 'review', event: 'implementation_complete' } + ] + }, + { + name: 'review', + type: 'sequential', + agents: ['reviewer'], + timeout: 3600000, // 1 hour + transitions: [ + { target: 'complete', event: 'approved' }, + { target: 'implement', event: 'rejected' } + ] + }, + { + name: 'complete', + type: 'sequential', + transitions: [] + } + ] + }); + + console.log('Pipeline created:', pipeline.id); + console.log('Initial state:', pipeline.currentState); + + // Start the pipeline + await openclaw.startPipeline(pipeline.id); + console.log('Pipeline started'); + + // Execute parallel agents (4 projects × 3 roles pattern) + console.log('\n--- Parallel Agent Execution (4×3 Pattern) ---'); + const projects = ['frontend', 'backend', 'database', 'tests']; + const roles = ['security', 'performance', 'quality'] as const; + + const tasks = projects.flatMap(project => + roles.map(role => ({ + type: 'explorer' as const, + prompt: `Analyze ${project} for ${role} considerations`, + context: { project, role } + })) + ); + + console.log(`Executing ${tasks.length} agents in parallel...`); + const results = await openclaw.executeParallelAgents(tasks); + + for (const [agentId, result] of results) { + console.log(`Agent ${agentId}: ${result.success ? '✓' : '✗'}`); + } + + // Transition pipeline states + console.log('\n--- Pipeline State Transitions ---'); + await openclaw.transitionPipeline(pipeline.id, 'analysis_complete'); + console.log('Current state:', openclaw.getPipelineStatus(pipeline.id)?.currentState); + + await openclaw.transitionPipeline(pipeline.id, 'design_approved'); + console.log('Current state:', openclaw.getPipelineStatus(pipeline.id)?.currentState); + + // Create isolated workspace + console.log('\n--- Workspace Isolation ---'); + const workspace = await openclaw.createWorkspace({ + permissions: ['read', 'write'], + quota: { + maxFiles: 1000, + maxSize: 100 * 1024 * 1024 // 100MB + } + }); + console.log('Workspace created:', workspace.id); + + // Use persistent memory + await openclaw.remember('pipelineState', { + pipelineId: pipeline.id, + currentState: openclaw.getPipelineStatus(pipeline.id)?.currentState + }); + + // Save context + await openclaw.saveContext('feature-auth'); + console.log('Context saved'); + + // Get token stats + const tokenStats = openclaw.getTokenStats(); + console.log('\nToken Stats:', tokenStats); + + // Export full state + const state = openclaw.exportState(); + console.log('\nExported state:', { + messages: state.context.messages.length, + agents: state.agents.length, + pipelines: state.pipelines.length + }); +} + +// Run the example +openclawExample().catch(console.error); diff --git a/examples/04-state-machine-pipeline.ts b/examples/04-state-machine-pipeline.ts new file mode 100644 index 0000000..ea18638 --- /dev/null +++ b/examples/04-state-machine-pipeline.ts @@ -0,0 +1,150 @@ +/** + * Example: Deterministic State Machine Pipeline + * + * Demonstrates how to use the deterministic state machine + * and parallel execution engine for multi-agent workflows. + */ + +import { + DeterministicStateMachine, + ParallelExecutionEngine, + EventBus +} from '../pipeline-system'; + +async function stateMachineExample() { + // Create event bus for coordination + const eventBus = new EventBus(); + + // Subscribe to events + eventBus.subscribe('state:*', (event) => { + console.log(`[Event] ${event.type}:`, event.data); + }); + + // Define state machine configuration + const smConfig = { + initialState: 'idle', + states: { + idle: { + onEnter: async (context: any) => { + console.log('Entering IDLE state'); + context.startTime = Date.now(); + }, + transitions: [ + { + target: 'initializing', + event: 'start', + condition: (ctx: any) => ctx.config !== null + } + ] + }, + initializing: { + onEnter: async (context: any) => { + console.log('Initializing...'); + // Simulate async initialization + await new Promise(resolve => setTimeout(resolve, 100)); + }, + onExit: async (context: any) => { + console.log('Initialization complete'); + context.initialized = true; + }, + transitions: [ + { target: 'processing', event: 'initialized' } + ] + }, + processing: { + onEnter: async (context: any) => { + console.log('Processing started'); + }, + transitions: [ + { target: 'completed', event: 'done', condition: (ctx: any) => ctx.success }, + { target: 'error', event: 'error' } + ] + }, + completed: { + onEnter: async (context: any) => { + context.endTime = Date.now(); + console.log(`Completed in ${context.endTime - context.startTime}ms`); + }, + final: true + }, + error: { + onEnter: async (context: any) => { + console.error('Error state reached'); + }, + final: true + } + } + }; + + // Create state machine + const stateMachine = new DeterministicStateMachine(smConfig); + + // Initialize with context + const context = { config: { maxRetries: 3 }, success: true }; + + // Start the state machine + console.log('--- Starting State Machine ---'); + await stateMachine.start(context); + console.log('Current state:', stateMachine.getState()); + + // Trigger transitions + console.log('\n--- Triggering Transitions ---'); + + // Check if transition is valid + if (stateMachine.canTransition('start')) { + await stateMachine.transition('start', context); + console.log('After start:', stateMachine.getState()); + } + + if (stateMachine.canTransition('initialized')) { + await stateMachine.transition('initialized', context); + console.log('After initialized:', stateMachine.getState()); + } + + if (stateMachine.canTransition('done')) { + await stateMachine.transition('done', context); + console.log('After done:', stateMachine.getState()); + } + + // Check if final state + console.log('\nIs final state:', stateMachine.isFinalState()); + + // Now demonstrate parallel execution + console.log('\n--- Parallel Execution Engine ---'); + + const executor = new ParallelExecutionEngine({ + maxConcurrency: 4, + taskQueue: 'priority', + timeout: 30000 + }); + + // Define tasks + const tasks = [ + { id: 'task-1', type: 'analyze', priority: 1, data: { target: 'module-a' } }, + { id: 'task-2', type: 'analyze', priority: 2, data: { target: 'module-b' } }, + { id: 'task-3', type: 'analyze', priority: 1, data: { target: 'module-c' } }, + { id: 'task-4', type: 'analyze', priority: 3, data: { target: 'module-d' } } + ]; + + // Execute in parallel with custom handler + const results = await executor.executeAll(tasks, async (task) => { + console.log(`Processing ${task.id}...`); + await new Promise(resolve => setTimeout(resolve, 100)); + return { taskId: task.id, result: `Analyzed ${task.data.target}` }; + }); + + console.log('\nResults:', results); + + // Get execution stats + const stats = executor.getStats(); + console.log('Execution stats:', stats); + + // Cleanup + executor.shutdown(); + eventBus.clearAllSubscriptions(); + + console.log('\n--- Example Complete ---'); +} + +// Run the example +stateMachineExample().catch(console.error); diff --git a/package.json b/package.json new file mode 100644 index 0000000..a88f999 --- /dev/null +++ b/package.json @@ -0,0 +1,67 @@ +{ + "name": "agentic-compaction-pipeline", + "version": "1.1.0", + "description": "Comprehensive context compaction and deterministic multi-agent pipeline orchestration system with Claude Code and OpenClaw integration", + "main": "agent-system/index.ts", + "types": "agent-system/index.ts", + "scripts": { + "build": "tsc", + "lint": "eslint . --ext .ts", + "test": "jest", + "clean": "rm -rf dist" + }, + "keywords": [ + "ai", + "agents", + "context-compaction", + "claude-code", + "openclaw", + "multi-agent", + "pipeline", + "orchestration", + "token-management", + "llm" + ], + "author": "Z.AI GLM-5", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.rommark.dev/admin/Agentic-Compaction-and-Pipleline-by-GLM-5.git" + }, + "bugs": { + "url": "https://github.rommark.dev/admin/Agentic-Compaction-and-Pipleline-by-GLM-5/issues" + }, + "homepage": "https://github.rommark.dev/admin/Agentic-Compaction-and-Pipleline-by-GLM-5", + "engines": { + "node": ">=18.0.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + }, + "peerDependencies": { + "typescript": ">=5.0.0" + }, + "exports": { + ".": { + "types": "./agent-system/index.ts", + "default": "./agent-system/index.ts" + }, + "./agent-system": { + "types": "./agent-system/index.ts", + "default": "./agent-system/index.ts" + }, + "./pipeline-system": { + "types": "./pipeline-system/index.ts", + "default": "./pipeline-system/index.ts" + }, + "./integrations/claude-code": { + "types": "./agent-system/integrations/claude-code.ts", + "default": "./agent-system/integrations/claude-code.ts" + }, + "./integrations/openclaw": { + "types": "./agent-system/integrations/openclaw.ts", + "default": "./agent-system/integrations/openclaw.ts" + } + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..ed0121e --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,34 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": ".", + "resolveJsonModule": true, + "allowSyntheticDefaultImports": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "exactOptionalPropertyTypes": false, + "noFallthroughCasesInSwitch": true + }, + "include": [ + "agent-system/**/*.ts", + "pipeline-system/**/*.ts" + ], + "exclude": [ + "node_modules", + "dist", + "downloads" + ] +}