Complete Agent Pipeline System with Claude Code & OpenClaw Integration
- Added Claude Code integration with full context compaction support - Added OpenClaw integration with deterministic pipeline support - Implemented parallel agent execution (4 projects x 3 roles pattern) - Added workspace isolation with permissions and quotas - Implemented Lobster-compatible YAML workflow parser - Added persistent memory store for cross-session context - Created comprehensive README with hero section This project was 100% autonomously built by Z.AI GLM-5
This commit is contained in:
admin
653
pipeline-system/core/state-machine.ts
Normal file
653
pipeline-system/core/state-machine.ts
Normal file
@@ -0,0 +1,653 @@
|
||||
/**
|
||||
* Deterministic State Machine Core
|
||||
*
|
||||
* A state machine that controls agent flow WITHOUT LLM decision-making.
|
||||
* States, transitions, and events are defined declaratively.
|
||||
*
|
||||
* Key principle: The LLM does creative work, the state machine handles the plumbing.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export type StateStatus = 'idle' | 'active' | 'waiting' | 'completed' | 'failed' | 'paused';
|
||||
|
||||
export interface State {
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'start' | 'end' | 'action' | 'parallel' | 'choice' | 'wait' | 'loop';
|
||||
agent?: string; // Agent to invoke in this state
|
||||
action?: string; // Action to execute
|
||||
timeout?: number; // Timeout in ms
|
||||
retry?: RetryConfig; // Retry configuration
|
||||
onEnter?: Transition[]; // Transitions on entering state
|
||||
onExit?: Transition[]; // Transitions on exiting state
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface Transition {
|
||||
event: string; // Event that triggers this transition
|
||||
target: string; // Target state ID
|
||||
condition?: Condition; // Optional condition
|
||||
guard?: string; // Guard function name
|
||||
}
|
||||
|
||||
export interface Condition {
|
||||
type: 'equals' | 'contains' | 'exists' | 'custom';
|
||||
field: string;
|
||||
value?: unknown;
|
||||
expression?: string;
|
||||
}
|
||||
|
||||
export interface RetryConfig {
|
||||
maxAttempts: number;
|
||||
backoff: 'fixed' | 'exponential' | 'linear';
|
||||
initialDelay: number;
|
||||
maxDelay: number;
|
||||
}
|
||||
|
||||
export interface StateMachineDefinition {
|
||||
id: string;
|
||||
name: string;
|
||||
version: string;
|
||||
description?: string;
|
||||
initial: string;
|
||||
states: Record<string, State>;
|
||||
events?: string[]; // Allowed events
|
||||
context?: Record<string, unknown>; // Initial context
|
||||
onError?: ErrorHandling;
|
||||
}
|
||||
|
||||
export interface ErrorHandling {
|
||||
strategy: 'fail' | 'retry' | 'transition';
|
||||
targetState?: string;
|
||||
maxRetries?: number;
|
||||
}
|
||||
|
||||
export interface StateMachineInstance {
|
||||
id: string;
|
||||
definition: StateMachineDefinition;
|
||||
currentState: string;
|
||||
previousState?: string;
|
||||
status: StateStatus;
|
||||
context: Record<string, unknown>;
|
||||
history: StateTransition[];
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
startedAt?: Date;
|
||||
completedAt?: Date;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface StateTransition {
|
||||
from: string;
|
||||
to: string;
|
||||
event: string;
|
||||
timestamp: Date;
|
||||
context?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface Event {
|
||||
type: string;
|
||||
source: string;
|
||||
target?: string;
|
||||
payload: unknown;
|
||||
timestamp: Date;
|
||||
correlationId?: string;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// State Machine Engine
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* DeterministicStateMachine - Core engine for deterministic flow control
|
||||
*/
|
||||
export class DeterministicStateMachine extends EventEmitter {
|
||||
private definition: StateMachineDefinition;
|
||||
private instance: StateMachineInstance;
|
||||
private eventQueue: Event[] = [];
|
||||
private processing = false;
|
||||
private timeoutId?: ReturnType<typeof setTimeout>;
|
||||
|
||||
constructor(definition: StateMachineDefinition, instanceId?: string) {
|
||||
super();
|
||||
this.definition = definition;
|
||||
this.instance = this.createInstance(instanceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new state machine instance
|
||||
*/
|
||||
private createInstance(instanceId?: string): StateMachineInstance {
|
||||
return {
|
||||
id: instanceId || randomUUID(),
|
||||
definition: this.definition,
|
||||
currentState: this.definition.initial,
|
||||
status: 'idle',
|
||||
context: { ...this.definition.context } || {},
|
||||
history: [],
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the state machine
|
||||
*/
|
||||
start(): void {
|
||||
if (this.instance.status !== 'idle') {
|
||||
throw new Error(`Cannot start state machine in ${this.instance.status} status`);
|
||||
}
|
||||
|
||||
this.instance.status = 'active';
|
||||
this.instance.startedAt = new Date();
|
||||
this.emit('started', { instance: this.instance });
|
||||
|
||||
// Enter initial state
|
||||
this.enterState(this.instance.currentState);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an event to the state machine
|
||||
*/
|
||||
sendEvent(event: Omit<Event, 'timestamp'>): void {
|
||||
const fullEvent: Event = {
|
||||
...event,
|
||||
timestamp: new Date()
|
||||
};
|
||||
|
||||
this.eventQueue.push(fullEvent);
|
||||
this.emit('eventQueued', { event: fullEvent });
|
||||
|
||||
this.processQueue();
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the event queue
|
||||
*/
|
||||
private async processQueue(): Promise<void> {
|
||||
if (this.processing || this.eventQueue.length === 0) return;
|
||||
|
||||
this.processing = true;
|
||||
|
||||
try {
|
||||
while (this.eventQueue.length > 0 && this.instance.status === 'active') {
|
||||
const event = this.eventQueue.shift()!;
|
||||
await this.handleEvent(event);
|
||||
}
|
||||
} finally {
|
||||
this.processing = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a single event
|
||||
*/
|
||||
private async handleEvent(event: Event): Promise<void> {
|
||||
const currentState = this.getCurrentState();
|
||||
|
||||
this.emit('eventProcessed', { event, state: currentState });
|
||||
|
||||
// Find matching transition
|
||||
const transition = this.findTransition(currentState, event);
|
||||
|
||||
if (!transition) {
|
||||
this.emit('noTransition', { event, state: currentState });
|
||||
return;
|
||||
}
|
||||
|
||||
// Check condition if present
|
||||
if (transition.condition && !this.evaluateCondition(transition.condition)) {
|
||||
this.emit('conditionFailed', { event, transition });
|
||||
return;
|
||||
}
|
||||
|
||||
// Execute transition
|
||||
await this.executeTransition(transition, event);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a matching transition for the event
|
||||
*/
|
||||
private findTransition(state: State, event: Event): Transition | undefined {
|
||||
const transitions = state.onExit || [];
|
||||
return transitions.find(t => {
|
||||
// Check event type match
|
||||
if (t.event !== event.type) return false;
|
||||
|
||||
// Check target filter if event has specific target
|
||||
if (event.target && event.target !== this.instance.id) return false;
|
||||
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate a transition condition
|
||||
*/
|
||||
private evaluateCondition(condition: Condition): boolean {
|
||||
const value = this.getDeepValue(this.instance.context, condition.field);
|
||||
|
||||
switch (condition.type) {
|
||||
case 'equals':
|
||||
return value === condition.value;
|
||||
case 'contains':
|
||||
if (Array.isArray(value)) {
|
||||
return value.includes(condition.value);
|
||||
}
|
||||
return String(value).includes(String(condition.value));
|
||||
case 'exists':
|
||||
return value !== undefined && value !== null;
|
||||
case 'custom':
|
||||
// Custom conditions would be evaluated by a condition registry
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a state transition
|
||||
*/
|
||||
private async executeTransition(transition: Transition, event: Event): Promise<void> {
|
||||
const fromState = this.instance.currentState;
|
||||
const toState = transition.target;
|
||||
|
||||
// Record transition
|
||||
const transitionRecord: StateTransition = {
|
||||
from: fromState,
|
||||
to: toState,
|
||||
event: event.type,
|
||||
timestamp: new Date(),
|
||||
context: { ...this.instance.context }
|
||||
};
|
||||
this.instance.history.push(transitionRecord);
|
||||
|
||||
// Exit current state
|
||||
await this.exitState(fromState);
|
||||
|
||||
// Update instance
|
||||
this.instance.previousState = fromState;
|
||||
this.instance.currentState = toState;
|
||||
this.instance.updatedAt = new Date();
|
||||
|
||||
// Merge event payload into context
|
||||
if (event.payload && typeof event.payload === 'object') {
|
||||
this.instance.context = {
|
||||
...this.instance.context,
|
||||
...event.payload as Record<string, unknown>
|
||||
};
|
||||
}
|
||||
|
||||
this.emit('transition', { from: fromState, to: toState, event });
|
||||
|
||||
// Enter new state
|
||||
await this.enterState(toState);
|
||||
}
|
||||
|
||||
/**
|
||||
* Enter a state
|
||||
*/
|
||||
private async enterState(stateId: string): Promise<void> {
|
||||
const state = this.definition.states[stateId];
|
||||
if (!state) {
|
||||
this.handleError(`State ${stateId} not found`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.emit('enteringState', { state });
|
||||
|
||||
// Handle state types
|
||||
switch (state.type) {
|
||||
case 'end':
|
||||
this.instance.status = 'completed';
|
||||
this.instance.completedAt = new Date();
|
||||
this.emit('completed', { instance: this.instance });
|
||||
break;
|
||||
|
||||
case 'action':
|
||||
// Emit event for external action handler
|
||||
this.emit('action', {
|
||||
state,
|
||||
context: this.instance.context,
|
||||
instanceId: this.instance.id
|
||||
});
|
||||
|
||||
// Set timeout if specified
|
||||
if (state.timeout) {
|
||||
this.setTimeout(state.timeout, stateId);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'parallel':
|
||||
this.handleParallelState(state);
|
||||
break;
|
||||
|
||||
case 'choice':
|
||||
this.handleChoiceState(state);
|
||||
break;
|
||||
|
||||
case 'wait':
|
||||
// Wait for external event
|
||||
this.instance.status = 'waiting';
|
||||
break;
|
||||
|
||||
case 'loop':
|
||||
this.handleLoopState(state);
|
||||
break;
|
||||
|
||||
default:
|
||||
// Process onEnter transitions
|
||||
if (state.onEnter) {
|
||||
for (const transition of state.onEnter) {
|
||||
// Auto-transitions trigger immediately
|
||||
if (transition.event === '*') {
|
||||
await this.executeTransition(transition, {
|
||||
type: '*',
|
||||
source: stateId,
|
||||
payload: {},
|
||||
timestamp: new Date()
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('enteredState', { state });
|
||||
}
|
||||
|
||||
/**
|
||||
* Exit a state
|
||||
*/
|
||||
private async exitState(stateId: string): Promise<void> {
|
||||
const state = this.definition.states[stateId];
|
||||
|
||||
// Clear any pending timeout
|
||||
if (this.timeoutId) {
|
||||
clearTimeout(this.timeoutId);
|
||||
this.timeoutId = undefined;
|
||||
}
|
||||
|
||||
this.emit('exitingState', { state });
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle parallel state (fork into concurrent branches)
|
||||
*/
|
||||
private handleParallelState(state: State): void {
|
||||
this.emit('parallel', {
|
||||
state,
|
||||
branches: state.onEnter?.map(t => t.target) || [],
|
||||
context: this.instance.context
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle choice state (conditional branching)
|
||||
*/
|
||||
private handleChoiceState(state: State): void {
|
||||
const transitions = state.onExit || [];
|
||||
|
||||
for (const transition of transitions) {
|
||||
if (transition.condition && this.evaluateCondition(transition.condition)) {
|
||||
this.sendEvent({
|
||||
type: transition.event,
|
||||
source: state.id,
|
||||
payload: {}
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// No condition matched - use default transition
|
||||
const defaultTransition = transitions.find(t => !t.condition);
|
||||
if (defaultTransition) {
|
||||
this.sendEvent({
|
||||
type: defaultTransition.event,
|
||||
source: state.id,
|
||||
payload: {}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle loop state
|
||||
*/
|
||||
private handleLoopState(state: State): void {
|
||||
const loopCount = (this.instance.context._loopCount as Record<string, number>)?.[state.id] || 0;
|
||||
const maxIterations = (state.metadata?.maxIterations as number) || 3;
|
||||
|
||||
if (loopCount < maxIterations) {
|
||||
// Continue loop
|
||||
this.instance.context._loopCount = {
|
||||
...this.instance.context._loopCount as Record<string, number>,
|
||||
[state.id]: loopCount + 1
|
||||
};
|
||||
|
||||
this.emit('loopIteration', {
|
||||
state,
|
||||
iteration: loopCount + 1,
|
||||
maxIterations
|
||||
});
|
||||
|
||||
// Trigger loop body
|
||||
const loopTransition = state.onExit?.find(t => t.event === 'continue');
|
||||
if (loopTransition) {
|
||||
this.sendEvent({
|
||||
type: 'continue',
|
||||
source: state.id,
|
||||
payload: { iteration: loopCount + 1 }
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Exit loop
|
||||
const exitTransition = state.onExit?.find(t => t.event === 'exit');
|
||||
if (exitTransition) {
|
||||
this.sendEvent({
|
||||
type: 'exit',
|
||||
source: state.id,
|
||||
payload: { iterations: loopCount }
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a timeout for the current state
|
||||
*/
|
||||
private setTimeout(duration: number, stateId: string): void {
|
||||
this.timeoutId = setTimeout(() => {
|
||||
this.emit('timeout', { stateId });
|
||||
this.sendEvent({
|
||||
type: 'timeout',
|
||||
source: stateId,
|
||||
payload: { timedOut: true }
|
||||
});
|
||||
}, duration);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle errors
|
||||
*/
|
||||
private handleError(error: string): void {
|
||||
this.instance.error = error;
|
||||
this.instance.status = 'failed';
|
||||
this.instance.completedAt = new Date();
|
||||
this.emit('error', { error, instance: this.instance });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current state definition
|
||||
*/
|
||||
getCurrentState(): State {
|
||||
return this.definition.states[this.instance.currentState];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get instance info
|
||||
*/
|
||||
getInstance(): StateMachineInstance {
|
||||
return { ...this.instance };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update context
|
||||
*/
|
||||
updateContext(updates: Record<string, unknown>): void {
|
||||
this.instance.context = { ...this.instance.context, ...updates };
|
||||
this.instance.updatedAt = new Date();
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause the state machine
|
||||
*/
|
||||
pause(): void {
|
||||
if (this.instance.status === 'active') {
|
||||
this.instance.status = 'paused';
|
||||
this.emit('paused', { instance: this.instance });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume the state machine
|
||||
*/
|
||||
resume(): void {
|
||||
if (this.instance.status === 'paused') {
|
||||
this.instance.status = 'active';
|
||||
this.emit('resumed', { instance: this.instance });
|
||||
this.processQueue();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel the state machine
|
||||
*/
|
||||
cancel(): void {
|
||||
this.instance.status = 'failed';
|
||||
this.instance.error = 'Cancelled';
|
||||
this.instance.completedAt = new Date();
|
||||
|
||||
if (this.timeoutId) {
|
||||
clearTimeout(this.timeoutId);
|
||||
}
|
||||
|
||||
this.eventQueue = [];
|
||||
this.emit('cancelled', { instance: this.instance });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get deep value from object by dot-notation path
|
||||
*/
|
||||
private getDeepValue(obj: Record<string, unknown>, path: string): unknown {
|
||||
return path.split('.').reduce<unknown>((acc, key) => {
|
||||
if (acc && typeof acc === 'object' && key in acc) {
|
||||
return (acc as Record<string, unknown>)[key];
|
||||
}
|
||||
return undefined;
|
||||
}, obj);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// State Machine Registry
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* StateMachineRegistry - Manages multiple state machine instances
|
||||
*/
|
||||
export class StateMachineRegistry {
|
||||
private definitions: Map<string, StateMachineDefinition> = new Map();
|
||||
private instances: Map<string, DeterministicStateMachine> = new Map();
|
||||
|
||||
/**
|
||||
* Register a state machine definition
|
||||
*/
|
||||
register(definition: StateMachineDefinition): void {
|
||||
this.definitions.set(definition.id, definition);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new instance of a state machine
|
||||
*/
|
||||
createInstance(definitionId: string, instanceId?: string): DeterministicStateMachine {
|
||||
const definition = this.definitions.get(definitionId);
|
||||
if (!definition) {
|
||||
throw new Error(`State machine definition ${definitionId} not found`);
|
||||
}
|
||||
|
||||
const sm = new DeterministicStateMachine(definition, instanceId);
|
||||
this.instances.set(sm.getInstance().id, sm);
|
||||
|
||||
return sm;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an instance by ID
|
||||
*/
|
||||
getInstance(instanceId: string): DeterministicStateMachine | undefined {
|
||||
return this.instances.get(instanceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all instances
|
||||
*/
|
||||
getAllInstances(): DeterministicStateMachine[] {
|
||||
return Array.from(this.instances.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get instances by status
|
||||
*/
|
||||
getInstancesByStatus(status: StateStatus): DeterministicStateMachine[] {
|
||||
return this.getAllInstances().filter(sm => sm.getInstance().status === status);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove an instance
|
||||
*/
|
||||
removeInstance(instanceId: string): boolean {
|
||||
const sm = this.instances.get(instanceId);
|
||||
if (sm) {
|
||||
sm.cancel();
|
||||
return this.instances.delete(instanceId);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics
|
||||
*/
|
||||
getStats(): {
|
||||
definitions: number;
|
||||
instances: number;
|
||||
byStatus: Record<StateStatus, number>;
|
||||
} {
|
||||
const byStatus: Record<StateStatus, number> = {
|
||||
idle: 0,
|
||||
active: 0,
|
||||
waiting: 0,
|
||||
completed: 0,
|
||||
failed: 0,
|
||||
paused: 0
|
||||
};
|
||||
|
||||
for (const sm of this.instances.values()) {
|
||||
byStatus[sm.getInstance().status]++;
|
||||
}
|
||||
|
||||
return {
|
||||
definitions: this.definitions.size,
|
||||
instances: this.instances.size,
|
||||
byStatus
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton registry
|
||||
export const stateMachineRegistry = new StateMachineRegistry();
|
||||
624
pipeline-system/engine/parallel-executor.ts
Normal file
624
pipeline-system/engine/parallel-executor.ts
Normal file
@@ -0,0 +1,624 @@
|
||||
/**
|
||||
* Parallel Execution Engine
|
||||
*
|
||||
* Manages concurrent agent sessions with resource pooling.
|
||||
* Supports: 4 projects × 3 roles = up to 12 concurrent sessions.
|
||||
*
|
||||
* Key features:
|
||||
* - Worker pool with configurable concurrency limits
|
||||
* - Resource isolation per agent session
|
||||
* - Automatic scaling based on load
|
||||
* - Task queuing with priority support
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export type AgentRole = 'programmer' | 'reviewer' | 'tester' | 'planner' | 'analyst' | 'custom';
|
||||
export type TaskStatus = 'pending' | 'queued' | 'running' | 'completed' | 'failed' | 'cancelled';
|
||||
export type WorkerStatus = 'idle' | 'busy' | 'draining' | 'terminated';
|
||||
|
||||
export interface AgentSession {
|
||||
id: string;
|
||||
projectId: string;
|
||||
role: AgentRole;
|
||||
model?: string; // e.g., 'opus', 'sonnet' for cost optimization
|
||||
workspace: string;
|
||||
tools: string[];
|
||||
memory: Record<string, unknown>;
|
||||
identity: AgentIdentity;
|
||||
status: 'active' | 'idle' | 'terminated';
|
||||
createdAt: Date;
|
||||
lastActivity: Date;
|
||||
}
|
||||
|
||||
export interface AgentIdentity {
|
||||
name: string;
|
||||
description: string;
|
||||
personality?: string;
|
||||
systemPrompt?: string;
|
||||
}
|
||||
|
||||
export interface PipelineTask {
|
||||
id: string;
|
||||
projectId: string;
|
||||
role: AgentRole;
|
||||
type: string;
|
||||
description: string;
|
||||
priority: 'low' | 'medium' | 'high' | 'critical';
|
||||
input: unknown;
|
||||
dependencies: string[];
|
||||
timeout: number;
|
||||
retryCount: number;
|
||||
maxRetries: number;
|
||||
status: TaskStatus;
|
||||
assignedWorker?: string;
|
||||
result?: unknown;
|
||||
error?: string;
|
||||
createdAt: Date;
|
||||
startedAt?: Date;
|
||||
completedAt?: Date;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface Worker {
|
||||
id: string;
|
||||
status: WorkerStatus;
|
||||
currentTask?: string;
|
||||
sessions: Map<string, AgentSession>;
|
||||
completedTasks: number;
|
||||
failedTasks: number;
|
||||
createdAt: Date;
|
||||
lastActivity: Date;
|
||||
}
|
||||
|
||||
export interface ExecutionConfig {
|
||||
maxWorkers: number;
|
||||
maxConcurrentPerWorker: number;
|
||||
taskTimeout: number;
|
||||
retryAttempts: number;
|
||||
retryDelay: number;
|
||||
drainTimeout: number;
|
||||
}
|
||||
|
||||
export interface ExecutionResult {
|
||||
taskId: string;
|
||||
success: boolean;
|
||||
output?: unknown;
|
||||
error?: string;
|
||||
duration: number;
|
||||
workerId: string;
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Parallel Executor
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* ParallelExecutionEngine - Manages concurrent agent sessions
|
||||
*/
|
||||
export class ParallelExecutionEngine extends EventEmitter {
|
||||
private config: ExecutionConfig;
|
||||
private workers: Map<string, Worker> = new Map();
|
||||
private taskQueue: PipelineTask[] = [];
|
||||
private runningTasks: Map<string, { task: PipelineTask; worker: Worker; session: AgentSession }> = new Map();
|
||||
private completedTasks: PipelineTask[] = [];
|
||||
private failedTasks: PipelineTask[] = [];
|
||||
private sessions: Map<string, AgentSession> = new Map();
|
||||
private processing = false;
|
||||
private processInterval?: ReturnType<typeof setInterval>;
|
||||
private taskHandlers: Map<string, (task: PipelineTask, session: AgentSession) => Promise<unknown>> = new Map();
|
||||
|
||||
constructor(config?: Partial<ExecutionConfig>) {
|
||||
super();
|
||||
this.config = {
|
||||
maxWorkers: config?.maxWorkers || 4,
|
||||
maxConcurrentPerWorker: config?.maxConcurrentPerWorker || 3,
|
||||
taskTimeout: config?.taskTimeout || 300000, // 5 minutes
|
||||
retryAttempts: config?.retryAttempts || 3,
|
||||
retryDelay: config?.retryDelay || 5000,
|
||||
drainTimeout: config?.drainTimeout || 60000,
|
||||
...config
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the execution engine
|
||||
*/
|
||||
start(): void {
|
||||
// Initialize workers
|
||||
for (let i = 0; i < this.config.maxWorkers; i++) {
|
||||
this.createWorker();
|
||||
}
|
||||
|
||||
// Start processing loop
|
||||
this.processing = true;
|
||||
this.processInterval = setInterval(() => this.processQueue(), 100);
|
||||
|
||||
this.emit('started', { workerCount: this.workers.size });
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the execution engine
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
this.processing = false;
|
||||
|
||||
if (this.processInterval) {
|
||||
clearInterval(this.processInterval);
|
||||
}
|
||||
|
||||
// Wait for running tasks to complete or drain
|
||||
await this.drain();
|
||||
|
||||
// Terminate workers
|
||||
for (const worker of this.workers.values()) {
|
||||
worker.status = 'terminated';
|
||||
}
|
||||
|
||||
this.emit('stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new worker
|
||||
*/
|
||||
private createWorker(): Worker {
|
||||
const worker: Worker = {
|
||||
id: `worker-${randomUUID().substring(0, 8)}`,
|
||||
status: 'idle',
|
||||
sessions: new Map(),
|
||||
completedTasks: 0,
|
||||
failedTasks: 0,
|
||||
createdAt: new Date(),
|
||||
lastActivity: new Date()
|
||||
};
|
||||
|
||||
this.workers.set(worker.id, worker);
|
||||
this.emit('workerCreated', { worker });
|
||||
|
||||
return worker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an agent session
|
||||
*/
|
||||
createSession(config: {
|
||||
projectId: string;
|
||||
role: AgentRole;
|
||||
model?: string;
|
||||
workspace: string;
|
||||
tools: string[];
|
||||
identity: AgentIdentity;
|
||||
}): AgentSession {
|
||||
const session: AgentSession = {
|
||||
id: `session-${config.projectId}-${config.role}-${randomUUID().substring(0, 8)}`,
|
||||
projectId: config.projectId,
|
||||
role: config.role,
|
||||
model: config.model || this.getDefaultModelForRole(config.role),
|
||||
workspace: config.workspace,
|
||||
tools: config.tools,
|
||||
memory: {},
|
||||
identity: config.identity,
|
||||
status: 'idle',
|
||||
createdAt: new Date(),
|
||||
lastActivity: new Date()
|
||||
};
|
||||
|
||||
this.sessions.set(session.id, session);
|
||||
this.emit('sessionCreated', { session });
|
||||
|
||||
return session;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get default model for a role (cost optimization)
|
||||
*/
|
||||
private getDefaultModelForRole(role: AgentRole): string {
|
||||
switch (role) {
|
||||
case 'programmer':
|
||||
return 'opus'; // Best for complex coding
|
||||
case 'reviewer':
|
||||
return 'sonnet'; // Cost-effective for review
|
||||
case 'tester':
|
||||
return 'sonnet'; // Good for test generation
|
||||
case 'planner':
|
||||
return 'opus'; // Complex planning
|
||||
case 'analyst':
|
||||
return 'sonnet';
|
||||
default:
|
||||
return 'sonnet';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Submit a task for execution
|
||||
*/
|
||||
submitTask(task: Omit<PipelineTask, 'id' | 'status' | 'retryCount' | 'createdAt'>): PipelineTask {
|
||||
const fullTask: PipelineTask = {
|
||||
...task,
|
||||
id: `task-${randomUUID().substring(0, 8)}`,
|
||||
status: 'pending',
|
||||
retryCount: 0,
|
||||
createdAt: new Date()
|
||||
};
|
||||
|
||||
this.taskQueue.push(fullTask);
|
||||
this.emit('taskSubmitted', { task: fullTask });
|
||||
|
||||
// Sort by priority
|
||||
this.prioritizeQueue();
|
||||
|
||||
return fullTask;
|
||||
}
|
||||
|
||||
/**
|
||||
* Submit multiple tasks for parallel execution
|
||||
*/
|
||||
submitBatch(tasks: Array<Omit<PipelineTask, 'id' | 'status' | 'retryCount' | 'createdAt'>>): PipelineTask[] {
|
||||
return tasks.map(task => this.submitTask(task));
|
||||
}
|
||||
|
||||
/**
|
||||
* Prioritize the task queue
|
||||
*/
|
||||
private prioritizeQueue(): void {
|
||||
const priorityOrder = { critical: 0, high: 1, medium: 2, low: 3 };
|
||||
|
||||
this.taskQueue.sort((a, b) => {
|
||||
// First by priority
|
||||
const priorityDiff = priorityOrder[a.priority] - priorityOrder[b.priority];
|
||||
if (priorityDiff !== 0) return priorityDiff;
|
||||
|
||||
// Then by creation time (FIFO within priority)
|
||||
return a.createdAt.getTime() - b.createdAt.getTime();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the task queue
|
||||
*/
|
||||
private async processQueue(): Promise<void> {
|
||||
if (!this.processing) return;
|
||||
|
||||
// Find tasks ready to run (dependencies met)
|
||||
const readyTasks = this.getReadyTasks();
|
||||
|
||||
for (const task of readyTasks) {
|
||||
const worker = this.findAvailableWorker(task);
|
||||
if (!worker) break; // No workers available
|
||||
|
||||
await this.executeTask(task, worker);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get tasks that are ready to execute
|
||||
*/
|
||||
private getReadyTasks(): PipelineTask[] {
|
||||
return this.taskQueue.filter(task => {
|
||||
if (task.status !== 'pending') return false;
|
||||
|
||||
// Check dependencies
|
||||
for (const depId of task.dependencies) {
|
||||
const depTask = this.getTask(depId);
|
||||
if (!depTask || depTask.status !== 'completed') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Find an available worker for a task
|
||||
*/
|
||||
private findAvailableWorker(task: PipelineTask): Worker | undefined {
|
||||
// First, try to find a worker already handling the project
|
||||
for (const worker of this.workers.values()) {
|
||||
if (worker.status !== 'idle' && worker.status !== 'busy') continue;
|
||||
|
||||
const hasProject = Array.from(worker.sessions.values())
|
||||
.some(s => s.projectId === task.projectId);
|
||||
|
||||
if (hasProject && worker.sessions.size < this.config.maxConcurrentPerWorker) {
|
||||
return worker;
|
||||
}
|
||||
}
|
||||
|
||||
// Then, find any available worker
|
||||
for (const worker of this.workers.values()) {
|
||||
if (worker.status !== 'idle' && worker.status !== 'busy') continue;
|
||||
|
||||
if (worker.sessions.size < this.config.maxConcurrentPerWorker) {
|
||||
return worker;
|
||||
}
|
||||
}
|
||||
|
||||
// Create new worker if under limit
|
||||
if (this.workers.size < this.config.maxWorkers) {
|
||||
return this.createWorker();
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a task
|
||||
*/
|
||||
private async executeTask(task: PipelineTask, worker: Worker): Promise<void> {
|
||||
// Move task from queue to running
|
||||
const taskIndex = this.taskQueue.indexOf(task);
|
||||
if (taskIndex > -1) {
|
||||
this.taskQueue.splice(taskIndex, 1);
|
||||
}
|
||||
|
||||
task.status = 'running';
|
||||
task.startedAt = new Date();
|
||||
task.assignedWorker = worker.id;
|
||||
|
||||
// Create or get session
|
||||
const session = this.getOrCreateSession(task, worker);
|
||||
|
||||
// Track running task
|
||||
this.runningTasks.set(task.id, { task, worker, session });
|
||||
|
||||
// Update worker status
|
||||
worker.status = 'busy';
|
||||
worker.currentTask = task.id;
|
||||
worker.lastActivity = new Date();
|
||||
|
||||
this.emit('taskStarted', { task, worker, session });
|
||||
|
||||
// Get task handler
|
||||
const handler = this.taskHandlers.get(task.type) || this.defaultTaskHandler;
|
||||
|
||||
try {
|
||||
// Execute with timeout
|
||||
const result = await Promise.race([
|
||||
handler(task, session),
|
||||
this.createTimeout(task)
|
||||
]);
|
||||
|
||||
task.result = result;
|
||||
task.status = 'completed';
|
||||
task.completedAt = new Date();
|
||||
|
||||
worker.completedTasks++;
|
||||
this.completedTasks.push(task);
|
||||
|
||||
this.emit('taskCompleted', { task, worker, session, result });
|
||||
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
task.error = errorMessage;
|
||||
task.retryCount++;
|
||||
|
||||
if (task.retryCount < task.maxRetries) {
|
||||
// Retry
|
||||
task.status = 'pending';
|
||||
this.taskQueue.push(task);
|
||||
this.emit('taskRetrying', { task, attempt: task.retryCount });
|
||||
} else {
|
||||
// Failed
|
||||
task.status = 'failed';
|
||||
task.completedAt = new Date();
|
||||
worker.failedTasks++;
|
||||
this.failedTasks.push(task);
|
||||
this.emit('taskFailed', { task, worker, error: errorMessage });
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
this.runningTasks.delete(task.id);
|
||||
worker.currentTask = undefined;
|
||||
worker.lastActivity = new Date();
|
||||
|
||||
// Update worker status
|
||||
if (worker.sessions.size === 0 || this.runningTasks.size === 0) {
|
||||
worker.status = 'idle';
|
||||
}
|
||||
|
||||
session.lastActivity = new Date();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create session for a task
|
||||
*/
|
||||
private getOrCreateSession(task: PipelineTask, worker: Worker): AgentSession {
|
||||
// Look for existing session for this project/role
|
||||
for (const session of worker.sessions.values()) {
|
||||
if (session.projectId === task.projectId && session.role === task.role) {
|
||||
return session;
|
||||
}
|
||||
}
|
||||
|
||||
// Create new session
|
||||
const session = this.createSession({
|
||||
projectId: task.projectId,
|
||||
role: task.role,
|
||||
workspace: `workspace/${task.projectId}/${task.role}`,
|
||||
tools: this.getToolsForRole(task.role),
|
||||
identity: this.getIdentityForRole(task.role)
|
||||
});
|
||||
|
||||
worker.sessions.set(session.id, session);
|
||||
|
||||
return session;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get tools available for a role
|
||||
*/
|
||||
private getToolsForRole(role: AgentRole): string[] {
|
||||
const toolMap: Record<AgentRole, string[]> = {
|
||||
programmer: ['read', 'write', 'execute', 'git', 'test', 'lint', 'build'],
|
||||
reviewer: ['read', 'diff', 'comment', 'lint', 'test'],
|
||||
tester: ['read', 'execute', 'test', 'mock'],
|
||||
planner: ['read', 'write', 'diagram'],
|
||||
analyst: ['read', 'query', 'report'],
|
||||
custom: ['read']
|
||||
};
|
||||
|
||||
return toolMap[role] || ['read'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get identity for a role
|
||||
*/
|
||||
private getIdentityForRole(role: AgentRole): AgentIdentity {
|
||||
const identityMap: Record<AgentRole, AgentIdentity> = {
|
||||
programmer: {
|
||||
name: 'Code Architect',
|
||||
description: 'Expert developer who writes clean, efficient code',
|
||||
personality: 'Methodical, detail-oriented, focuses on best practices'
|
||||
},
|
||||
reviewer: {
|
||||
name: 'Code Reviewer',
|
||||
description: 'Experienced engineer who catches bugs and improves code quality',
|
||||
personality: 'Thorough, constructive, focuses on maintainability'
|
||||
},
|
||||
tester: {
|
||||
name: 'QA Engineer',
|
||||
description: 'Test specialist who ensures code correctness',
|
||||
personality: 'Systematic, edge-case focused, quality-driven'
|
||||
},
|
||||
planner: {
|
||||
name: 'Technical Architect',
|
||||
description: 'Strategic thinker who plans implementation',
|
||||
personality: 'Analytical, systematic, big-picture focused'
|
||||
},
|
||||
analyst: {
|
||||
name: 'Data Analyst',
|
||||
description: 'Data specialist who extracts insights',
|
||||
personality: 'Curious, methodical, detail-oriented'
|
||||
},
|
||||
custom: {
|
||||
name: 'Custom Agent',
|
||||
description: 'Generic agent for custom tasks',
|
||||
personality: 'Adaptable'
|
||||
}
|
||||
};
|
||||
|
||||
return identityMap[role] || identityMap.custom;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default task handler
|
||||
*/
|
||||
private async defaultTaskHandler(task: PipelineTask, session: AgentSession): Promise<unknown> {
|
||||
// This would be replaced by actual LLM invocation
|
||||
return {
|
||||
message: `Task ${task.type} completed by ${session.identity.name}`,
|
||||
projectId: task.projectId,
|
||||
role: task.role
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create timeout promise
|
||||
*/
|
||||
private createTimeout(task: PipelineTask): Promise<never> {
|
||||
return new Promise((_, reject) => {
|
||||
setTimeout(() => {
|
||||
reject(new Error(`Task ${task.id} timed out after ${task.timeout}ms`));
|
||||
}, task.timeout);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get task by ID
|
||||
*/
|
||||
getTask(taskId: string): PipelineTask | undefined {
|
||||
return (
|
||||
this.taskQueue.find(t => t.id === taskId) ||
|
||||
this.runningTasks.get(taskId)?.task ||
|
||||
this.completedTasks.find(t => t.id === taskId) ||
|
||||
this.failedTasks.find(t => t.id === taskId)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a task handler
|
||||
*/
|
||||
registerHandler(taskType: string, handler: (task: PipelineTask, session: AgentSession) => Promise<unknown>): void {
|
||||
this.taskHandlers.set(taskType, handler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain - wait for running tasks to complete
|
||||
*/
|
||||
private async drain(): Promise<void> {
|
||||
while (this.runningTasks.size > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get engine statistics
|
||||
*/
|
||||
getStats(): {
|
||||
workers: { total: number; idle: number; busy: number };
|
||||
tasks: { pending: number; running: number; completed: number; failed: number };
|
||||
sessions: number;
|
||||
} {
|
||||
let idleWorkers = 0;
|
||||
let busyWorkers = 0;
|
||||
|
||||
for (const worker of this.workers.values()) {
|
||||
if (worker.status === 'idle') idleWorkers++;
|
||||
else if (worker.status === 'busy') busyWorkers++;
|
||||
}
|
||||
|
||||
return {
|
||||
workers: {
|
||||
total: this.workers.size,
|
||||
idle: idleWorkers,
|
||||
busy: busyWorkers
|
||||
},
|
||||
tasks: {
|
||||
pending: this.taskQueue.length,
|
||||
running: this.runningTasks.size,
|
||||
completed: this.completedTasks.length,
|
||||
failed: this.failedTasks.length
|
||||
},
|
||||
sessions: this.sessions.size
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get sessions by project
|
||||
*/
|
||||
getSessionsByProject(projectId: string): AgentSession[] {
|
||||
return Array.from(this.sessions.values()).filter(s => s.projectId === projectId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all sessions
|
||||
*/
|
||||
getAllSessions(): AgentSession[] {
|
||||
return Array.from(this.sessions.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminate a session
|
||||
*/
|
||||
terminateSession(sessionId: string): boolean {
|
||||
const session = this.sessions.get(sessionId);
|
||||
if (session) {
|
||||
session.status = 'terminated';
|
||||
this.emit('sessionTerminated', { session });
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Default instance
|
||||
export const defaultExecutor = new ParallelExecutionEngine();
|
||||
570
pipeline-system/events/event-bus.ts
Normal file
570
pipeline-system/events/event-bus.ts
Normal file
@@ -0,0 +1,570 @@
|
||||
/**
|
||||
* Event-Driven Coordination System
|
||||
*
|
||||
* Event bus for inter-agent communication.
|
||||
* Agents finish work → emit event → next step triggers automatically.
|
||||
*
|
||||
* Key features:
|
||||
* - Pub/sub event distribution
|
||||
* - Event correlation and routing
|
||||
* - Event replay for debugging
|
||||
* - Dead letter queue for failed handlers
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export type EventPriority = 'low' | 'normal' | 'high' | 'critical';
|
||||
|
||||
export interface PipelineEvent {
|
||||
id: string;
|
||||
type: string;
|
||||
source: string;
|
||||
target?: string;
|
||||
payload: unknown;
|
||||
priority: EventPriority;
|
||||
timestamp: Date;
|
||||
correlationId?: string;
|
||||
causationId?: string; // ID of event that caused this event
|
||||
metadata?: Record<string, unknown>;
|
||||
retryCount?: number;
|
||||
}
|
||||
|
||||
export interface EventHandler {
|
||||
id: string;
|
||||
eventType: string | string[] | '*';
|
||||
filter?: EventFilter;
|
||||
handler: (event: PipelineEvent) => Promise<void> | void;
|
||||
priority?: number;
|
||||
once?: boolean;
|
||||
}
|
||||
|
||||
export interface EventFilter {
|
||||
source?: string | string[];
|
||||
target?: string | string[];
|
||||
payloadPattern?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface Subscription {
|
||||
id: string;
|
||||
eventType: string;
|
||||
handlerId: string;
|
||||
active: boolean;
|
||||
createdAt: Date;
|
||||
eventsReceived: number;
|
||||
}
|
||||
|
||||
export interface EventBusConfig {
|
||||
maxHistorySize: number;
|
||||
deadLetterQueueSize: number;
|
||||
retryAttempts: number;
|
||||
retryDelay: number;
|
||||
enableReplay: boolean;
|
||||
}
|
||||
|
||||
export interface EventBusStats {
|
||||
eventsPublished: number;
|
||||
eventsProcessed: number;
|
||||
eventsFailed: number;
|
||||
handlersRegistered: number;
|
||||
queueSize: number;
|
||||
historySize: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Event Bus
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* EventBus - Central event distribution system
|
||||
*/
|
||||
export class EventBus extends EventEmitter {
|
||||
private config: EventBusConfig;
|
||||
private handlers: Map<string, EventHandler> = new Map();
|
||||
private eventQueue: PipelineEvent[] = [];
|
||||
private history: PipelineEvent[] = [];
|
||||
private deadLetterQueue: PipelineEvent[] = [];
|
||||
private processing = false;
|
||||
private stats = {
|
||||
eventsPublished: 0,
|
||||
eventsProcessed: 0,
|
||||
eventsFailed: 0
|
||||
};
|
||||
private processInterval?: ReturnType<typeof setInterval>;
|
||||
|
||||
constructor(config?: Partial<EventBusConfig>) {
|
||||
super();
|
||||
this.config = {
|
||||
maxHistorySize: 1000,
|
||||
deadLetterQueueSize: 100,
|
||||
retryAttempts: 3,
|
||||
retryDelay: 1000,
|
||||
enableReplay: true,
|
||||
...config
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the event bus
|
||||
*/
|
||||
start(): void {
|
||||
this.processing = true;
|
||||
this.processInterval = setInterval(() => this.processQueue(), 50);
|
||||
this.emit('started');
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the event bus
|
||||
*/
|
||||
stop(): void {
|
||||
this.processing = false;
|
||||
if (this.processInterval) {
|
||||
clearInterval(this.processInterval);
|
||||
}
|
||||
this.emit('stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish an event
|
||||
*/
|
||||
publish(event: Omit<PipelineEvent, 'id' | 'timestamp'>): string {
|
||||
const fullEvent: PipelineEvent = {
|
||||
...event,
|
||||
id: `evt-${randomUUID().substring(0, 8)}`,
|
||||
timestamp: new Date(),
|
||||
retryCount: event.retryCount || 0
|
||||
};
|
||||
|
||||
// Add to queue
|
||||
this.eventQueue.push(fullEvent);
|
||||
this.stats.eventsPublished++;
|
||||
|
||||
// Add to history
|
||||
if (this.config.enableReplay) {
|
||||
this.history.push(fullEvent);
|
||||
if (this.history.length > this.config.maxHistorySize) {
|
||||
this.history.shift();
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('eventPublished', { event: fullEvent });
|
||||
|
||||
return fullEvent.id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish a batch of events
|
||||
*/
|
||||
publishBatch(events: Array<Omit<PipelineEvent, 'id' | 'timestamp'>>): string[] {
|
||||
return events.map(event => this.publish(event));
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to events
|
||||
*/
|
||||
subscribe(config: {
|
||||
eventType: string | string[] | '*';
|
||||
handler: (event: PipelineEvent) => Promise<void> | void;
|
||||
filter?: EventFilter;
|
||||
priority?: number;
|
||||
once?: boolean;
|
||||
}): string {
|
||||
const handlerId = `handler-${randomUUID().substring(0, 8)}`;
|
||||
|
||||
const handler: EventHandler = {
|
||||
id: handlerId,
|
||||
eventType: config.eventType,
|
||||
filter: config.filter,
|
||||
handler: config.handler,
|
||||
priority: config.priority || 0,
|
||||
once: config.once || false
|
||||
};
|
||||
|
||||
this.handlers.set(handlerId, handler);
|
||||
this.emit('handlerRegistered', { handler });
|
||||
|
||||
return handlerId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unsubscribe from events
|
||||
*/
|
||||
unsubscribe(handlerId: string): boolean {
|
||||
const result = this.handlers.delete(handlerId);
|
||||
if (result) {
|
||||
this.emit('handlerUnregistered', { handlerId });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the event queue
|
||||
*/
|
||||
private async processQueue(): Promise<void> {
|
||||
if (!this.processing || this.eventQueue.length === 0) return;
|
||||
|
||||
const event = this.eventQueue.shift()!;
|
||||
|
||||
// Find matching handlers
|
||||
const matchingHandlers = this.findMatchingHandlers(event);
|
||||
|
||||
// Sort by priority (higher first)
|
||||
matchingHandlers.sort((a, b) => (b.priority || 0) - (a.priority || 0));
|
||||
|
||||
// Execute handlers
|
||||
for (const handler of matchingHandlers) {
|
||||
try {
|
||||
await handler.handler(event);
|
||||
this.stats.eventsProcessed++;
|
||||
|
||||
// Remove one-time handlers
|
||||
if (handler.once) {
|
||||
this.handlers.delete(handler.id);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
this.stats.eventsFailed++;
|
||||
|
||||
// Retry logic
|
||||
const retryCount = (event.retryCount || 0) + 1;
|
||||
|
||||
if (retryCount < this.config.retryAttempts) {
|
||||
// Re-queue with incremented retry count
|
||||
setTimeout(() => {
|
||||
this.publish({
|
||||
...event,
|
||||
retryCount
|
||||
});
|
||||
}, this.config.retryDelay * retryCount);
|
||||
|
||||
this.emit('eventRetry', { event, error, retryCount });
|
||||
} else {
|
||||
// Move to dead letter queue
|
||||
this.addToDeadLetterQueue(event, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('eventProcessed', { event, handlerCount: matchingHandlers.length });
|
||||
}
|
||||
|
||||
/**
|
||||
* Find handlers matching an event
|
||||
*/
|
||||
private findMatchingHandlers(event: PipelineEvent): EventHandler[] {
|
||||
const matching: EventHandler[] = [];
|
||||
|
||||
for (const handler of this.handlers.values()) {
|
||||
// Check event type match
|
||||
if (handler.eventType !== '*') {
|
||||
const types = Array.isArray(handler.eventType) ? handler.eventType : [handler.eventType];
|
||||
if (!types.includes(event.type)) continue;
|
||||
}
|
||||
|
||||
// Check filters
|
||||
if (handler.filter && !this.matchesFilter(event, handler.filter)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
matching.push(handler);
|
||||
}
|
||||
|
||||
return matching;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if event matches filter
|
||||
*/
|
||||
private matchesFilter(event: PipelineEvent, filter: EventFilter): boolean {
|
||||
// Check source filter
|
||||
if (filter.source) {
|
||||
const sources = Array.isArray(filter.source) ? filter.source : [filter.source];
|
||||
if (!sources.includes(event.source)) return false;
|
||||
}
|
||||
|
||||
// Check target filter
|
||||
if (filter.target) {
|
||||
const targets = Array.isArray(filter.target) ? filter.target : [filter.target];
|
||||
if (event.target && !targets.includes(event.target)) return false;
|
||||
}
|
||||
|
||||
// Check payload pattern
|
||||
if (filter.payloadPattern) {
|
||||
const payload = event.payload as Record<string, unknown>;
|
||||
for (const [key, value] of Object.entries(filter.payloadPattern)) {
|
||||
if (payload[key] !== value) return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add event to dead letter queue
|
||||
*/
|
||||
private addToDeadLetterQueue(event: PipelineEvent, error: unknown): void {
|
||||
this.deadLetterQueue.push({
|
||||
...event,
|
||||
metadata: {
|
||||
...event.metadata,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
failedAt: new Date().toISOString()
|
||||
}
|
||||
});
|
||||
|
||||
// Trim queue
|
||||
if (this.deadLetterQueue.length > this.config.deadLetterQueueSize) {
|
||||
this.deadLetterQueue.shift();
|
||||
}
|
||||
|
||||
this.emit('eventDeadLettered', { event, error });
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay events from history
|
||||
*/
|
||||
replay(fromTimestamp?: Date, toTimestamp?: Date): void {
|
||||
if (!this.config.enableReplay) {
|
||||
throw new Error('Event replay is disabled');
|
||||
}
|
||||
|
||||
const events = this.history.filter(event => {
|
||||
if (fromTimestamp && event.timestamp < fromTimestamp) return false;
|
||||
if (toTimestamp && event.timestamp > toTimestamp) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
for (const event of events) {
|
||||
this.eventQueue.push({
|
||||
...event,
|
||||
id: `replay-${event.id}`,
|
||||
metadata: { ...event.metadata, replayed: true }
|
||||
});
|
||||
}
|
||||
|
||||
this.emit('replayStarted', { count: events.length });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get events from history
|
||||
*/
|
||||
getHistory(filter?: {
|
||||
type?: string;
|
||||
source?: string;
|
||||
from?: Date;
|
||||
to?: Date;
|
||||
}): PipelineEvent[] {
|
||||
let events = [...this.history];
|
||||
|
||||
if (filter) {
|
||||
if (filter.type) {
|
||||
events = events.filter(e => e.type === filter.type);
|
||||
}
|
||||
if (filter.source) {
|
||||
events = events.filter(e => e.source === filter.source);
|
||||
}
|
||||
if (filter.from) {
|
||||
events = events.filter(e => e.timestamp >= filter.from!);
|
||||
}
|
||||
if (filter.to) {
|
||||
events = events.filter(e => e.timestamp <= filter.to!);
|
||||
}
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get dead letter queue
|
||||
*/
|
||||
getDeadLetterQueue(): PipelineEvent[] {
|
||||
return [...this.deadLetterQueue];
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear dead letter queue
|
||||
*/
|
||||
clearDeadLetterQueue(): void {
|
||||
this.deadLetterQueue = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics
|
||||
*/
|
||||
getStats(): EventBusStats {
|
||||
return {
|
||||
eventsPublished: this.stats.eventsPublished,
|
||||
eventsProcessed: this.stats.eventsProcessed,
|
||||
eventsFailed: this.stats.eventsFailed,
|
||||
handlersRegistered: this.handlers.size,
|
||||
queueSize: this.eventQueue.length,
|
||||
historySize: this.history.length
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Request-response pattern
|
||||
*/
|
||||
async request<T = unknown>(
|
||||
event: Omit<PipelineEvent, 'id' | 'timestamp'>,
|
||||
timeout = 30000
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const correlationId = `req-${randomUUID().substring(0, 8)}`;
|
||||
|
||||
// Subscribe to response
|
||||
const responseHandler = this.subscribe({
|
||||
eventType: `${event.type}.response`,
|
||||
filter: { payloadPattern: { correlationId } },
|
||||
once: true,
|
||||
handler: (response) => {
|
||||
clearTimeout(timeoutId);
|
||||
resolve(response.payload as T);
|
||||
}
|
||||
});
|
||||
|
||||
// Set timeout
|
||||
const timeoutId = setTimeout(() => {
|
||||
this.unsubscribe(responseHandler);
|
||||
reject(new Error(`Request timeout for event ${event.type}`));
|
||||
}, timeout);
|
||||
|
||||
// Publish request with correlation ID
|
||||
this.publish({
|
||||
...event,
|
||||
metadata: { ...event.metadata, correlationId }
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a correlated event chain
|
||||
*/
|
||||
createChain(firstEvent: Omit<PipelineEvent, 'id' | 'timestamp' | 'correlationId'>): EventChain {
|
||||
const correlationId = `chain-${randomUUID().substring(0, 8)}`;
|
||||
|
||||
return new EventChain(this, correlationId, firstEvent);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Event Chain
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* EventChain - Builder for correlated event sequences
|
||||
*/
|
||||
export class EventChain {
|
||||
private bus: EventBus;
|
||||
private correlationId: string;
|
||||
private events: PipelineEvent[] = [];
|
||||
private currentEvent?: PipelineEvent;
|
||||
|
||||
constructor(bus: EventBus, correlationId: string, firstEvent: Omit<PipelineEvent, 'id' | 'timestamp' | 'correlationId'>) {
|
||||
this.bus = bus;
|
||||
this.correlationId = correlationId;
|
||||
this.currentEvent = {
|
||||
...firstEvent,
|
||||
id: '',
|
||||
timestamp: new Date(),
|
||||
correlationId
|
||||
} as PipelineEvent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add next event in chain
|
||||
*/
|
||||
then(event: Omit<PipelineEvent, 'id' | 'timestamp' | 'correlationId' | 'causationId'>): this {
|
||||
if (this.currentEvent) {
|
||||
this.events.push(this.currentEvent);
|
||||
|
||||
this.currentEvent = {
|
||||
...event,
|
||||
id: '',
|
||||
timestamp: new Date(),
|
||||
correlationId: this.correlationId,
|
||||
causationId: this.currentEvent.id || undefined
|
||||
} as PipelineEvent;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the chain
|
||||
*/
|
||||
execute(): string[] {
|
||||
if (this.currentEvent) {
|
||||
this.events.push(this.currentEvent);
|
||||
}
|
||||
|
||||
return this.events.map(event =>
|
||||
this.bus.publish({
|
||||
...event,
|
||||
correlationId: this.correlationId
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get correlation ID
|
||||
*/
|
||||
getCorrelationId(): string {
|
||||
return this.correlationId;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Predefined Pipeline Events
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Standard pipeline event types
|
||||
*/
|
||||
export const PipelineEventTypes = {
|
||||
// Agent lifecycle
|
||||
AGENT_STARTED: 'agent.started',
|
||||
AGENT_COMPLETED: 'agent.completed',
|
||||
AGENT_FAILED: 'agent.failed',
|
||||
AGENT_TIMEOUT: 'agent.timeout',
|
||||
|
||||
// Task lifecycle
|
||||
TASK_CREATED: 'task.created',
|
||||
TASK_ASSIGNED: 'task.assigned',
|
||||
TASK_STARTED: 'task.started',
|
||||
TASK_COMPLETED: 'task.completed',
|
||||
TASK_FAILED: 'task.failed',
|
||||
|
||||
// Code pipeline
|
||||
CODE_WRITTEN: 'code.written',
|
||||
CODE_REVIEWED: 'code.reviewed',
|
||||
CODE_APPROVED: 'code.approved',
|
||||
CODE_REJECTED: 'code.rejected',
|
||||
CODE_TESTED: 'code.tested',
|
||||
TESTS_PASSED: 'tests.passed',
|
||||
TESTS_FAILED: 'tests.failed',
|
||||
|
||||
// State machine
|
||||
STATE_ENTERED: 'state.entered',
|
||||
STATE_EXITED: 'state.exited',
|
||||
TRANSITION: 'state.transition',
|
||||
|
||||
// Coordination
|
||||
PIPELINE_STARTED: 'pipeline.started',
|
||||
PIPELINE_COMPLETED: 'pipeline.completed',
|
||||
PIPELINE_PAUSED: 'pipeline.paused',
|
||||
PIPELINE_RESUMED: 'pipeline.resumed',
|
||||
|
||||
// Human interaction
|
||||
HUMAN_INPUT_REQUIRED: 'human.input_required',
|
||||
HUMAN_INPUT_RECEIVED: 'human.input_received',
|
||||
HUMAN_APPROVAL_REQUIRED: 'human.approval_required',
|
||||
HUMAN_APPROVED: 'human.approved',
|
||||
HUMAN_REJECTED: 'human.rejected'
|
||||
} as const;
|
||||
|
||||
// Default event bus instance
|
||||
export const defaultEventBus = new EventBus();
|
||||
206
pipeline-system/index.ts
Normal file
206
pipeline-system/index.ts
Normal file
@@ -0,0 +1,206 @@
|
||||
/**
|
||||
* Deterministic Multi-Agent Pipeline System
|
||||
*
|
||||
* A comprehensive system for building deterministic, parallel, event-driven
|
||||
* multi-agent pipelines that integrate with Claude Code and OpenClaw.
|
||||
*
|
||||
* Key Features:
|
||||
* - Deterministic orchestration (state machine, not LLM decision)
|
||||
* - Parallel execution (up to 12 concurrent agent sessions)
|
||||
* - Event-driven coordination (agents finish → next triggers)
|
||||
* - Full agent capabilities (tools, memory, identity, workspace)
|
||||
*
|
||||
* @module pipeline-system
|
||||
*/
|
||||
|
||||
// Core
|
||||
export {
|
||||
DeterministicStateMachine,
|
||||
StateMachineRegistry,
|
||||
stateMachineRegistry
|
||||
} from './core/state-machine';
|
||||
export type {
|
||||
State,
|
||||
StateStatus,
|
||||
Transition,
|
||||
Condition,
|
||||
RetryConfig,
|
||||
StateMachineDefinition,
|
||||
StateMachineInstance,
|
||||
StateTransition,
|
||||
Event,
|
||||
ErrorHandling
|
||||
} from './core/state-machine';
|
||||
|
||||
// Engine
|
||||
export {
|
||||
ParallelExecutionEngine,
|
||||
defaultExecutor
|
||||
} from './engine/parallel-executor';
|
||||
export type {
|
||||
AgentRole,
|
||||
TaskStatus,
|
||||
WorkerStatus,
|
||||
AgentSession,
|
||||
AgentIdentity,
|
||||
PipelineTask,
|
||||
Worker,
|
||||
ExecutionConfig,
|
||||
ExecutionResult
|
||||
} from './engine/parallel-executor';
|
||||
|
||||
// Events
|
||||
export {
|
||||
EventBus,
|
||||
EventChain,
|
||||
PipelineEventTypes,
|
||||
defaultEventBus
|
||||
} from './events/event-bus';
|
||||
export type {
|
||||
PipelineEvent,
|
||||
EventHandler,
|
||||
EventFilter,
|
||||
Subscription,
|
||||
EventBusConfig,
|
||||
EventBusStats,
|
||||
EventPriority
|
||||
} from './events/event-bus';
|
||||
|
||||
// Workspace
|
||||
export {
|
||||
WorkspaceManager,
|
||||
WorkspaceFactory,
|
||||
defaultWorkspaceFactory
|
||||
} from './workspace/agent-workspace';
|
||||
export type {
|
||||
Permission,
|
||||
WorkspaceConfig,
|
||||
ResourceLimits,
|
||||
MountPoint,
|
||||
AgentTool,
|
||||
ToolContext,
|
||||
ToolResult,
|
||||
MemoryStore
|
||||
} from './workspace/agent-workspace';
|
||||
|
||||
// Workflows
|
||||
export {
|
||||
WorkflowParser,
|
||||
WorkflowRegistry,
|
||||
CODE_PIPELINE_WORKFLOW,
|
||||
PARALLEL_PROJECTS_WORKFLOW,
|
||||
HUMAN_APPROVAL_WORKFLOW,
|
||||
defaultWorkflowRegistry
|
||||
} from './workflows/yaml-workflow';
|
||||
export type {
|
||||
YAMLWorkflow,
|
||||
YAMLState,
|
||||
YAMLTransition,
|
||||
YAMLCondition,
|
||||
YAMLRetryConfig,
|
||||
YAMLLoopConfig
|
||||
} from './workflows/yaml-workflow';
|
||||
|
||||
// Integrations
|
||||
export {
|
||||
PipelineOrchestrator,
|
||||
createCodePipeline,
|
||||
createParallelPipeline,
|
||||
runWorkflow,
|
||||
defaultOrchestrator
|
||||
} from './integrations/claude-code';
|
||||
export type {
|
||||
PipelineConfig,
|
||||
ProjectConfig,
|
||||
TaskConfig,
|
||||
PipelineResult,
|
||||
ProjectResult,
|
||||
TaskResult,
|
||||
AgentMessage
|
||||
} from './integrations/claude-code';
|
||||
|
||||
// Version
|
||||
export const VERSION = '1.0.0';
|
||||
|
||||
/**
|
||||
* Quick Start Example:
|
||||
*
|
||||
* ```typescript
|
||||
* import {
|
||||
* PipelineOrchestrator,
|
||||
* createCodePipeline,
|
||||
* runWorkflow
|
||||
* } from './pipeline-system';
|
||||
*
|
||||
* // Option 1: Simple code pipeline
|
||||
* const pipelineId = await createCodePipeline([
|
||||
* {
|
||||
* id: 'project-1',
|
||||
* name: 'My Project',
|
||||
* tasks: [
|
||||
* { type: 'implement', description: 'Create auth module', role: 'programmer' },
|
||||
* { type: 'review', description: 'Review auth module', role: 'reviewer' },
|
||||
* { type: 'test', description: 'Test auth module', role: 'tester' }
|
||||
* ]
|
||||
* }
|
||||
* ]);
|
||||
*
|
||||
* // Option 2: Run predefined workflow
|
||||
* const workflowId = await runWorkflow('code-pipeline', {
|
||||
* projectId: 'my-project',
|
||||
* requirements: 'Build REST API'
|
||||
* });
|
||||
*
|
||||
* // Option 3: Custom configuration
|
||||
* const orchestrator = new PipelineOrchestrator();
|
||||
* await orchestrator.initialize();
|
||||
*
|
||||
* const customPipelineId = await orchestrator.createPipeline({
|
||||
* name: 'Custom Pipeline',
|
||||
* projects: [...],
|
||||
* roles: ['programmer', 'reviewer', 'tester'],
|
||||
* maxConcurrency: 12
|
||||
* });
|
||||
*
|
||||
* // Subscribe to events
|
||||
* orchestrator.onEvent('agent.completed', (event) => {
|
||||
* console.log('Agent completed:', event.payload);
|
||||
* });
|
||||
* ```
|
||||
*
|
||||
* ## Architecture
|
||||
*
|
||||
* ```
|
||||
* ┌─────────────────────────────────────────────────────────────────┐
|
||||
* │ Pipeline Orchestrator │
|
||||
* │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||
* │ │ State Machine│ │Event Bus │ │ Parallel Exec│ │
|
||||
* │ │ (Deterministic│ │(Coordination)│ │ (Concurrency)│ │
|
||||
* │ └──────────────┘ └──────────────┘ └──────────────┘ │
|
||||
* │ │ │ │ │
|
||||
* │ ┌──────┴────────────────┴─────────────────┴──────┐ │
|
||||
* │ │ Agent Workspaces │ │
|
||||
* │ │ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ │
|
||||
* │ │ │Programmer│ │Reviewer │ │ Tester │ │ │
|
||||
* │ │ │Workspace │ │Workspace│ │Workspace│ │ │
|
||||
* │ │ │ • Tools │ │ • Tools │ │ • Tools │ │ │
|
||||
* │ │ │ • Memory │ │ • Memory│ │ • Memory│ │ │
|
||||
* │ │ │ • Files │ │ • Files │ │ • Files │ │ │
|
||||
* │ │ └─────────┘ └─────────┘ └─────────┘ │ │
|
||||
* │ └────────────────────────────────────────────────┘ │
|
||||
* └─────────────────────────────────────────────────────────────────┘
|
||||
* │
|
||||
* ▼
|
||||
* ┌─────────────────────────────────────────────────────────────────┐
|
||||
* │ LLM Provider (ZAI SDK) │
|
||||
* └─────────────────────────────────────────────────────────────────┘
|
||||
* ```
|
||||
*
|
||||
* ## Key Principles
|
||||
*
|
||||
* 1. **Deterministic Flow**: State machines control the pipeline, not LLM decisions
|
||||
* 2. **Event-Driven**: Agents communicate through events, enabling loose coupling
|
||||
* 3. **Parallel Execution**: Multiple agents work concurrently with resource isolation
|
||||
* 4. **Workspace Isolation**: Each agent has its own tools, memory, and file space
|
||||
* 5. **YAML Workflows**: Define pipelines declaratively, compatible with Lobster
|
||||
*/
|
||||
599
pipeline-system/integrations/claude-code.ts
Normal file
599
pipeline-system/integrations/claude-code.ts
Normal file
@@ -0,0 +1,599 @@
|
||||
/**
|
||||
* Claude Code Integration Layer
|
||||
*
|
||||
* Provides easy integration with Claude Code and OpenClaw.
|
||||
* Single API surface for all pipeline operations.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'crypto';
|
||||
import ZAI from 'z-ai-web-dev-sdk';
|
||||
import {
|
||||
DeterministicStateMachine,
|
||||
StateMachineDefinition,
|
||||
StateMachineRegistry,
|
||||
stateMachineRegistry
|
||||
} from '../core/state-machine';
|
||||
import {
|
||||
ParallelExecutionEngine,
|
||||
PipelineTask,
|
||||
AgentRole,
|
||||
AgentSession,
|
||||
defaultExecutor
|
||||
} from '../engine/parallel-executor';
|
||||
import {
|
||||
EventBus,
|
||||
PipelineEvent,
|
||||
PipelineEventTypes,
|
||||
defaultEventBus
|
||||
} from '../events/event-bus';
|
||||
import {
|
||||
WorkspaceManager,
|
||||
WorkspaceFactory,
|
||||
AgentIdentity,
|
||||
defaultWorkspaceFactory
|
||||
} from '../workspace/agent-workspace';
|
||||
import {
|
||||
WorkflowRegistry,
|
||||
YAMLWorkflow,
|
||||
defaultWorkflowRegistry
|
||||
} from '../workflows/yaml-workflow';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export interface PipelineConfig {
|
||||
name: string;
|
||||
projects: ProjectConfig[];
|
||||
roles: AgentRole[];
|
||||
maxConcurrency?: number;
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
export interface ProjectConfig {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
repository?: string;
|
||||
branch?: string;
|
||||
tasks: TaskConfig[];
|
||||
}
|
||||
|
||||
export interface TaskConfig {
|
||||
type: string;
|
||||
description: string;
|
||||
role: AgentRole;
|
||||
priority?: 'low' | 'medium' | 'high' | 'critical';
|
||||
dependencies?: string[];
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
export interface PipelineResult {
|
||||
pipelineId: string;
|
||||
status: 'running' | 'completed' | 'failed' | 'cancelled';
|
||||
startTime: Date;
|
||||
endTime?: Date;
|
||||
projects: ProjectResult[];
|
||||
}
|
||||
|
||||
export interface ProjectResult {
|
||||
projectId: string;
|
||||
status: 'pending' | 'running' | 'completed' | 'failed';
|
||||
tasks: TaskResult[];
|
||||
}
|
||||
|
||||
export interface TaskResult {
|
||||
taskId: string;
|
||||
status: 'pending' | 'running' | 'completed' | 'failed';
|
||||
output?: unknown;
|
||||
error?: string;
|
||||
duration?: number;
|
||||
}
|
||||
|
||||
export interface AgentMessage {
|
||||
role: 'system' | 'user' | 'assistant';
|
||||
content: string;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Pipeline Orchestrator
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* PipelineOrchestrator - Main integration class
|
||||
*
|
||||
* Single entry point for Claude Code and OpenClaw integration.
|
||||
*/
|
||||
export class PipelineOrchestrator {
|
||||
private zai: Awaited<ReturnType<typeof ZAI.create>> | null = null;
|
||||
private executor: ParallelExecutionEngine;
|
||||
private eventBus: EventBus;
|
||||
private workflowRegistry: WorkflowRegistry;
|
||||
private workspaceFactory: WorkspaceFactory;
|
||||
private smRegistry: StateMachineRegistry;
|
||||
private pipelines: Map<string, PipelineResult> = new Map();
|
||||
private initialized = false;
|
||||
|
||||
constructor(config?: {
|
||||
executor?: ParallelExecutionEngine;
|
||||
eventBus?: EventBus;
|
||||
workflowRegistry?: WorkflowRegistry;
|
||||
workspaceFactory?: WorkspaceFactory;
|
||||
}) {
|
||||
this.executor = config?.executor || defaultExecutor;
|
||||
this.eventBus = config?.eventBus || defaultEventBus;
|
||||
this.workflowRegistry = config?.workflowRegistry || defaultWorkflowRegistry;
|
||||
this.workspaceFactory = config?.workspaceFactory || defaultWorkspaceFactory;
|
||||
this.smRegistry = stateMachineRegistry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the pipeline system
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
|
||||
// Initialize ZAI SDK
|
||||
this.zai = await ZAI.create();
|
||||
|
||||
// Start executor
|
||||
this.executor.start();
|
||||
|
||||
// Start event bus
|
||||
this.eventBus.start();
|
||||
|
||||
// Register task handler
|
||||
this.executor.registerHandler('agent-task', this.executeAgentTask.bind(this));
|
||||
|
||||
// Set up event subscriptions
|
||||
this.setupEventSubscriptions();
|
||||
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up event subscriptions for coordination
|
||||
*/
|
||||
private setupEventSubscriptions(): void {
|
||||
// Agent completion triggers next step
|
||||
this.eventBus.subscribe({
|
||||
eventType: PipelineEventTypes.AGENT_COMPLETED,
|
||||
handler: async (event) => {
|
||||
const { projectId, role, output } = event.payload as Record<string, unknown>;
|
||||
|
||||
// Determine next role in pipeline
|
||||
const nextRole = this.getNextRole(role as AgentRole);
|
||||
|
||||
if (nextRole) {
|
||||
// Emit event to trigger next agent
|
||||
this.eventBus.publish({
|
||||
type: PipelineEventTypes.TASK_STARTED,
|
||||
source: 'orchestrator',
|
||||
payload: { projectId, role: nextRole, previousOutput: output }
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Handle failures
|
||||
this.eventBus.subscribe({
|
||||
eventType: PipelineEventTypes.AGENT_FAILED,
|
||||
handler: async (event) => {
|
||||
const { projectId, error } = event.payload as Record<string, unknown>;
|
||||
console.error(`Agent failed for project ${projectId}:`, error);
|
||||
|
||||
// Emit pipeline failure event
|
||||
this.eventBus.publish({
|
||||
type: PipelineEventTypes.PIPELINE_COMPLETED,
|
||||
source: 'orchestrator',
|
||||
payload: { projectId, status: 'failed', error }
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get next role in the pipeline sequence
|
||||
*/
|
||||
private getNextRole(currentRole: AgentRole): AgentRole | null {
|
||||
const sequence: AgentRole[] = ['programmer', 'reviewer', 'tester'];
|
||||
const currentIndex = sequence.indexOf(currentRole);
|
||||
|
||||
if (currentIndex < sequence.length - 1) {
|
||||
return sequence[currentIndex + 1];
|
||||
}
|
||||
|
||||
return null; // End of pipeline
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute an agent task
|
||||
*/
|
||||
private async executeAgentTask(
|
||||
task: PipelineTask,
|
||||
session: AgentSession
|
||||
): Promise<unknown> {
|
||||
if (!this.zai) {
|
||||
throw new Error('Pipeline not initialized');
|
||||
}
|
||||
|
||||
// Create workspace for this task
|
||||
const workspace = this.workspaceFactory.createWorkspace({
|
||||
projectId: session.projectId,
|
||||
agentId: session.id,
|
||||
role: session.role,
|
||||
permissions: this.getPermissionsForRole(session.role)
|
||||
});
|
||||
|
||||
// Set agent identity
|
||||
workspace.setIdentity(session.identity);
|
||||
|
||||
// Build messages for LLM
|
||||
const messages = this.buildMessages(task, session, workspace);
|
||||
|
||||
try {
|
||||
// Call LLM
|
||||
const response = await this.zai.chat.completions.create({
|
||||
messages,
|
||||
thinking: { type: 'disabled' }
|
||||
});
|
||||
|
||||
const output = response.choices?.[0]?.message?.content || '';
|
||||
|
||||
// Save output to workspace
|
||||
workspace.writeFile(`output/${task.id}.txt`, output);
|
||||
|
||||
// Store in memory for next agent
|
||||
workspace.memorize(`task.${task.id}.output`, output);
|
||||
|
||||
// Emit completion event
|
||||
this.eventBus.publish({
|
||||
type: PipelineEventTypes.AGENT_COMPLETED,
|
||||
source: session.id,
|
||||
payload: {
|
||||
taskId: task.id,
|
||||
projectId: session.projectId,
|
||||
role: session.role,
|
||||
output
|
||||
}
|
||||
});
|
||||
|
||||
return { output, workspace: workspace.getPath() };
|
||||
|
||||
} catch (error) {
|
||||
// Emit failure event
|
||||
this.eventBus.publish({
|
||||
type: PipelineEventTypes.AGENT_FAILED,
|
||||
source: session.id,
|
||||
payload: {
|
||||
taskId: task.id,
|
||||
projectId: session.projectId,
|
||||
role: session.role,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build messages for LLM
|
||||
*/
|
||||
private buildMessages(
|
||||
task: PipelineTask,
|
||||
session: AgentSession,
|
||||
workspace: WorkspaceManager
|
||||
): AgentMessage[] {
|
||||
const messages: AgentMessage[] = [];
|
||||
|
||||
// System prompt with identity
|
||||
messages.push({
|
||||
role: 'system',
|
||||
content: this.buildSystemPrompt(session, workspace)
|
||||
});
|
||||
|
||||
// Task description
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: `## Task\n${task.description}\n\n## Context\nProject: ${session.projectId}\nRole: ${session.role}\n\n## Instructions\nComplete this task and provide your output.`
|
||||
});
|
||||
|
||||
// Add any previous context from memory
|
||||
const previousOutput = workspace.recall('previous.output');
|
||||
if (previousOutput) {
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: `## Previous Work\n${JSON.stringify(previousOutput, null, 2)}`
|
||||
});
|
||||
}
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build system prompt for agent
|
||||
*/
|
||||
private buildSystemPrompt(session: AgentSession, workspace: WorkspaceManager): string {
|
||||
const identity = session.identity;
|
||||
const role = session.role;
|
||||
|
||||
const roleInstructions: Record<AgentRole, string> = {
|
||||
programmer: `You are responsible for writing clean, efficient, and well-documented code.
|
||||
- Follow best practices and coding standards
|
||||
- Write tests for your code
|
||||
- Ensure code is production-ready`,
|
||||
reviewer: `You are responsible for reviewing code for quality, bugs, and improvements.
|
||||
- Check for security vulnerabilities
|
||||
- Verify coding standards
|
||||
- Suggest improvements
|
||||
- Approve or request changes`,
|
||||
tester: `You are responsible for testing the code thoroughly.
|
||||
- Write comprehensive test cases
|
||||
- Test edge cases and error handling
|
||||
- Verify functionality meets requirements
|
||||
- Report test results clearly`,
|
||||
planner: `You are responsible for planning and architecture.
|
||||
- Break down complex tasks
|
||||
- Design system architecture
|
||||
- Identify dependencies
|
||||
- Create implementation plans`,
|
||||
analyst: `You are responsible for analysis and reporting.
|
||||
- Analyze data and metrics
|
||||
- Identify patterns and insights
|
||||
- Create reports and recommendations`,
|
||||
custom: `You are a custom agent with specific instructions.`
|
||||
};
|
||||
|
||||
return `# Agent Identity
|
||||
|
||||
Name: ${identity.name}
|
||||
Role: ${role}
|
||||
Description: ${identity.description}
|
||||
|
||||
# Personality
|
||||
${identity.personality || 'Professional and efficient.'}
|
||||
|
||||
# Role Instructions
|
||||
${roleInstructions[role] || roleInstructions.custom}
|
||||
|
||||
# Workspace
|
||||
Your workspace is at: ${workspace.getPath()}
|
||||
|
||||
# Available Tools
|
||||
${session.tools.map(t => `- ${t}`).join('\n')}
|
||||
|
||||
# Constraints
|
||||
- Stay within your role boundaries
|
||||
- Communicate clearly and concisely
|
||||
- Report progress and issues promptly`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get permissions for a role
|
||||
*/
|
||||
private getPermissionsForRole(role: AgentRole): string[] {
|
||||
const permissionMap: Record<AgentRole, string[]> = {
|
||||
programmer: ['read', 'write', 'execute', 'git'],
|
||||
reviewer: ['read', 'diff'],
|
||||
tester: ['read', 'execute', 'test'],
|
||||
planner: ['read', 'write'],
|
||||
analyst: ['read'],
|
||||
custom: ['read']
|
||||
};
|
||||
return permissionMap[role] || ['read'];
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Public API
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Create and start a pipeline
|
||||
*/
|
||||
async createPipeline(config: PipelineConfig): Promise<string> {
|
||||
await this.initialize();
|
||||
|
||||
const pipelineId = `pipeline-${randomUUID().substring(0, 8)}`;
|
||||
const result: PipelineResult = {
|
||||
pipelineId,
|
||||
status: 'running',
|
||||
startTime: new Date(),
|
||||
projects: config.projects.map(p => ({
|
||||
projectId: p.id,
|
||||
status: 'pending',
|
||||
tasks: []
|
||||
}))
|
||||
};
|
||||
|
||||
this.pipelines.set(pipelineId, result);
|
||||
|
||||
// Create tasks for all projects and roles
|
||||
const tasks: PipelineTask[] = [];
|
||||
|
||||
for (const project of config.projects) {
|
||||
for (const taskConfig of project.tasks) {
|
||||
const task = this.executor.submitTask({
|
||||
projectId: project.id,
|
||||
role: taskConfig.role,
|
||||
type: taskConfig.type || 'agent-task',
|
||||
description: taskConfig.description,
|
||||
priority: taskConfig.priority || 'medium',
|
||||
input: { project, task: taskConfig },
|
||||
dependencies: taskConfig.dependencies || [],
|
||||
timeout: taskConfig.timeout || config.timeout || 300000,
|
||||
maxRetries: 3
|
||||
});
|
||||
tasks.push(task);
|
||||
}
|
||||
}
|
||||
|
||||
// Emit pipeline started event
|
||||
this.eventBus.publish({
|
||||
type: PipelineEventTypes.PIPELINE_STARTED,
|
||||
source: 'orchestrator',
|
||||
payload: { pipelineId, config, taskCount: tasks.length }
|
||||
});
|
||||
|
||||
return pipelineId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create pipeline from YAML workflow
|
||||
*/
|
||||
async createPipelineFromYAML(workflowId: string, context?: Record<string, unknown>): Promise<string> {
|
||||
await this.initialize();
|
||||
|
||||
const workflow = this.workflowRegistry.get(workflowId);
|
||||
if (!workflow) {
|
||||
throw new Error(`Workflow ${workflowId} not found`);
|
||||
}
|
||||
|
||||
const definition = this.workflowRegistry.getParsed(workflowId)!;
|
||||
|
||||
// Create state machine instance
|
||||
const sm = this.smRegistry.createInstance(workflowId);
|
||||
|
||||
// Update context if provided
|
||||
if (context) {
|
||||
sm.updateContext(context);
|
||||
}
|
||||
|
||||
// Start the state machine
|
||||
sm.start();
|
||||
|
||||
// Listen for state transitions
|
||||
sm.on('transition', ({ from, to, event }) => {
|
||||
this.eventBus.publish({
|
||||
type: PipelineEventTypes.TRANSITION,
|
||||
source: sm.getInstance().id,
|
||||
payload: { workflowId, from, to, event }
|
||||
});
|
||||
});
|
||||
|
||||
// Listen for actions
|
||||
sm.on('action', async ({ state, context }) => {
|
||||
if (state.agent || state.metadata?.role) {
|
||||
// Submit task to executor
|
||||
this.executor.submitTask({
|
||||
projectId: context.projectId as string || 'default',
|
||||
role: state.metadata?.role as AgentRole || 'programmer',
|
||||
type: 'agent-task',
|
||||
description: `Execute ${state.name}`,
|
||||
priority: 'high',
|
||||
input: { state, context },
|
||||
dependencies: [],
|
||||
timeout: state.timeout || 300000,
|
||||
maxRetries: state.retry?.maxAttempts || 3
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return sm.getInstance().id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a custom workflow
|
||||
*/
|
||||
registerWorkflow(yaml: YAMLWorkflow): StateMachineDefinition {
|
||||
return this.workflowRegistry.register(yaml);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get pipeline status
|
||||
*/
|
||||
getPipelineStatus(pipelineId: string): PipelineResult | undefined {
|
||||
return this.pipelines.get(pipelineId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel a pipeline
|
||||
*/
|
||||
async cancelPipeline(pipelineId: string): Promise<void> {
|
||||
const pipeline = this.pipelines.get(pipelineId);
|
||||
if (pipeline) {
|
||||
pipeline.status = 'cancelled';
|
||||
pipeline.endTime = new Date();
|
||||
|
||||
this.eventBus.publish({
|
||||
type: PipelineEventTypes.PIPELINE_COMPLETED,
|
||||
source: 'orchestrator',
|
||||
payload: { pipelineId, status: 'cancelled' }
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get system statistics
|
||||
*/
|
||||
getStats(): {
|
||||
pipelines: number;
|
||||
executor: ReturnType<ParallelExecutionEngine['getStats']>;
|
||||
eventBus: ReturnType<EventBus['getStats']>;
|
||||
workspaces: ReturnType<WorkspaceFactory['getStats']>;
|
||||
} {
|
||||
return {
|
||||
pipelines: this.pipelines.size,
|
||||
executor: this.executor.getStats(),
|
||||
eventBus: this.eventBus.getStats(),
|
||||
workspaces: this.workspaceFactory.getStats()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to pipeline events
|
||||
*/
|
||||
onEvent(eventType: string, handler: (event: PipelineEvent) => void): () => void {
|
||||
return this.eventBus.subscribe({ eventType, handler });
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown the pipeline system
|
||||
*/
|
||||
async shutdown(): Promise<void> {
|
||||
await this.executor.stop();
|
||||
this.eventBus.stop();
|
||||
this.initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Quick Start Functions
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Create a simple code pipeline
|
||||
*/
|
||||
export async function createCodePipeline(projects: ProjectConfig[]): Promise<string> {
|
||||
const orchestrator = new PipelineOrchestrator();
|
||||
|
||||
return orchestrator.createPipeline({
|
||||
name: 'Code Pipeline',
|
||||
projects,
|
||||
roles: ['programmer', 'reviewer', 'tester'],
|
||||
maxConcurrency: 12, // 4 projects × 3 roles
|
||||
timeout: 300000
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a parallel execution pipeline
|
||||
*/
|
||||
export async function createParallelPipeline(config: PipelineConfig): Promise<string> {
|
||||
const orchestrator = new PipelineOrchestrator();
|
||||
return orchestrator.createPipeline(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a predefined workflow
|
||||
*/
|
||||
export async function runWorkflow(
|
||||
workflowId: string,
|
||||
context?: Record<string, unknown>
|
||||
): Promise<string> {
|
||||
const orchestrator = new PipelineOrchestrator();
|
||||
return orchestrator.createPipelineFromYAML(workflowId, context);
|
||||
}
|
||||
|
||||
// Default orchestrator instance
|
||||
export const defaultOrchestrator = new PipelineOrchestrator();
|
||||
540
pipeline-system/workflows/yaml-workflow.ts
Normal file
540
pipeline-system/workflows/yaml-workflow.ts
Normal file
@@ -0,0 +1,540 @@
|
||||
/**
|
||||
* YAML Workflow Integration (Lobster-Compatible)
|
||||
*
|
||||
* Parses YAML workflow definitions and converts them to
|
||||
* deterministic state machine definitions.
|
||||
*
|
||||
* Compatible with OpenClaw/Lobster workflow format.
|
||||
*/
|
||||
|
||||
import { StateMachineDefinition, State, Transition, RetryConfig } from '../core/state-machine';
|
||||
import { AgentRole } from '../engine/parallel-executor';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export interface YAMLWorkflow {
|
||||
id: string;
|
||||
name: string;
|
||||
version?: string;
|
||||
description?: string;
|
||||
initial: string;
|
||||
states: Record<string, YAMLState>;
|
||||
events?: string[];
|
||||
context?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface YAMLState {
|
||||
type: 'start' | 'end' | 'action' | 'parallel' | 'choice' | 'wait' | 'loop' | 'subworkflow';
|
||||
agent?: string;
|
||||
role?: AgentRole;
|
||||
action?: string;
|
||||
timeout?: number | string;
|
||||
retry?: YAMLRetryConfig;
|
||||
on?: Record<string, YAMLTransition | string>;
|
||||
branches?: Record<string, string>;
|
||||
conditions?: YAMLCondition[];
|
||||
subworkflow?: string;
|
||||
loop?: YAMLLoopConfig;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface YAMLTransition {
|
||||
target: string;
|
||||
condition?: YAMLCondition;
|
||||
guard?: string;
|
||||
}
|
||||
|
||||
export interface YAMLCondition {
|
||||
type: 'equals' | 'contains' | 'exists' | 'custom';
|
||||
field: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
export interface YAMLRetryConfig {
|
||||
maxAttempts: number;
|
||||
backoff?: 'fixed' | 'exponential' | 'linear';
|
||||
initialDelay?: number | string;
|
||||
maxDelay?: number | string;
|
||||
}
|
||||
|
||||
export interface YAMLLoopConfig {
|
||||
maxIterations: number;
|
||||
iterator?: string;
|
||||
body: string;
|
||||
exitCondition?: YAMLCondition;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Workflow Parser
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* WorkflowParser - Parses YAML workflows to state machine definitions
|
||||
*/
|
||||
export class WorkflowParser {
|
||||
/**
|
||||
* Parse a YAML workflow to a state machine definition
|
||||
*/
|
||||
parse(yaml: YAMLWorkflow): StateMachineDefinition {
|
||||
const states: Record<string, State> = {};
|
||||
|
||||
for (const [stateId, yamlState] of Object.entries(yaml.states)) {
|
||||
states[stateId] = this.parseState(stateId, yamlState);
|
||||
}
|
||||
|
||||
return {
|
||||
id: yaml.id,
|
||||
name: yaml.name,
|
||||
version: yaml.version || '1.0.0',
|
||||
description: yaml.description,
|
||||
initial: yaml.initial,
|
||||
states,
|
||||
events: yaml.events,
|
||||
context: yaml.context
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single state
|
||||
*/
|
||||
private parseState(stateId: string, yamlState: YAMLState): State {
|
||||
const state: State = {
|
||||
id: stateId,
|
||||
name: stateId,
|
||||
type: yamlState.type,
|
||||
agent: yamlState.agent,
|
||||
action: yamlState.action,
|
||||
timeout: this.parseDuration(yamlState.timeout),
|
||||
metadata: {
|
||||
...yamlState.metadata,
|
||||
role: yamlState.role
|
||||
}
|
||||
};
|
||||
|
||||
// Parse retry config
|
||||
if (yamlState.retry) {
|
||||
state.retry = {
|
||||
maxAttempts: yamlState.retry.maxAttempts,
|
||||
backoff: yamlState.retry.backoff || 'exponential',
|
||||
initialDelay: this.parseDuration(yamlState.retry.initialDelay) || 1000,
|
||||
maxDelay: this.parseDuration(yamlState.retry.maxDelay) || 60000
|
||||
};
|
||||
}
|
||||
|
||||
// Parse transitions (on)
|
||||
if (yamlState.on) {
|
||||
const transitions = this.parseTransitions(yamlState.on);
|
||||
state.onExit = transitions;
|
||||
}
|
||||
|
||||
// Parse parallel branches
|
||||
if (yamlState.branches) {
|
||||
state.type = 'parallel';
|
||||
state.onEnter = Object.entries(yamlState.branches).map(([event, target]) => ({
|
||||
event,
|
||||
target
|
||||
}));
|
||||
}
|
||||
|
||||
// Parse loop config
|
||||
if (yamlState.loop) {
|
||||
state.type = 'loop';
|
||||
state.metadata = {
|
||||
...state.metadata,
|
||||
maxIterations: yamlState.loop.maxIterations,
|
||||
iterator: yamlState.loop.iterator,
|
||||
body: yamlState.loop.body
|
||||
};
|
||||
|
||||
// Add loop transitions
|
||||
state.onExit = [
|
||||
{ event: 'continue', target: yamlState.loop.body },
|
||||
{ event: 'exit', target: yamlState.on?.['exit'] as string || 'end' }
|
||||
];
|
||||
}
|
||||
|
||||
// Parse subworkflow
|
||||
if (yamlState.subworkflow) {
|
||||
state.type = 'action';
|
||||
state.action = 'subworkflow';
|
||||
state.metadata = {
|
||||
...state.metadata,
|
||||
subworkflow: yamlState.subworkflow
|
||||
};
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse transitions from YAML format
|
||||
*/
|
||||
private parseTransitions(on: Record<string, YAMLTransition | string>): Transition[] {
|
||||
const transitions: Transition[] = [];
|
||||
|
||||
for (const [event, transition] of Object.entries(on)) {
|
||||
if (typeof transition === 'string') {
|
||||
transitions.push({ event, target: transition });
|
||||
} else {
|
||||
transitions.push({
|
||||
event,
|
||||
target: transition.target,
|
||||
condition: transition.condition ? this.parseCondition(transition.condition) : undefined,
|
||||
guard: transition.guard
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return transitions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a condition
|
||||
*/
|
||||
private parseCondition(yamlCond: YAMLCondition): Transition['condition'] {
|
||||
return {
|
||||
type: yamlCond.type,
|
||||
field: yamlCond.field,
|
||||
value: yamlCond.value
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse duration string (e.g., '30s', '5m', '1h')
|
||||
*/
|
||||
private parseDuration(duration?: number | string): number | undefined {
|
||||
if (typeof duration === 'number') return duration;
|
||||
if (!duration) return undefined;
|
||||
|
||||
const match = duration.match(/^(\d+)(ms|s|m|h)?$/);
|
||||
if (!match) return undefined;
|
||||
|
||||
const value = parseInt(match[1]);
|
||||
const unit = match[2] || 'ms';
|
||||
|
||||
switch (unit) {
|
||||
case 'ms': return value;
|
||||
case 's': return value * 1000;
|
||||
case 'm': return value * 60 * 1000;
|
||||
case 'h': return value * 60 * 60 * 1000;
|
||||
default: return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Workflow Registry
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* WorkflowRegistry - Manages workflow definitions
|
||||
*/
|
||||
export class WorkflowRegistry {
|
||||
private workflows: Map<string, YAMLWorkflow> = new Map();
|
||||
private parser: WorkflowParser;
|
||||
|
||||
constructor() {
|
||||
this.parser = new WorkflowParser();
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a workflow from YAML object
|
||||
*/
|
||||
register(yaml: YAMLWorkflow): StateMachineDefinition {
|
||||
this.workflows.set(yaml.id, yaml);
|
||||
return this.parser.parse(yaml);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a workflow by ID
|
||||
*/
|
||||
get(id: string): YAMLWorkflow | undefined {
|
||||
return this.workflows.get(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parsed state machine definition
|
||||
*/
|
||||
getParsed(id: string): StateMachineDefinition | undefined {
|
||||
const yaml = this.workflows.get(id);
|
||||
if (yaml) {
|
||||
return this.parser.parse(yaml);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all workflows
|
||||
*/
|
||||
list(): string[] {
|
||||
return Array.from(this.workflows.keys());
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Predefined Workflows
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Standard Code Pipeline Workflow
|
||||
*
|
||||
* Code → Review → Test → Done
|
||||
* With max 3 review iterations
|
||||
*/
|
||||
export const CODE_PIPELINE_WORKFLOW: YAMLWorkflow = {
|
||||
id: 'code-pipeline',
|
||||
name: 'Code Pipeline',
|
||||
version: '1.0.0',
|
||||
description: 'Code → Review → Test pipeline with deterministic flow',
|
||||
initial: 'start',
|
||||
context: {
|
||||
reviewIteration: 0,
|
||||
maxReviewIterations: 3
|
||||
},
|
||||
states: {
|
||||
start: {
|
||||
type: 'start',
|
||||
on: {
|
||||
'start': 'code'
|
||||
}
|
||||
},
|
||||
code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
timeout: '30m',
|
||||
retry: {
|
||||
maxAttempts: 2,
|
||||
backoff: 'exponential',
|
||||
initialDelay: '5s',
|
||||
maxDelay: '1m'
|
||||
},
|
||||
on: {
|
||||
'completed': 'review',
|
||||
'failed': 'failed'
|
||||
}
|
||||
},
|
||||
review: {
|
||||
type: 'choice',
|
||||
conditions: [
|
||||
{ type: 'equals', field: 'reviewApproved', value: true }
|
||||
],
|
||||
on: {
|
||||
'approved': 'test',
|
||||
'rejected': 'review_loop',
|
||||
'failed': 'failed'
|
||||
}
|
||||
},
|
||||
review_loop: {
|
||||
type: 'loop',
|
||||
loop: {
|
||||
maxIterations: 3,
|
||||
body: 'code'
|
||||
},
|
||||
on: {
|
||||
'exit': 'failed'
|
||||
}
|
||||
},
|
||||
test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
timeout: '15m',
|
||||
on: {
|
||||
'passed': 'end',
|
||||
'failed': 'test_failed'
|
||||
}
|
||||
},
|
||||
test_failed: {
|
||||
type: 'choice',
|
||||
on: {
|
||||
'retry': 'code',
|
||||
'abort': 'failed'
|
||||
}
|
||||
},
|
||||
end: {
|
||||
type: 'end'
|
||||
},
|
||||
failed: {
|
||||
type: 'end',
|
||||
metadata: { status: 'failed' }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Parallel Multi-Project Workflow
|
||||
*
|
||||
* Runs multiple projects in parallel
|
||||
*/
|
||||
export const PARALLEL_PROJECTS_WORKFLOW: YAMLWorkflow = {
|
||||
id: 'parallel-projects',
|
||||
name: 'Parallel Projects Pipeline',
|
||||
version: '1.0.0',
|
||||
description: 'Run multiple projects in parallel with synchronized completion',
|
||||
initial: 'start',
|
||||
states: {
|
||||
start: {
|
||||
type: 'start',
|
||||
on: {
|
||||
'start': 'parallel'
|
||||
}
|
||||
},
|
||||
parallel: {
|
||||
type: 'parallel',
|
||||
branches: {
|
||||
'project1': 'project1_code',
|
||||
'project2': 'project2_code',
|
||||
'project3': 'project3_code',
|
||||
'project4': 'project4_code'
|
||||
},
|
||||
on: {
|
||||
'all_completed': 'end',
|
||||
'any_failed': 'failed'
|
||||
}
|
||||
},
|
||||
project1_code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
agent: 'project1-programmer',
|
||||
on: { 'completed': 'project1_review' }
|
||||
},
|
||||
project1_review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
agent: 'project1-reviewer',
|
||||
on: { 'completed': 'project1_test' }
|
||||
},
|
||||
project1_test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
agent: 'project1-tester',
|
||||
on: { 'completed': 'join' }
|
||||
},
|
||||
project2_code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
agent: 'project2-programmer',
|
||||
on: { 'completed': 'project2_review' }
|
||||
},
|
||||
project2_review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
agent: 'project2-reviewer',
|
||||
on: { 'completed': 'project2_test' }
|
||||
},
|
||||
project2_test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
agent: 'project2-tester',
|
||||
on: { 'completed': 'join' }
|
||||
},
|
||||
project3_code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
agent: 'project3-programmer',
|
||||
on: { 'completed': 'project3_review' }
|
||||
},
|
||||
project3_review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
agent: 'project3-reviewer',
|
||||
on: { 'completed': 'project3_test' }
|
||||
},
|
||||
project3_test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
agent: 'project3-tester',
|
||||
on: { 'completed': 'join' }
|
||||
},
|
||||
project4_code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
agent: 'project4-programmer',
|
||||
on: { 'completed': 'project4_review' }
|
||||
},
|
||||
project4_review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
agent: 'project4-reviewer',
|
||||
on: { 'completed': 'project4_test' }
|
||||
},
|
||||
project4_test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
agent: 'project4-tester',
|
||||
on: { 'completed': 'join' }
|
||||
},
|
||||
join: {
|
||||
type: 'wait',
|
||||
on: {
|
||||
'all_joined': 'end'
|
||||
}
|
||||
},
|
||||
end: {
|
||||
type: 'end'
|
||||
},
|
||||
failed: {
|
||||
type: 'end',
|
||||
metadata: { status: 'failed' }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Human-in-the-Loop Workflow
|
||||
*/
|
||||
export const HUMAN_APPROVAL_WORKFLOW: YAMLWorkflow = {
|
||||
id: 'human-approval',
|
||||
name: 'Human Approval Workflow',
|
||||
version: '1.0.0',
|
||||
description: 'Workflow with human approval gates',
|
||||
initial: 'start',
|
||||
states: {
|
||||
start: {
|
||||
type: 'start',
|
||||
on: { 'start': 'plan' }
|
||||
},
|
||||
plan: {
|
||||
type: 'action',
|
||||
role: 'planner',
|
||||
on: { 'completed': 'await_approval' }
|
||||
},
|
||||
await_approval: {
|
||||
type: 'wait',
|
||||
timeout: '24h',
|
||||
on: {
|
||||
'approved': 'execute',
|
||||
'rejected': 'plan',
|
||||
'timeout': 'notify_timeout'
|
||||
}
|
||||
},
|
||||
notify_timeout: {
|
||||
type: 'action',
|
||||
action: 'notify',
|
||||
metadata: { message: 'Approval timeout' },
|
||||
on: { 'completed': 'await_approval' }
|
||||
},
|
||||
execute: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
on: { 'completed': 'review' }
|
||||
},
|
||||
review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
on: { 'completed': 'end' }
|
||||
},
|
||||
end: {
|
||||
type: 'end'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Default registry with predefined workflows
|
||||
export const defaultWorkflowRegistry = new WorkflowRegistry();
|
||||
|
||||
// Register predefined workflows
|
||||
defaultWorkflowRegistry.register(CODE_PIPELINE_WORKFLOW);
|
||||
defaultWorkflowRegistry.register(PARALLEL_PROJECTS_WORKFLOW);
|
||||
defaultWorkflowRegistry.register(HUMAN_APPROVAL_WORKFLOW);
|
||||
642
pipeline-system/workspace/agent-workspace.ts
Normal file
642
pipeline-system/workspace/agent-workspace.ts
Normal file
@@ -0,0 +1,642 @@
|
||||
/**
|
||||
* Agent Workspace Isolation
|
||||
*
|
||||
* Each agent gets its own tools, memory, identity, and workspace.
|
||||
* Provides isolation and resource management for parallel agents.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
import { mkdirSync, rmSync, existsSync, writeFileSync, readFileSync, readdirSync, statSync } from 'fs';
|
||||
import { join, resolve, relative } from 'path';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export type Permission = 'read' | 'write' | 'execute' | 'delete' | 'network' | 'git';
|
||||
|
||||
export interface WorkspaceConfig {
|
||||
id: string;
|
||||
projectId: string;
|
||||
agentId: string;
|
||||
role: string;
|
||||
basePath: string;
|
||||
permissions: Permission[];
|
||||
resourceLimits: ResourceLimits;
|
||||
environment: Record<string, string>;
|
||||
mountPoints: MountPoint[];
|
||||
}
|
||||
|
||||
export interface ResourceLimits {
|
||||
maxMemoryMB: number;
|
||||
maxCpuPercent: number;
|
||||
maxFileSizeMB: number;
|
||||
maxExecutionTimeMs: number;
|
||||
maxFileCount: number;
|
||||
}
|
||||
|
||||
export interface MountPoint {
|
||||
source: string;
|
||||
target: string;
|
||||
readOnly: boolean;
|
||||
}
|
||||
|
||||
export interface AgentTool {
|
||||
name: string;
|
||||
description: string;
|
||||
permissions: Permission[];
|
||||
execute: (params: unknown, context: ToolContext) => Promise<ToolResult>;
|
||||
}
|
||||
|
||||
export interface ToolContext {
|
||||
workspace: WorkspaceManager;
|
||||
agentId: string;
|
||||
sessionId: string;
|
||||
permissions: Permission[];
|
||||
}
|
||||
|
||||
export interface ToolResult {
|
||||
success: boolean;
|
||||
output?: unknown;
|
||||
error?: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface MemoryStore {
|
||||
shortTerm: Map<string, unknown>;
|
||||
longTerm: Map<string, unknown>;
|
||||
session: Map<string, unknown>;
|
||||
}
|
||||
|
||||
export interface AgentIdentity {
|
||||
id: string;
|
||||
name: string;
|
||||
role: string;
|
||||
description: string;
|
||||
personality: string;
|
||||
systemPrompt: string;
|
||||
capabilities: string[];
|
||||
constraints: string[];
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Workspace Manager
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* WorkspaceManager - Isolated workspace for an agent
|
||||
*/
|
||||
export class WorkspaceManager extends EventEmitter {
|
||||
private config: WorkspaceConfig;
|
||||
private workspacePath: string;
|
||||
private memory: MemoryStore;
|
||||
private identity: AgentIdentity;
|
||||
private tools: Map<string, AgentTool> = new Map();
|
||||
private fileHandles: Map<string, unknown> = new Map();
|
||||
private active = true;
|
||||
|
||||
constructor(config: WorkspaceConfig) {
|
||||
super();
|
||||
this.config = config;
|
||||
this.workspacePath = resolve(config.basePath, config.projectId, config.agentId);
|
||||
this.memory = {
|
||||
shortTerm: new Map(),
|
||||
longTerm: new Map(),
|
||||
session: new Map()
|
||||
};
|
||||
|
||||
this.initializeWorkspace();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the workspace directory
|
||||
*/
|
||||
private initializeWorkspace(): void {
|
||||
if (!existsSync(this.workspacePath)) {
|
||||
mkdirSync(this.workspacePath, { recursive: true });
|
||||
}
|
||||
|
||||
// Create subdirectories
|
||||
const subdirs = ['memory', 'output', 'cache', 'logs'];
|
||||
for (const dir of subdirs) {
|
||||
const path = join(this.workspacePath, dir);
|
||||
if (!existsSync(path)) {
|
||||
mkdirSync(path, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('workspaceInitialized', { path: this.workspacePath });
|
||||
}
|
||||
|
||||
/**
|
||||
* Set agent identity
|
||||
*/
|
||||
setIdentity(identity: AgentIdentity): void {
|
||||
this.identity = identity;
|
||||
this.emit('identitySet', { identity });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get agent identity
|
||||
*/
|
||||
getIdentity(): AgentIdentity | undefined {
|
||||
return this.identity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a tool
|
||||
*/
|
||||
registerTool(tool: AgentTool): void {
|
||||
// Check if agent has required permissions
|
||||
const hasPermission = tool.permissions.every(p =>
|
||||
this.config.permissions.includes(p)
|
||||
);
|
||||
|
||||
if (!hasPermission) {
|
||||
throw new Error(`Agent does not have required permissions for tool: ${tool.name}`);
|
||||
}
|
||||
|
||||
this.tools.set(tool.name, tool);
|
||||
this.emit('toolRegistered', { tool });
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister a tool
|
||||
*/
|
||||
unregisterTool(name: string): boolean {
|
||||
return this.tools.delete(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a tool
|
||||
*/
|
||||
async executeTool(name: string, params: unknown): Promise<ToolResult> {
|
||||
const tool = this.tools.get(name);
|
||||
if (!tool) {
|
||||
return { success: false, error: `Tool not found: ${name}` };
|
||||
}
|
||||
|
||||
const context: ToolContext = {
|
||||
workspace: this,
|
||||
agentId: this.config.agentId,
|
||||
sessionId: this.config.id,
|
||||
permissions: this.config.permissions
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await tool.execute(params, context);
|
||||
this.emit('toolExecuted', { name, params, result });
|
||||
return result;
|
||||
} catch (error) {
|
||||
const result: ToolResult = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
};
|
||||
this.emit('toolError', { name, params, error: result.error });
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get available tools
|
||||
*/
|
||||
getAvailableTools(): AgentTool[] {
|
||||
return Array.from(this.tools.values());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Memory Management
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Store value in short-term memory
|
||||
*/
|
||||
remember(key: string, value: unknown): void {
|
||||
this.memory.shortTerm.set(key, value);
|
||||
this.emit('memoryStored', { type: 'shortTerm', key });
|
||||
}
|
||||
|
||||
/**
|
||||
* Store value in long-term memory
|
||||
*/
|
||||
memorize(key: string, value: unknown): void {
|
||||
this.memory.longTerm.set(key, value);
|
||||
this.saveMemoryToFile(key, value, 'longTerm');
|
||||
this.emit('memoryStored', { type: 'longTerm', key });
|
||||
}
|
||||
|
||||
/**
|
||||
* Store value in session memory
|
||||
*/
|
||||
storeSession(key: string, value: unknown): void {
|
||||
this.memory.session.set(key, value);
|
||||
this.emit('memoryStored', { type: 'session', key });
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve value from memory
|
||||
*/
|
||||
recall(key: string): unknown | undefined {
|
||||
return (
|
||||
this.memory.shortTerm.get(key) ||
|
||||
this.memory.longTerm.get(key) ||
|
||||
this.memory.session.get(key)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if memory exists
|
||||
*/
|
||||
hasMemory(key: string): boolean {
|
||||
return (
|
||||
this.memory.shortTerm.has(key) ||
|
||||
this.memory.longTerm.has(key) ||
|
||||
this.memory.session.has(key)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Forget a memory
|
||||
*/
|
||||
forget(key: string): boolean {
|
||||
return (
|
||||
this.memory.shortTerm.delete(key) ||
|
||||
this.memory.longTerm.delete(key) ||
|
||||
this.memory.session.delete(key)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all short-term memory
|
||||
*/
|
||||
clearShortTerm(): void {
|
||||
this.memory.shortTerm.clear();
|
||||
this.emit('memoryCleared', { type: 'shortTerm' });
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear session memory
|
||||
*/
|
||||
clearSession(): void {
|
||||
this.memory.session.clear();
|
||||
this.emit('memoryCleared', { type: 'session' });
|
||||
}
|
||||
|
||||
/**
|
||||
* Save memory to file
|
||||
*/
|
||||
private saveMemoryToFile(key: string, value: unknown, type: string): void {
|
||||
const memoryPath = join(this.workspacePath, 'memory', `${type}.json`);
|
||||
let data: Record<string, unknown> = {};
|
||||
|
||||
if (existsSync(memoryPath)) {
|
||||
try {
|
||||
data = JSON.parse(readFileSync(memoryPath, 'utf-8'));
|
||||
} catch {
|
||||
data = {};
|
||||
}
|
||||
}
|
||||
|
||||
data[key] = value;
|
||||
writeFileSync(memoryPath, JSON.stringify(data, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Load long-term memory from file
|
||||
*/
|
||||
loadLongTermMemory(): void {
|
||||
const memoryPath = join(this.workspacePath, 'memory', 'longTerm.json');
|
||||
if (existsSync(memoryPath)) {
|
||||
try {
|
||||
const data = JSON.parse(readFileSync(memoryPath, 'utf-8'));
|
||||
for (const [key, value] of Object.entries(data)) {
|
||||
this.memory.longTerm.set(key, value);
|
||||
}
|
||||
} catch {
|
||||
// Ignore errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// File Operations
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Read a file
|
||||
*/
|
||||
readFile(path: string): string {
|
||||
this.checkPermission('read');
|
||||
const fullPath = this.resolvePath(path);
|
||||
this.checkPathInWorkspace(fullPath);
|
||||
|
||||
return readFileSync(fullPath, 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a file
|
||||
*/
|
||||
writeFile(path: string, content: string): void {
|
||||
this.checkPermission('write');
|
||||
const fullPath = this.resolvePath(path);
|
||||
this.checkPathInWorkspace(fullPath);
|
||||
this.checkFileSize(content.length);
|
||||
|
||||
writeFileSync(fullPath, content, 'utf-8');
|
||||
this.emit('fileWritten', { path: fullPath });
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file
|
||||
*/
|
||||
deleteFile(path: string): void {
|
||||
this.checkPermission('delete');
|
||||
const fullPath = this.resolvePath(path);
|
||||
this.checkPathInWorkspace(fullPath);
|
||||
|
||||
rmSync(fullPath, { force: true });
|
||||
this.emit('fileDeleted', { path: fullPath });
|
||||
}
|
||||
|
||||
/**
|
||||
* List files in a directory
|
||||
*/
|
||||
listFiles(path: string = ''): string[] {
|
||||
this.checkPermission('read');
|
||||
const fullPath = this.resolvePath(path);
|
||||
this.checkPathInWorkspace(fullPath);
|
||||
|
||||
if (!existsSync(fullPath)) return [];
|
||||
|
||||
return readdirSync(fullPath).map(name => join(path, name));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file exists
|
||||
*/
|
||||
fileExists(path: string): boolean {
|
||||
const fullPath = this.resolvePath(path);
|
||||
this.checkPathInWorkspace(fullPath);
|
||||
return existsSync(fullPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file stats
|
||||
*/
|
||||
getFileStats(path: string): { size: number; modified: Date; isDirectory: boolean } | null {
|
||||
const fullPath = this.resolvePath(path);
|
||||
this.checkPathInWorkspace(fullPath);
|
||||
|
||||
if (!existsSync(fullPath)) return null;
|
||||
|
||||
const stats = statSync(fullPath);
|
||||
return {
|
||||
size: stats.size,
|
||||
modified: stats.mtime,
|
||||
isDirectory: stats.isDirectory()
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Permission & Security
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Check if agent has a permission
|
||||
*/
|
||||
hasPermission(permission: Permission): boolean {
|
||||
return this.config.permissions.includes(permission);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check permission and throw if missing
|
||||
*/
|
||||
private checkPermission(permission: Permission): void {
|
||||
if (!this.hasPermission(permission)) {
|
||||
throw new Error(`Permission denied: ${permission}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve path relative to workspace
|
||||
*/
|
||||
private resolvePath(path: string): string {
|
||||
return resolve(this.workspacePath, path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if path is within workspace
|
||||
*/
|
||||
private checkPathInWorkspace(fullPath: string): void {
|
||||
const relativePath = relative(this.workspacePath, fullPath);
|
||||
if (relativePath.startsWith('..') || relativePath.startsWith('/')) {
|
||||
throw new Error('Path is outside workspace boundaries');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check file size limit
|
||||
*/
|
||||
private checkFileSize(size: number): void {
|
||||
const maxBytes = this.config.resourceLimits.maxFileSizeMB * 1024 * 1024;
|
||||
if (size > maxBytes) {
|
||||
throw new Error(`File size exceeds limit: ${this.config.resourceLimits.maxFileSizeMB}MB`);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Lifecycle
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Get workspace path
|
||||
*/
|
||||
getPath(): string {
|
||||
return this.workspacePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get workspace config
|
||||
*/
|
||||
getConfig(): WorkspaceConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up workspace
|
||||
*/
|
||||
cleanup(): void {
|
||||
this.active = false;
|
||||
this.clearSession();
|
||||
this.emit('workspaceCleanup', { path: this.workspacePath });
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy workspace (delete files)
|
||||
*/
|
||||
destroy(): void {
|
||||
this.cleanup();
|
||||
|
||||
if (existsSync(this.workspacePath)) {
|
||||
rmSync(this.workspacePath, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
this.emit('workspaceDestroyed', { path: this.workspacePath });
|
||||
}
|
||||
|
||||
/**
|
||||
* Export workspace state
|
||||
*/
|
||||
exportState(): {
|
||||
config: WorkspaceConfig;
|
||||
memory: Record<string, unknown>;
|
||||
identity?: AgentIdentity;
|
||||
tools: string[];
|
||||
} {
|
||||
return {
|
||||
config: this.getConfig(),
|
||||
memory: {
|
||||
shortTerm: Object.fromEntries(this.memory.shortTerm),
|
||||
longTerm: Object.fromEntries(this.memory.longTerm),
|
||||
session: Object.fromEntries(this.memory.session)
|
||||
},
|
||||
identity: this.identity,
|
||||
tools: Array.from(this.tools.keys())
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Workspace Factory
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* WorkspaceFactory - Creates and manages workspaces
|
||||
*/
|
||||
export class WorkspaceFactory {
|
||||
private basePath: string;
|
||||
private workspaces: Map<string, WorkspaceManager> = new Map();
|
||||
|
||||
constructor(basePath: string = './workspaces') {
|
||||
this.basePath = resolve(basePath);
|
||||
|
||||
if (!existsSync(this.basePath)) {
|
||||
mkdirSync(this.basePath, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new workspace
|
||||
*/
|
||||
createWorkspace(config: {
|
||||
projectId: string;
|
||||
agentId: string;
|
||||
role: string;
|
||||
permissions?: Permission[];
|
||||
resourceLimits?: Partial<ResourceLimits>;
|
||||
}): WorkspaceManager {
|
||||
const id = `ws-${randomUUID().substring(0, 8)}`;
|
||||
|
||||
const fullConfig: WorkspaceConfig = {
|
||||
id,
|
||||
projectId: config.projectId,
|
||||
agentId: config.agentId,
|
||||
role: config.role,
|
||||
basePath: this.basePath,
|
||||
permissions: config.permissions || ['read'],
|
||||
resourceLimits: {
|
||||
maxMemoryMB: 512,
|
||||
maxCpuPercent: 50,
|
||||
maxFileSizeMB: 10,
|
||||
maxExecutionTimeMs: 60000,
|
||||
maxFileCount: 1000,
|
||||
...config.resourceLimits
|
||||
},
|
||||
environment: {},
|
||||
mountPoints: []
|
||||
};
|
||||
|
||||
const workspace = new WorkspaceManager(fullConfig);
|
||||
this.workspaces.set(id, workspace);
|
||||
|
||||
return workspace;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a workspace by ID
|
||||
*/
|
||||
getWorkspace(id: string): WorkspaceManager | undefined {
|
||||
return this.workspaces.get(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get workspaces by project
|
||||
*/
|
||||
getWorkspacesByProject(projectId: string): WorkspaceManager[] {
|
||||
return Array.from(this.workspaces.values())
|
||||
.filter(w => w.getConfig().projectId === projectId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all workspaces
|
||||
*/
|
||||
getAllWorkspaces(): WorkspaceManager[] {
|
||||
return Array.from(this.workspaces.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy a workspace
|
||||
*/
|
||||
destroyWorkspace(id: string): boolean {
|
||||
const workspace = this.workspaces.get(id);
|
||||
if (workspace) {
|
||||
workspace.destroy();
|
||||
return this.workspaces.delete(id);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy all workspaces for a project
|
||||
*/
|
||||
destroyProjectWorkspaces(projectId: string): number {
|
||||
const projectWorkspaces = this.getWorkspacesByProject(projectId);
|
||||
let count = 0;
|
||||
|
||||
for (const workspace of projectWorkspaces) {
|
||||
workspace.destroy();
|
||||
this.workspaces.delete(workspace.getConfig().id);
|
||||
count++;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get factory stats
|
||||
*/
|
||||
getStats(): {
|
||||
totalWorkspaces: number;
|
||||
byProject: Record<string, number>;
|
||||
byRole: Record<string, number>;
|
||||
} {
|
||||
const byProject: Record<string, number> = {};
|
||||
const byRole: Record<string, number> = {};
|
||||
|
||||
for (const workspace of this.workspaces.values()) {
|
||||
const config = workspace.getConfig();
|
||||
byProject[config.projectId] = (byProject[config.projectId] || 0) + 1;
|
||||
byRole[config.role] = (byRole[config.role] || 0) + 1;
|
||||
}
|
||||
|
||||
return {
|
||||
totalWorkspaces: this.workspaces.size,
|
||||
byProject,
|
||||
byRole
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Default factory instance
|
||||
export const defaultWorkspaceFactory = new WorkspaceFactory();
|
||||
Reference in New Issue
Block a user