Complete Agent Pipeline System with Claude Code & OpenClaw Integration
- Added Claude Code integration with full context compaction support - Added OpenClaw integration with deterministic pipeline support - Implemented parallel agent execution (4 projects x 3 roles pattern) - Added workspace isolation with permissions and quotas - Implemented Lobster-compatible YAML workflow parser - Added persistent memory store for cross-session context - Created comprehensive README with hero section This project was 100% autonomously built by Z.AI GLM-5
This commit is contained in:
admin
540
pipeline-system/workflows/yaml-workflow.ts
Normal file
540
pipeline-system/workflows/yaml-workflow.ts
Normal file
@@ -0,0 +1,540 @@
|
||||
/**
|
||||
* YAML Workflow Integration (Lobster-Compatible)
|
||||
*
|
||||
* Parses YAML workflow definitions and converts them to
|
||||
* deterministic state machine definitions.
|
||||
*
|
||||
* Compatible with OpenClaw/Lobster workflow format.
|
||||
*/
|
||||
|
||||
import { StateMachineDefinition, State, Transition, RetryConfig } from '../core/state-machine';
|
||||
import { AgentRole } from '../engine/parallel-executor';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export interface YAMLWorkflow {
|
||||
id: string;
|
||||
name: string;
|
||||
version?: string;
|
||||
description?: string;
|
||||
initial: string;
|
||||
states: Record<string, YAMLState>;
|
||||
events?: string[];
|
||||
context?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface YAMLState {
|
||||
type: 'start' | 'end' | 'action' | 'parallel' | 'choice' | 'wait' | 'loop' | 'subworkflow';
|
||||
agent?: string;
|
||||
role?: AgentRole;
|
||||
action?: string;
|
||||
timeout?: number | string;
|
||||
retry?: YAMLRetryConfig;
|
||||
on?: Record<string, YAMLTransition | string>;
|
||||
branches?: Record<string, string>;
|
||||
conditions?: YAMLCondition[];
|
||||
subworkflow?: string;
|
||||
loop?: YAMLLoopConfig;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface YAMLTransition {
|
||||
target: string;
|
||||
condition?: YAMLCondition;
|
||||
guard?: string;
|
||||
}
|
||||
|
||||
export interface YAMLCondition {
|
||||
type: 'equals' | 'contains' | 'exists' | 'custom';
|
||||
field: string;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
export interface YAMLRetryConfig {
|
||||
maxAttempts: number;
|
||||
backoff?: 'fixed' | 'exponential' | 'linear';
|
||||
initialDelay?: number | string;
|
||||
maxDelay?: number | string;
|
||||
}
|
||||
|
||||
export interface YAMLLoopConfig {
|
||||
maxIterations: number;
|
||||
iterator?: string;
|
||||
body: string;
|
||||
exitCondition?: YAMLCondition;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Workflow Parser
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* WorkflowParser - Parses YAML workflows to state machine definitions
|
||||
*/
|
||||
export class WorkflowParser {
|
||||
/**
|
||||
* Parse a YAML workflow to a state machine definition
|
||||
*/
|
||||
parse(yaml: YAMLWorkflow): StateMachineDefinition {
|
||||
const states: Record<string, State> = {};
|
||||
|
||||
for (const [stateId, yamlState] of Object.entries(yaml.states)) {
|
||||
states[stateId] = this.parseState(stateId, yamlState);
|
||||
}
|
||||
|
||||
return {
|
||||
id: yaml.id,
|
||||
name: yaml.name,
|
||||
version: yaml.version || '1.0.0',
|
||||
description: yaml.description,
|
||||
initial: yaml.initial,
|
||||
states,
|
||||
events: yaml.events,
|
||||
context: yaml.context
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single state
|
||||
*/
|
||||
private parseState(stateId: string, yamlState: YAMLState): State {
|
||||
const state: State = {
|
||||
id: stateId,
|
||||
name: stateId,
|
||||
type: yamlState.type,
|
||||
agent: yamlState.agent,
|
||||
action: yamlState.action,
|
||||
timeout: this.parseDuration(yamlState.timeout),
|
||||
metadata: {
|
||||
...yamlState.metadata,
|
||||
role: yamlState.role
|
||||
}
|
||||
};
|
||||
|
||||
// Parse retry config
|
||||
if (yamlState.retry) {
|
||||
state.retry = {
|
||||
maxAttempts: yamlState.retry.maxAttempts,
|
||||
backoff: yamlState.retry.backoff || 'exponential',
|
||||
initialDelay: this.parseDuration(yamlState.retry.initialDelay) || 1000,
|
||||
maxDelay: this.parseDuration(yamlState.retry.maxDelay) || 60000
|
||||
};
|
||||
}
|
||||
|
||||
// Parse transitions (on)
|
||||
if (yamlState.on) {
|
||||
const transitions = this.parseTransitions(yamlState.on);
|
||||
state.onExit = transitions;
|
||||
}
|
||||
|
||||
// Parse parallel branches
|
||||
if (yamlState.branches) {
|
||||
state.type = 'parallel';
|
||||
state.onEnter = Object.entries(yamlState.branches).map(([event, target]) => ({
|
||||
event,
|
||||
target
|
||||
}));
|
||||
}
|
||||
|
||||
// Parse loop config
|
||||
if (yamlState.loop) {
|
||||
state.type = 'loop';
|
||||
state.metadata = {
|
||||
...state.metadata,
|
||||
maxIterations: yamlState.loop.maxIterations,
|
||||
iterator: yamlState.loop.iterator,
|
||||
body: yamlState.loop.body
|
||||
};
|
||||
|
||||
// Add loop transitions
|
||||
state.onExit = [
|
||||
{ event: 'continue', target: yamlState.loop.body },
|
||||
{ event: 'exit', target: yamlState.on?.['exit'] as string || 'end' }
|
||||
];
|
||||
}
|
||||
|
||||
// Parse subworkflow
|
||||
if (yamlState.subworkflow) {
|
||||
state.type = 'action';
|
||||
state.action = 'subworkflow';
|
||||
state.metadata = {
|
||||
...state.metadata,
|
||||
subworkflow: yamlState.subworkflow
|
||||
};
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse transitions from YAML format
|
||||
*/
|
||||
private parseTransitions(on: Record<string, YAMLTransition | string>): Transition[] {
|
||||
const transitions: Transition[] = [];
|
||||
|
||||
for (const [event, transition] of Object.entries(on)) {
|
||||
if (typeof transition === 'string') {
|
||||
transitions.push({ event, target: transition });
|
||||
} else {
|
||||
transitions.push({
|
||||
event,
|
||||
target: transition.target,
|
||||
condition: transition.condition ? this.parseCondition(transition.condition) : undefined,
|
||||
guard: transition.guard
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return transitions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a condition
|
||||
*/
|
||||
private parseCondition(yamlCond: YAMLCondition): Transition['condition'] {
|
||||
return {
|
||||
type: yamlCond.type,
|
||||
field: yamlCond.field,
|
||||
value: yamlCond.value
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse duration string (e.g., '30s', '5m', '1h')
|
||||
*/
|
||||
private parseDuration(duration?: number | string): number | undefined {
|
||||
if (typeof duration === 'number') return duration;
|
||||
if (!duration) return undefined;
|
||||
|
||||
const match = duration.match(/^(\d+)(ms|s|m|h)?$/);
|
||||
if (!match) return undefined;
|
||||
|
||||
const value = parseInt(match[1]);
|
||||
const unit = match[2] || 'ms';
|
||||
|
||||
switch (unit) {
|
||||
case 'ms': return value;
|
||||
case 's': return value * 1000;
|
||||
case 'm': return value * 60 * 1000;
|
||||
case 'h': return value * 60 * 60 * 1000;
|
||||
default: return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Workflow Registry
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* WorkflowRegistry - Manages workflow definitions
|
||||
*/
|
||||
export class WorkflowRegistry {
|
||||
private workflows: Map<string, YAMLWorkflow> = new Map();
|
||||
private parser: WorkflowParser;
|
||||
|
||||
constructor() {
|
||||
this.parser = new WorkflowParser();
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a workflow from YAML object
|
||||
*/
|
||||
register(yaml: YAMLWorkflow): StateMachineDefinition {
|
||||
this.workflows.set(yaml.id, yaml);
|
||||
return this.parser.parse(yaml);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a workflow by ID
|
||||
*/
|
||||
get(id: string): YAMLWorkflow | undefined {
|
||||
return this.workflows.get(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parsed state machine definition
|
||||
*/
|
||||
getParsed(id: string): StateMachineDefinition | undefined {
|
||||
const yaml = this.workflows.get(id);
|
||||
if (yaml) {
|
||||
return this.parser.parse(yaml);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all workflows
|
||||
*/
|
||||
list(): string[] {
|
||||
return Array.from(this.workflows.keys());
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Predefined Workflows
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Standard Code Pipeline Workflow
|
||||
*
|
||||
* Code → Review → Test → Done
|
||||
* With max 3 review iterations
|
||||
*/
|
||||
export const CODE_PIPELINE_WORKFLOW: YAMLWorkflow = {
|
||||
id: 'code-pipeline',
|
||||
name: 'Code Pipeline',
|
||||
version: '1.0.0',
|
||||
description: 'Code → Review → Test pipeline with deterministic flow',
|
||||
initial: 'start',
|
||||
context: {
|
||||
reviewIteration: 0,
|
||||
maxReviewIterations: 3
|
||||
},
|
||||
states: {
|
||||
start: {
|
||||
type: 'start',
|
||||
on: {
|
||||
'start': 'code'
|
||||
}
|
||||
},
|
||||
code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
timeout: '30m',
|
||||
retry: {
|
||||
maxAttempts: 2,
|
||||
backoff: 'exponential',
|
||||
initialDelay: '5s',
|
||||
maxDelay: '1m'
|
||||
},
|
||||
on: {
|
||||
'completed': 'review',
|
||||
'failed': 'failed'
|
||||
}
|
||||
},
|
||||
review: {
|
||||
type: 'choice',
|
||||
conditions: [
|
||||
{ type: 'equals', field: 'reviewApproved', value: true }
|
||||
],
|
||||
on: {
|
||||
'approved': 'test',
|
||||
'rejected': 'review_loop',
|
||||
'failed': 'failed'
|
||||
}
|
||||
},
|
||||
review_loop: {
|
||||
type: 'loop',
|
||||
loop: {
|
||||
maxIterations: 3,
|
||||
body: 'code'
|
||||
},
|
||||
on: {
|
||||
'exit': 'failed'
|
||||
}
|
||||
},
|
||||
test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
timeout: '15m',
|
||||
on: {
|
||||
'passed': 'end',
|
||||
'failed': 'test_failed'
|
||||
}
|
||||
},
|
||||
test_failed: {
|
||||
type: 'choice',
|
||||
on: {
|
||||
'retry': 'code',
|
||||
'abort': 'failed'
|
||||
}
|
||||
},
|
||||
end: {
|
||||
type: 'end'
|
||||
},
|
||||
failed: {
|
||||
type: 'end',
|
||||
metadata: { status: 'failed' }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Parallel Multi-Project Workflow
|
||||
*
|
||||
* Runs multiple projects in parallel
|
||||
*/
|
||||
export const PARALLEL_PROJECTS_WORKFLOW: YAMLWorkflow = {
|
||||
id: 'parallel-projects',
|
||||
name: 'Parallel Projects Pipeline',
|
||||
version: '1.0.0',
|
||||
description: 'Run multiple projects in parallel with synchronized completion',
|
||||
initial: 'start',
|
||||
states: {
|
||||
start: {
|
||||
type: 'start',
|
||||
on: {
|
||||
'start': 'parallel'
|
||||
}
|
||||
},
|
||||
parallel: {
|
||||
type: 'parallel',
|
||||
branches: {
|
||||
'project1': 'project1_code',
|
||||
'project2': 'project2_code',
|
||||
'project3': 'project3_code',
|
||||
'project4': 'project4_code'
|
||||
},
|
||||
on: {
|
||||
'all_completed': 'end',
|
||||
'any_failed': 'failed'
|
||||
}
|
||||
},
|
||||
project1_code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
agent: 'project1-programmer',
|
||||
on: { 'completed': 'project1_review' }
|
||||
},
|
||||
project1_review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
agent: 'project1-reviewer',
|
||||
on: { 'completed': 'project1_test' }
|
||||
},
|
||||
project1_test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
agent: 'project1-tester',
|
||||
on: { 'completed': 'join' }
|
||||
},
|
||||
project2_code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
agent: 'project2-programmer',
|
||||
on: { 'completed': 'project2_review' }
|
||||
},
|
||||
project2_review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
agent: 'project2-reviewer',
|
||||
on: { 'completed': 'project2_test' }
|
||||
},
|
||||
project2_test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
agent: 'project2-tester',
|
||||
on: { 'completed': 'join' }
|
||||
},
|
||||
project3_code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
agent: 'project3-programmer',
|
||||
on: { 'completed': 'project3_review' }
|
||||
},
|
||||
project3_review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
agent: 'project3-reviewer',
|
||||
on: { 'completed': 'project3_test' }
|
||||
},
|
||||
project3_test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
agent: 'project3-tester',
|
||||
on: { 'completed': 'join' }
|
||||
},
|
||||
project4_code: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
agent: 'project4-programmer',
|
||||
on: { 'completed': 'project4_review' }
|
||||
},
|
||||
project4_review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
agent: 'project4-reviewer',
|
||||
on: { 'completed': 'project4_test' }
|
||||
},
|
||||
project4_test: {
|
||||
type: 'action',
|
||||
role: 'tester',
|
||||
agent: 'project4-tester',
|
||||
on: { 'completed': 'join' }
|
||||
},
|
||||
join: {
|
||||
type: 'wait',
|
||||
on: {
|
||||
'all_joined': 'end'
|
||||
}
|
||||
},
|
||||
end: {
|
||||
type: 'end'
|
||||
},
|
||||
failed: {
|
||||
type: 'end',
|
||||
metadata: { status: 'failed' }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Human-in-the-Loop Workflow
|
||||
*/
|
||||
export const HUMAN_APPROVAL_WORKFLOW: YAMLWorkflow = {
|
||||
id: 'human-approval',
|
||||
name: 'Human Approval Workflow',
|
||||
version: '1.0.0',
|
||||
description: 'Workflow with human approval gates',
|
||||
initial: 'start',
|
||||
states: {
|
||||
start: {
|
||||
type: 'start',
|
||||
on: { 'start': 'plan' }
|
||||
},
|
||||
plan: {
|
||||
type: 'action',
|
||||
role: 'planner',
|
||||
on: { 'completed': 'await_approval' }
|
||||
},
|
||||
await_approval: {
|
||||
type: 'wait',
|
||||
timeout: '24h',
|
||||
on: {
|
||||
'approved': 'execute',
|
||||
'rejected': 'plan',
|
||||
'timeout': 'notify_timeout'
|
||||
}
|
||||
},
|
||||
notify_timeout: {
|
||||
type: 'action',
|
||||
action: 'notify',
|
||||
metadata: { message: 'Approval timeout' },
|
||||
on: { 'completed': 'await_approval' }
|
||||
},
|
||||
execute: {
|
||||
type: 'action',
|
||||
role: 'programmer',
|
||||
on: { 'completed': 'review' }
|
||||
},
|
||||
review: {
|
||||
type: 'action',
|
||||
role: 'reviewer',
|
||||
on: { 'completed': 'end' }
|
||||
},
|
||||
end: {
|
||||
type: 'end'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Default registry with predefined workflows
|
||||
export const defaultWorkflowRegistry = new WorkflowRegistry();
|
||||
|
||||
// Register predefined workflows
|
||||
defaultWorkflowRegistry.register(CODE_PIPELINE_WORKFLOW);
|
||||
defaultWorkflowRegistry.register(PARALLEL_PROJECTS_WORKFLOW);
|
||||
defaultWorkflowRegistry.register(HUMAN_APPROVAL_WORKFLOW);
|
||||
Reference in New Issue
Block a user