/** * Computer Use Integration Module * Unified interface for all computer automation capabilities * * Integrates: * - Playwright browser automation (browser-use inspired) * - PowerShell desktop automation (Windows-Use inspired) * - Vision loop for autonomous control (Open-Interface inspired) * - Course correction for reliability */ import { spawn, execSync } from 'child_process'; import path from 'path'; import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); // Paths to executables const PLAYWRIGHT_BRIDGE = path.join(__dirname, '..', 'bin', 'playwright-bridge.js'); const INPUT_PS1 = path.join(__dirname, '..', 'bin', 'input.ps1'); /** * Execute a Playwright command */ export async function playwrightCommand(command, ...args) { return new Promise((resolve, reject) => { const nodeArgs = [PLAYWRIGHT_BRIDGE, command, ...args]; console.log(`[Playwright] ${command} ${args.join(' ')}`); const proc = spawn('node', nodeArgs, { cwd: path.dirname(PLAYWRIGHT_BRIDGE), shell: true }); let stdout = ''; let stderr = ''; proc.stdout.on('data', (data) => { stdout += data.toString(); }); proc.stderr.on('data', (data) => { stderr += data.toString(); }); proc.on('close', (code) => { if (code === 0) { resolve({ success: true, output: stdout.trim() }); } else { resolve({ success: false, output: stdout.trim(), error: stderr.trim() }); } }); proc.on('error', (err) => { reject(err); }); // Timeout after 30 seconds setTimeout(() => { proc.kill(); reject(new Error('Command timeout')); }, 30000); }); } /** * Execute a PowerShell command via input.ps1 */ export async function powershellCommand(command, ...args) { return new Promise((resolve, reject) => { const psArgs = ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', INPUT_PS1, command, ...args]; console.log(`[PowerShell] ${command} ${args.join(' ')}`); const proc = spawn('powershell', psArgs, { cwd: path.dirname(INPUT_PS1), shell: true }); let stdout = ''; let stderr = ''; proc.stdout.on('data', (data) => { stdout += data.toString(); }); proc.stderr.on('data', (data) => { stderr += data.toString(); }); proc.on('close', (code) => { if (code === 0) { resolve({ success: true, output: stdout.trim() }); } else { resolve({ success: false, output: stdout.trim(), error: stderr.trim() }); } }); proc.on('error', (err) => { reject(err); }); // Timeout after 30 seconds setTimeout(() => { proc.kill(); reject(new Error('Command timeout')); }, 30000); }); } /** * Intelligent command router * Automatically routes to Playwright or PowerShell based on command type */ export async function executeCommand(commandString) { const trimmed = commandString.trim(); // Parse the command string if (trimmed.startsWith('node') && trimmed.includes('playwright-bridge')) { // Extract Playwright command const match = trimmed.match(/playwright-bridge\.js\s+(\w+)\s*(.*)/); if (match) { const cmd = match[1]; const argsStr = match[2] || ''; const args = argsStr.match(/"[^"]+"|'[^']+'|\S+/g) || []; const cleanArgs = args.map(a => a.replace(/^["']|["']$/g, '')); return await playwrightCommand(cmd, ...cleanArgs); } } else if (trimmed.startsWith('powershell') && trimmed.includes('input.ps1')) { // Extract PowerShell command const match = trimmed.match(/input\.ps1\s+(\w+)\s*(.*)/); if (match) { const cmd = match[1]; const argsStr = match[2] || ''; const args = argsStr.match(/"[^"]+"|'[^']+'|\S+/g) || []; const cleanArgs = args.map(a => a.replace(/^["']|["']$/g, '')); return await powershellCommand(cmd, ...cleanArgs); } } // Try to infer command type const browserKeywords = ['navigate', 'fill', 'click', 'press', 'content', 'elements', 'screenshot']; const desktopKeywords = ['open', 'uiclick', 'type', 'key', 'mouse', 'apps', 'focus', 'waitfor', 'app_state']; const words = trimmed.toLowerCase().split(/\s+/); const firstWord = words[0]; if (browserKeywords.includes(firstWord)) { return await playwrightCommand(firstWord, ...words.slice(1)); } else if (desktopKeywords.includes(firstWord)) { return await powershellCommand(firstWord, ...words.slice(1)); } return { success: false, error: 'Unknown command format' }; } /** * Execute multiple commands in sequence with verification */ export async function executeSequence(commands, options = {}) { const { onCommand = () => { }, onResult = () => { }, stopOnError = true, delayBetween = 500 } = options; const results = []; for (let i = 0; i < commands.length; i++) { const command = commands[i]; onCommand(i, command); try { const result = await executeCommand(command); results.push({ command, ...result }); onResult(i, result); if (!result.success && stopOnError) { break; } // Wait between commands if (i < commands.length - 1) { await new Promise(resolve => setTimeout(resolve, delayBetween)); } } catch (error) { results.push({ command, success: false, error: error.message }); if (stopOnError) break; } } return results; } /** * Browser automation shortcuts */ export const browser = { navigate: (url) => playwrightCommand('navigate', url), click: (selector) => playwrightCommand('click', selector), fill: (selector, text) => playwrightCommand('fill', selector, text), type: (text) => playwrightCommand('type', text), press: (key) => playwrightCommand('press', key), content: () => playwrightCommand('content'), elements: () => playwrightCommand('elements'), screenshot: (file) => playwrightCommand('screenshot', file || 'screenshot.png'), close: () => playwrightCommand('close') }; /** * Desktop automation shortcuts */ export const desktop = { open: (app) => powershellCommand('open', app), click: () => powershellCommand('click'), rightClick: () => powershellCommand('rightclick'), doubleClick: () => powershellCommand('doubleclick'), type: (text) => powershellCommand('type', text), key: (keyName) => powershellCommand('key', keyName), hotkey: (...keys) => powershellCommand('hotkey', keys.join('+')), mouse: (x, y) => powershellCommand('mouse', x, y), scroll: (amount) => powershellCommand('scroll', amount), uiClick: (element) => powershellCommand('uiclick', element), find: (element) => powershellCommand('find', element), apps: () => powershellCommand('apps'), focus: (window) => powershellCommand('focus', window), waitfor: (element, timeout) => powershellCommand('waitfor', element, timeout), appState: (window) => powershellCommand('app_state', window), screenshot: (file) => powershellCommand('screenshot', file || 'screenshot.png'), ocr: (region) => powershellCommand('ocr', region) }; export default { playwrightCommand, powershellCommand, executeCommand, executeSequence, browser, desktop, paths: { playwrightBridge: PLAYWRIGHT_BRIDGE, inputPs1: INPUT_PS1 } };