/** * IQ Exchange - Universal Self-Healing Intelligence Layer * * This is the BRAIN that sits between user requests and AI responses. * It dynamically: * 1. Analyzes any user request * 2. Routes to appropriate handler (code, file, browser, desktop, etc.) * 3. Executes actions and captures ALL output * 4. Detects errors and asks AI to fix them * 5. Retries until success or max attempts * * Works for ALL task types, not just computer use. */ import { spawn, exec } from 'child_process'; import path from 'path'; import fs from 'fs'; import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); // System paths const SYSTEM_PATHS = { playwrightBridge: path.join(__dirname, '..', 'bin', 'playwright-bridge.js').replace(/\\/g, '/'), inputPs1: path.join(__dirname, '..', 'bin', 'input.ps1').replace(/\\/g, '/'), projectRoot: path.join(__dirname, '..').replace(/\\/g, '/') }; /** * Task Type Detection */ const TASK_PATTERNS = { browser: /\b(website|browser|google|youtube|amazon|navigate|search online|search\b|open.*url|go to.*\.com|fill.*form|click.*button|chrome|chromium|edge|msedge|firefox)\b/i, desktop: /\b(open.*app|launch|click.*menu|type.*text|press.*key|screenshot|notepad|paint|calculator|telegram|discord)\b/i, code: /\b(write.*code|create.*file|function|class|module|implement|code.*for|script.*for)\b/i, file: /\b(create.*file|write.*file|save.*to|read.*file|edit.*file|delete.*file|rename)\b/i, shell: /\b(run.*command|terminal|shell|npm|node|pip|git|docker)\b/i, server: /\b(server|service|daemon|port|localhost|endpoint|api|health|status|logs|restart|start|stop|deploy|pm2|nginx|apache|systemctl)\b/i, query: /\b(what|how|why|explain|tell me|describe|list|show me)\b/i }; export function detectTaskType(request) { const types = []; for (const [type, pattern] of Object.entries(TASK_PATTERNS)) { if (pattern.test(request)) { types.push(type); } } return types.length > 0 ? types : ['general']; } /** * Execute any command and capture result */ export async function executeAny(command, options = {}) { const { timeout = 30000, cwd = SYSTEM_PATHS.projectRoot } = options; return new Promise((resolve) => { const startTime = Date.now(); let stdout = ''; let stderr = ''; // Parse command to determine execution method let proc; if (command.includes('playwright-bridge') || command.match(/^node\s/)) { // Node.js / Playwright command const cleanCmd = command.replace(/^node\s+/, ''); const parts = cleanCmd.match(/"[^"]+"|'[^']+'|\S+/g) || []; const cleanParts = parts.map(p => p.replace(/^["']|["']$/g, '')); // Ensure we use absolute path let scriptPath = cleanParts[0]; if (!path.isAbsolute(scriptPath)) { scriptPath = path.join(cwd, scriptPath); } proc = spawn('node', [scriptPath, ...cleanParts.slice(1)], { cwd, shell: true }); } else if (command.includes('powershell') || command.includes('input.ps1')) { // PowerShell command - extract and normalize let psCommand; if (command.includes('-File')) { // Already formatted correctly const match = command.match(/-File\s+["']?([^"'\s]+)["']?\s*(.*)/); if (match) { const scriptPath = match[1].includes('input.ps1') ? SYSTEM_PATHS.inputPs1 : match[1]; const args = match[2]; psCommand = ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', scriptPath, ...args.split(/\s+/).filter(Boolean)]; } } else { // Normalize common forms: // - powershell bin/input.ps1 // - powershell "bin/input.ps1" // - bin/input.ps1 const psPrefixStripped = command.replace(/^powershell\s*/i, ''); const inputPs1Match = psPrefixStripped.match(/["']?([^"'\s]*input\.ps1)["']?\s*(.*)/i) || command.match(/["']?([^"'\s]*input\.ps1)["']?\s*(.*)/i); if (inputPs1Match) { const argsStr = (inputPs1Match[2] || '').trim(); const args = argsStr.match(/"[^"]*"|'[^']*'|\S+/g) || []; const cleanArgs = args.map(a => a.replace(/^["']|["']$/g, '')); psCommand = ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', SYSTEM_PATHS.inputPs1, ...cleanArgs]; } else { // Fall back to running whatever was provided (no input.ps1 detected) psCommand = ['-Command', psPrefixStripped]; } } console.log("Running:", 'powershell', psCommand.join(' ')); // Debug log proc = spawn('powershell', psCommand || [command], { cwd, shell: true }); } else { // Generic command proc = spawn('cmd', ['/c', command], { cwd, shell: true }); } proc.stdout.on('data', (data) => { stdout += data.toString(); }); proc.stderr.on('data', (data) => { stderr += data.toString(); }); proc.on('close', (code) => { resolve({ success: code === 0 || stdout.includes('RESULT:'), exitCode: code, stdout: stdout.trim(), stderr: stderr.trim(), elapsed: Date.now() - startTime, command }); }); proc.on('error', (err) => { resolve({ success: false, error: err.message, stdout: stdout.trim(), stderr: stderr.trim(), command }); }); setTimeout(() => { proc.kill(); resolve({ success: false, error: 'TIMEOUT', stdout: stdout.trim(), stderr: stderr.trim(), command }); }, timeout); }); } /** * Extract executable code/commands from AI response */ export function extractExecutables(response) { const executables = []; // Match all code blocks const codeBlockRegex = /```(\w*)(?::run)?\r?\n([\s\S]*?)```/g; let match; while ((match = codeBlockRegex.exec(response)) !== null) { const lang = match[1].toLowerCase(); const code = match[2].replace(/\r/g, '').trim(); if (['bash', 'shell', 'powershell', 'ps1', 'cmd', 'sh'].includes(lang) || lang === '') { // Command to execute const lines = code.split('\n').filter(l => l.trim() && !l.startsWith('#') && !l.startsWith('//')); lines.forEach(line => { executables.push({ type: 'command', content: line.trim(), lang }); }); } else if (['javascript', 'js', 'typescript', 'ts', 'python', 'py'].includes(lang)) { // Code block - might need to write to file executables.push({ type: 'code', content: code, lang }); } } return executables; } function fallbackExtractCommandsFromText(response) { const lines = String(response || '') .replace(/\r/g, '') .split('\n') .map(l => l.trim()) .filter(Boolean); const commands = []; for (const line of lines) { if (/^(powershell|pwsh|cmd|node|npm|pnpm|yarn|git)\b/i.test(line)) { commands.push(line); } } return commands; } function sanitizeExtractedExecutables(executables) { const out = []; for (const e of Array.isArray(executables) ? executables : []) { if (!e || e.type !== 'command') continue; const cmd = String(e.content || '').replace(/\r/g, '').trim(); if (!cmd) continue; // Prevent garbage “commands” from slipping through (covers many “half formatted” model outputs) if (!/^(powershell|pwsh|cmd|node|npm|pnpm|yarn|git)\b/i.test(cmd)) continue; out.push({ ...e, content: cmd }); } return out; } /** * Check if response indicates task completion */ export function isComplete(response) { const completionMarkers = [ 'TASK_COMPLETE', 'task completed', 'successfully completed', 'done!', 'that should work', 'completed successfully' ]; return completionMarkers.some(m => response.toLowerCase().includes(m.toLowerCase())); } /** * Check if response indicates an error that needs fixing */ export function detectError(result) { if (!result.success) return true; const errorPatterns = [ /error:/i, /failed/i, /exception/i, /not found/i, /cannot find/i, /permission denied/i, /ENOENT/i, /EACCES/i ]; const output = result.stdout + result.stderr; return errorPatterns.some(p => p.test(output)); } /** * Build self-healing prompt for AI */ export function buildHealingPrompt(originalRequest, executionHistory, lastError) { return ` ═══════════════════════════════════════════════════════════════════════════════════ IQ EXCHANGE - SELF-HEALING MODE ═══════════════════════════════════════════════════════════════════════════════════ ORIGINAL REQUEST: "${originalRequest}" EXECUTION HISTORY: ${executionHistory.map((h, i) => ` [Attempt ${i + 1}] Command: ${h.command} Status: ${h.success ? '✅ SUCCESS' : '❌ FAILED'} Output: ${(h.stdout || h.stderr || h.error || 'No output').substring(0, 500)} `).join('\n')} LAST ERROR: ${lastError} ═══════════════════════════════════════════════════════════════════════════════════ AVAILABLE SYSTEM COMMANDS (use EXACT paths): ═══════════════════════════════════════════════════════════════════════════════════ BROWSER (Playwright - all actions in same session): node "${SYSTEM_PATHS.playwrightBridge}" navigate "URL" node "${SYSTEM_PATHS.playwrightBridge}" fill "selector" "text" node "${SYSTEM_PATHS.playwrightBridge}" click "selector" node "${SYSTEM_PATHS.playwrightBridge}" press "Enter" node "${SYSTEM_PATHS.playwrightBridge}" type "text" node "${SYSTEM_PATHS.playwrightBridge}" content DESKTOP (PowerShell - always use -File flag): powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" open "app.exe" powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" uiclick "ElementName" powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" waitfor "Text" 10 powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" app_state "WindowName" powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" ocr "full" powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" keyboard "text" powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" key KEYNAME powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" mouse X Y powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" click powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" drag X1 Y1 X2 Y2 ═══════════════════════════════════════════════════════════════════════════════════ YOUR TASK: 1. Analyze why the previous attempt failed 2. Provide CORRECTED commands that will work 3. Each command in its own code block 4. If the task is actually complete, just say "TASK_COMPLETE" ═══════════════════════════════════════════════════════════════════════════════════ `; } function buildTranslationPrompt(userRequest) { const request = String(userRequest || ''); const looksWeb = /\bhttps?:\/\//i.test(request) || /\bwww\./i.test(request) || /\bgoogle\.com\b/i.test(request) || /\b(search|browse|navigate|open\s+.*url|go\s+to)\b/i.test(request) || /\b(edge|msedge|chrome|chrom(e|ium)|firefox)\b/i.test(request); return [ 'You are the IQ Exchange Translation Layer.', 'Translate the USER REQUEST into executable commands.', '', 'You may output Desktop automation, Browser automation, and/or Server/Shell commands.', '', looksWeb ? 'IMPORTANT: This is a WEB task. Use ONLY BROWSER (Playwright bridge) commands. Do NOT use Desktop (input.ps1) to open Edge/Chrome or type into the address bar.' : 'IMPORTANT: If the request is about websites/URLs/search, use ONLY BROWSER (Playwright bridge) commands (not Desktop).', '', 'DESKTOP (Windows UIAutomation via input.ps1):', `powershell -NoProfile -ExecutionPolicy Bypass -File "${SYSTEM_PATHS.inputPs1}" `, 'Verbs: open, waitfor, app_state, ocr, screenshot, focus, uiclick, uipress, type, key, hotkey, startmenu, mouse, click, drag', 'Tip: Use `startmenu` for opening the Windows Start menu (more reliable than `key LWIN`).', '', 'BROWSER (Playwright bridge):', `node "${SYSTEM_PATHS.playwrightBridge}" `, 'Commands: navigate , click , fill , press , type , url, title, content, elements, screenshot , wait , close', '', 'SERVER/SHELL:', '- Prefer safe read-only checks (status/logs) before restarts/deploys.', '- Use powershell/cmd as appropriate. Keep commands explicit.', '', 'RULES:', '- Output ONLY one fenced code block.', '- One command per line.', '- Use the absolute paths exactly as shown in the templates above.', '- Include verification steps after actions when possible (app_state/ocr, url/title, health checks).', '- Prefer uiclick/uipress over mouse/click unless unavoidable.', '', `USER REQUEST: ${userRequest}` ].join('\n'); } function buildStrictTranslationPrompt(userRequest, previousResponse = '') { const tail = String(previousResponse || '').slice(0, 800); return [ buildTranslationPrompt(userRequest), '', 'STRICT MODE (must follow):', '- Output EXACTLY one fenced code block and nothing else.', '- One command per line; no numbering; no commentary.', '- If no action is possible, output a single comment line explaining why (inside the code block).', '', tail ? `Previous (bad) output (for debugging):\n${tail}` : '' ].filter(Boolean).join('\n'); } /** * Main IQ Exchange Class - The Universal Self-Healing Brain */ export class IQExchange { constructor(options = {}) { this.maxRetries = options.maxRetries || 5; this.sendToAI = options.sendToAI; // Required: async function that sends text to AI and gets response // Callbacks this.onTaskDetected = options.onTaskDetected || (() => { }); this.onExecuting = options.onExecuting || (() => { }); this.onResult = options.onResult || (() => { }); this.onRetrying = options.onRetrying || (() => { }); this.onComplete = options.onComplete || (() => { }); this.onGiveUp = options.onGiveUp || (() => { }); } /** * Translate a generic user request into robust executable commands * This acts as the "Translation Layer" */ async translateRequest(userRequest) { const prompt = buildTranslationPrompt(userRequest); const response = await this.sendToAI(prompt); const extracted = sanitizeExtractedExecutables(extractExecutables(response)); if (extracted.length > 0) return extracted; const fallback = fallbackExtractCommandsFromText(response); if (fallback.length > 0) { return sanitizeExtractedExecutables(fallback.map((cmd) => ({ type: 'command', content: cmd, lang: '' }))); } // Retry once in strict mode (models sometimes ignore formatting rules on first pass) const strictResponse = await this.sendToAI(buildStrictTranslationPrompt(userRequest, response)); const strictExtracted = sanitizeExtractedExecutables(extractExecutables(strictResponse)); if (strictExtracted.length > 0) return strictExtracted; const strictFallback = fallbackExtractCommandsFromText(strictResponse); return sanitizeExtractedExecutables(strictFallback.map((cmd) => ({ type: 'command', content: cmd, lang: '' }))); const legacyPrompt = ` ═══════════════════════════════════════════════════════════════════════════════════ AVAILABLE TOOLS (WINDOWS AUTOMATION): ═══════════════════════════════════════════════════════════════════════════════════ Use the following commands to automate the computer. All commands are run via PowerShell using 'bin/input.ps1'. ► VISION & CONTEXT (The Eyes) • app_state "App Name" -> Structural Vision: Dumps the specific UI tree (buttons, inputs) of a window. • ocr "region" -> Textual Vision: READS all text on screen. Use this to find text you can't click. • screenshot "file.png" -> Visual Vision: Captures the screen state. ► NAVIGATION & STATE • open "App Name" -> Launches or focuses an app (e.g. open "Notepad", open "Spotify") • waitfor "Text" 10 -> Waits up to 10s for text/element to appear. CRITICAL for reliability. • focus "Element Name" -> Focuses a specific element. ► INTERACTION (Robust UIA Hooks) • uiclick "Button Name" -> Clicks a button/text by name (Reliable). • uipress "Item Name" -> Toggles checkboxes, Selects list items, Expands tree items. • type "Text to type" -> Types text into the focused element. • key "Enter" -> Presses a key (Enter, Tab, Esc, Backspace, Delete). • hotkey "Ctrl+C" -> Presses a key combination. ► FALLBACK (Blind Mouse/Coord) • mouse x y -> Moves mouse to coordinates. • click -> Clicks current mouse position. ═══════════════════════════════════════════════════════════════════════════════════ INSTRUCTIONS: 1. Think step-by-step about how to accomplish the User Request. 2. Use 'app_state' or 'ocr' if you need to "see" what is on screen first. 3. Use 'waitfor' to ensure the app is ready before interacting. 4. Use 'uiclick' instead of 'mouse' whenever possible. 5. Output the commands in a single code block. USER REQUEST: "${userRequest}" ═══════════════════════════════════════════════════════════════════════════════════ Expected Output Format: \`\`\`powershell powershell bin/input.ps1 open "Notepad" powershell bin/input.ps1 waitfor "Untitled" 5 powershell bin/input.ps1 type "Hello World" \`\`\` `.trim(); const legacyResponse = await this.sendToAI(legacyPrompt); return extractExecutables(legacyResponse); } /** * Process a user request with self-healing */ async process(userRequest, aiResponse) { // 1. Detect task type const taskTypes = detectTaskType(userRequest); this.onTaskDetected(taskTypes); // 2. Extract executables from AI response const executables = extractExecutables(aiResponse); if (executables.length === 0) { // No commands to execute - just a text response return { type: 'text', response: aiResponse }; } // 3. Execute with self-healing loop const history = []; let retryCount = 0; let currentExecutables = executables; while (retryCount < this.maxRetries) { let allSucceeded = true; for (const exec of currentExecutables) { if (exec.type === 'command') { this.onExecuting(exec.content); const result = await executeAny(exec.content); history.push(result); this.onResult(result); if (detectError(result)) { allSucceeded = false; // Ask AI to fix retryCount++; this.onRetrying({ attempt: retryCount, error: result.stderr || result.error }); const healingPrompt = buildHealingPrompt( userRequest, history, result.stderr || result.error || result.stdout ); const correctedResponse = await this.sendToAI(healingPrompt); if (isComplete(correctedResponse)) { return { type: 'complete', history, retries: retryCount }; } currentExecutables = extractExecutables(correctedResponse); break; // Restart with new commands } } } if (allSucceeded) { this.onComplete({ history, retries: retryCount }); return { type: 'complete', history, retries: retryCount }; } } // Max retries reached this.onGiveUp({ history, retries: retryCount }); return { type: 'failed', history, retries: retryCount }; } } export default { IQExchange, detectTaskType, executeAny, extractExecutables, isComplete, detectError, buildHealingPrompt, SYSTEM_PATHS };