diff --git a/bin/input.ps1 b/bin/input.ps1 index ddd8240..1e7f0b3 100644 --- a/bin/input.ps1 +++ b/bin/input.ps1 @@ -99,27 +99,73 @@ switch ($Command.ToLower()) { if ($Params.Count -lt 1) { Write-Error "Usage: find 'Name'"; exit 1 } $targetName = $Params -join " " - Write-Host "Searching for UI Element: '$targetName'..." + Write-Host "Searching for VISIBLE UI Element: '$targetName'..." $root = [System.Windows.Automation.AutomationElement]::RootElement $cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $targetName) - # Try finding directly (fast) - $element = $root.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $cond) + # Find ALL matches, then filter for visibility (to avoid phantom offscreen elements) + $collection = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond) + $found = $false - if ($element) { - $rect = $element.Current.BoundingRectangle - $centerX = [int]($rect.X + ($rect.Width / 2)) - $centerY = [int]($rect.Y + ($rect.Height / 2)) - Write-Host "Found '$targetName' at ($centerX, $centerY)" - - # Auto-selection support return format - Write-Host "COORD:$centerX,$centerY" - } else { + if ($collection) { + foreach ($element in $collection) { + try { + if (-not $element.Current.IsOffscreen) { + $rect = $element.Current.BoundingRectangle + if ($rect.Width -gt 0 -and $rect.Height -gt 0) { + $centerX = [int]($rect.X + ($rect.Width / 2)) + $centerY = [int]($rect.Y + ($rect.Height / 2)) + Write-Host "Found Visible '$targetName' at ($centerX, $centerY)" + Write-Host "COORD:$centerX,$centerY" + $found = $true + break # Stop at first visible match + } + } + } catch {} + } + } + + if (-not $found) { Write-Host "Element '$targetName' not found visible on desktop." } } + "uiclick" { + if ($Params.Count -lt 1) { Write-Error "Usage: uiclick 'Name'"; exit 1 } + $targetName = $Params -join " " + Write-Host "Searching & Clicking: '$targetName'..." + + $root = [System.Windows.Automation.AutomationElement]::RootElement + $cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $targetName) + $collection = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond) + + $found = $false + foreach ($element in $collection) { + try { + if (-not $element.Current.IsOffscreen) { + $rect = $element.Current.BoundingRectangle + if ($rect.Width -gt 0) { + $centerX = [int]($rect.X + ($rect.Width / 2)) + $centerY = [int]($rect.Y + ($rect.Height / 2)) + + # Move & Click + [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point($centerX, $centerY) + Start-Sleep -Milliseconds 100 + [Win32]::mouse_event([Win32]::MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0) + [Win32]::mouse_event([Win32]::MOUSEEVENTF_LEFTUP, 0, 0, 0, 0) + + Write-Host "Clicked '$targetName' at ($centerX, $centerY)" + $found = $true + break + } + } + } catch {} + } + + if (-not $found) { Write-Host "Could not find visible '$targetName' to click." } + } + "apps" { $apps = Get-Process | Where-Object { $_.MainWindowTitle -ne "" } | Select-Object Id, MainWindowTitle if ($apps) { diff --git a/bin/opencode-ink.mjs b/bin/opencode-ink.mjs index 03804ad..0b092a0 100644 --- a/bin/opencode-ink.mjs +++ b/bin/opencode-ink.mjs @@ -491,20 +491,16 @@ Use it to control the mouse, keyboard, and "see" the system. ## 👁️ VISION & BLINDNESS PROTOCOL: You are a TEXT-BASED intelligence. You CANNOT see images/screenshots you take. -- **\`input.ps1 find "Name"\`**: **TRUE VISION**. Finds a UI element (button/window) by text and tells you where it is. -- **\`input.ps1 apps\`**: Your "Eyes" for windows. Returns TEXT list of open apps. -- **\`input.ps1 screen\`**: Your "Eyes" for geometry. Returns TEXT resolution. +- **\`input.ps1 uiclick "Name"\`**: **SMART ACTION**. Finds a VISIBLE button by name and clicks it automatically. +- **\`input.ps1 find "Name"\`**: Looks for VISIBLE elements only. Returns coordinates. +- **\`input.ps1 apps\`**: TEXT list of open apps. -### 📐 THE LAW OF ACCURACY: -1. **FIND FIRST**: If you need to click a button, SEARCH FOR IT. - - \`powershell bin/input.ps1 find "Start"\` -> Returns "Found at (30, 1190)". - - **THEN** use those coordinates to click. -2. **FALLBACK**: Only calculate coordinates manually if \`find\` fails. -3. **Start Menu Logic:** Bottom-Left corner. - - X = 0 to 50 - - Y = Height - 10 (e.g. 1190). - - *Target: 30, 1190* (NOT 1020). -4. **THEN** Click. +### 📐 THE LAW OF ACTION: +1. **SMART CLICK FIRST**: To click a named thing (Start, File, Edit), use: + \`powershell bin/input.ps1 uiclick "Start"\` + *This filters out invisible phantom buttons.* +2. **COORDINATES SECOND**: If \`uiclick\` fails, use \`find\` to get coords, then \`mouse\` + \`click\`. +3. **SHORTCUTS**: \`key LWIN\` is still the fastest way to open Start. ### ⚡ SHORTCUTS > MOUSE: Always prefer \`key LWIN\` over clicking. It works on ANY resolution.