feat: add 'uiclick' smart action and visibility filters to input.ps1
This commit is contained in:
@@ -99,27 +99,73 @@ switch ($Command.ToLower()) {
|
||||
if ($Params.Count -lt 1) { Write-Error "Usage: find 'Name'"; exit 1 }
|
||||
$targetName = $Params -join " "
|
||||
|
||||
Write-Host "Searching for UI Element: '$targetName'..."
|
||||
Write-Host "Searching for VISIBLE UI Element: '$targetName'..."
|
||||
|
||||
$root = [System.Windows.Automation.AutomationElement]::RootElement
|
||||
$cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $targetName)
|
||||
|
||||
# Try finding directly (fast)
|
||||
$element = $root.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $cond)
|
||||
# Find ALL matches, then filter for visibility (to avoid phantom offscreen elements)
|
||||
$collection = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond)
|
||||
$found = $false
|
||||
|
||||
if ($element) {
|
||||
$rect = $element.Current.BoundingRectangle
|
||||
$centerX = [int]($rect.X + ($rect.Width / 2))
|
||||
$centerY = [int]($rect.Y + ($rect.Height / 2))
|
||||
Write-Host "Found '$targetName' at ($centerX, $centerY)"
|
||||
if ($collection) {
|
||||
foreach ($element in $collection) {
|
||||
try {
|
||||
if (-not $element.Current.IsOffscreen) {
|
||||
$rect = $element.Current.BoundingRectangle
|
||||
if ($rect.Width -gt 0 -and $rect.Height -gt 0) {
|
||||
$centerX = [int]($rect.X + ($rect.Width / 2))
|
||||
$centerY = [int]($rect.Y + ($rect.Height / 2))
|
||||
Write-Host "Found Visible '$targetName' at ($centerX, $centerY)"
|
||||
Write-Host "COORD:$centerX,$centerY"
|
||||
$found = $true
|
||||
break # Stop at first visible match
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
# Auto-selection support return format
|
||||
Write-Host "COORD:$centerX,$centerY"
|
||||
} else {
|
||||
if (-not $found) {
|
||||
Write-Host "Element '$targetName' not found visible on desktop."
|
||||
}
|
||||
}
|
||||
|
||||
"uiclick" {
|
||||
if ($Params.Count -lt 1) { Write-Error "Usage: uiclick 'Name'"; exit 1 }
|
||||
$targetName = $Params -join " "
|
||||
Write-Host "Searching & Clicking: '$targetName'..."
|
||||
|
||||
$root = [System.Windows.Automation.AutomationElement]::RootElement
|
||||
$cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $targetName)
|
||||
$collection = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond)
|
||||
|
||||
$found = $false
|
||||
foreach ($element in $collection) {
|
||||
try {
|
||||
if (-not $element.Current.IsOffscreen) {
|
||||
$rect = $element.Current.BoundingRectangle
|
||||
if ($rect.Width -gt 0) {
|
||||
$centerX = [int]($rect.X + ($rect.Width / 2))
|
||||
$centerY = [int]($rect.Y + ($rect.Height / 2))
|
||||
|
||||
# Move & Click
|
||||
[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point($centerX, $centerY)
|
||||
Start-Sleep -Milliseconds 100
|
||||
[Win32]::mouse_event([Win32]::MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0)
|
||||
[Win32]::mouse_event([Win32]::MOUSEEVENTF_LEFTUP, 0, 0, 0, 0)
|
||||
|
||||
Write-Host "Clicked '$targetName' at ($centerX, $centerY)"
|
||||
$found = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
if (-not $found) { Write-Host "Could not find visible '$targetName' to click." }
|
||||
}
|
||||
|
||||
"apps" {
|
||||
$apps = Get-Process | Where-Object { $_.MainWindowTitle -ne "" } | Select-Object Id, MainWindowTitle
|
||||
if ($apps) {
|
||||
|
||||
@@ -491,20 +491,16 @@ Use it to control the mouse, keyboard, and "see" the system.
|
||||
|
||||
## 👁️ VISION & BLINDNESS PROTOCOL:
|
||||
You are a TEXT-BASED intelligence. You CANNOT see images/screenshots you take.
|
||||
- **\`input.ps1 find "Name"\`**: **TRUE VISION**. Finds a UI element (button/window) by text and tells you where it is.
|
||||
- **\`input.ps1 apps\`**: Your "Eyes" for windows. Returns TEXT list of open apps.
|
||||
- **\`input.ps1 screen\`**: Your "Eyes" for geometry. Returns TEXT resolution.
|
||||
- **\`input.ps1 uiclick "Name"\`**: **SMART ACTION**. Finds a VISIBLE button by name and clicks it automatically.
|
||||
- **\`input.ps1 find "Name"\`**: Looks for VISIBLE elements only. Returns coordinates.
|
||||
- **\`input.ps1 apps\`**: TEXT list of open apps.
|
||||
|
||||
### 📐 THE LAW OF ACCURACY:
|
||||
1. **FIND FIRST**: If you need to click a button, SEARCH FOR IT.
|
||||
- \`powershell bin/input.ps1 find "Start"\` -> Returns "Found at (30, 1190)".
|
||||
- **THEN** use those coordinates to click.
|
||||
2. **FALLBACK**: Only calculate coordinates manually if \`find\` fails.
|
||||
3. **Start Menu Logic:** Bottom-Left corner.
|
||||
- X = 0 to 50
|
||||
- Y = Height - 10 (e.g. 1190).
|
||||
- *Target: 30, 1190* (NOT 1020).
|
||||
4. **THEN** Click.
|
||||
### 📐 THE LAW OF ACTION:
|
||||
1. **SMART CLICK FIRST**: To click a named thing (Start, File, Edit), use:
|
||||
\`powershell bin/input.ps1 uiclick "Start"\`
|
||||
*This filters out invisible phantom buttons.*
|
||||
2. **COORDINATES SECOND**: If \`uiclick\` fails, use \`find\` to get coords, then \`mouse\` + \`click\`.
|
||||
3. **SHORTCUTS**: \`key LWIN\` is still the fastest way to open Start.
|
||||
|
||||
### ⚡ SHORTCUTS > MOUSE:
|
||||
Always prefer \`key LWIN\` over clicking. It works on ANY resolution.
|
||||
|
||||
Reference in New Issue
Block a user