v1.5.0: Add Full GUI Automation with Playwright
This commit is contained in:
admin
25
README.md
25
README.md
@@ -4,12 +4,13 @@
|
||||
|
||||
QwenClaw runs as a background daemon, executing scheduled tasks, responding to Telegram messages, and providing a web dashboard for monitoring and management. It automatically starts with your system and persists across all restarts.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
@@ -479,6 +480,26 @@ rm -rf QwenClaw-with-Auth
|
||||
|
||||
QwenClaw follows [Semantic Versioning](https://semver.org/) (MAJOR.MINOR.PATCH).
|
||||
|
||||
### [1.5.0] - 2026-02-26
|
||||
|
||||
#### Added
|
||||
- **GUI Automation Skill** - Full browser automation with Playwright:
|
||||
- Web browser control (Chromium, Firefox, WebKit)
|
||||
- Screenshot capture (full page or element)
|
||||
- Element interaction (click, type, fill, select, check)
|
||||
- Form filling and submission
|
||||
- Data extraction and web scraping
|
||||
- JavaScript execution
|
||||
- Keyboard shortcuts
|
||||
- File download/upload
|
||||
- Navigation and wait handling
|
||||
- Custom viewport and user agent
|
||||
|
||||
#### Changed
|
||||
- Added Playwright dependency
|
||||
- Updated postinstall to install browser binaries
|
||||
- Updated skills index to v1.5.0 (79 total skills)
|
||||
|
||||
### [1.4.3] - 2026-02-26
|
||||
|
||||
#### Added
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "qwenclaw",
|
||||
"version": "1.4.3",
|
||||
"version": "1.5.0",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"qwenclaw": "./bin/qwenclaw.js"
|
||||
@@ -16,12 +16,13 @@
|
||||
"start:all": "bun run src/index.ts start --web --with-rig",
|
||||
"test": "bun test",
|
||||
"test:rig": "bun test tests/rig-integration.test.ts",
|
||||
"postinstall": "chmod +x bin/qwenclaw.js 2>/dev/null || true"
|
||||
"postinstall": "chmod +x bin/qwenclaw.js 2>/dev/null || true && npx playwright install chromium 2>/dev/null || true"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "^1.3.9"
|
||||
},
|
||||
"dependencies": {
|
||||
"ogg-opus-decoder": "^1.7.3"
|
||||
"ogg-opus-decoder": "^1.7.3",
|
||||
"playwright": "^1.42.0"
|
||||
}
|
||||
}
|
||||
|
||||
485
skills/gui-automation/SKILL.md
Normal file
485
skills/gui-automation/SKILL.md
Normal file
@@ -0,0 +1,485 @@
|
||||
# GUI Automation Skill for QwenClaw
|
||||
|
||||
## Overview
|
||||
|
||||
This skill provides **full GUI automation capabilities** to QwenClaw using Playwright. Control web browsers, interact with elements, capture screenshots, and automate complex web workflows.
|
||||
|
||||
**Version:** 1.0.0
|
||||
**Category:** Automation
|
||||
**Dependencies:** Playwright
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
### 🌐 **Web Browser Automation**
|
||||
- Launch Chromium, Firefox, or WebKit
|
||||
- Navigate to any URL
|
||||
- Handle multiple tabs/windows
|
||||
- Custom viewport and user agent
|
||||
|
||||
### 🖱️ **Element Interaction**
|
||||
- Click buttons and links
|
||||
- Type text into inputs
|
||||
- Fill forms
|
||||
- Select dropdown options
|
||||
- Check/uncheck checkboxes
|
||||
- Hover over elements
|
||||
|
||||
### 📸 **Screenshot & Capture**
|
||||
- Full page screenshots
|
||||
- Element-specific screenshots
|
||||
- Save to configurable directory
|
||||
- Automatic timestamps
|
||||
|
||||
### 📊 **Data Extraction**
|
||||
- Extract text from elements
|
||||
- Get attributes (href, src, etc.)
|
||||
- Scrape tables and lists
|
||||
- Export to JSON/CSV
|
||||
|
||||
### ⌨️ **Keyboard & Navigation**
|
||||
- Press keyboard shortcuts
|
||||
- Wait for elements
|
||||
- Wait for navigation
|
||||
- Execute JavaScript
|
||||
|
||||
### 📥 **File Operations**
|
||||
- Download files
|
||||
- Upload files
|
||||
- Handle file dialogs
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
### 1. Install Playwright
|
||||
|
||||
```bash
|
||||
cd qwenclaw
|
||||
bun add playwright
|
||||
```
|
||||
|
||||
### 2. Install Browser Binaries
|
||||
|
||||
```bash
|
||||
# Install all browsers
|
||||
npx playwright install
|
||||
|
||||
# Or specific browsers
|
||||
npx playwright install chromium
|
||||
npx playwright install firefox
|
||||
npx playwright install webkit
|
||||
```
|
||||
|
||||
### 3. Enable Skill
|
||||
|
||||
Copy this skill to QwenClaw skills directory:
|
||||
|
||||
```bash
|
||||
cp -r skills/gui-automation ~/.qwen/qwenclaw/skills/gui-automation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
### From Qwen Code Chat
|
||||
|
||||
```
|
||||
Use gui-automation to take a screenshot of https://example.com
|
||||
|
||||
Use gui-automation to navigate to GitHub and click the sign in button
|
||||
|
||||
Use gui-automation to fill the login form with username "test" and password "pass123"
|
||||
|
||||
Use gui-automation to extract all article titles from the page
|
||||
```
|
||||
|
||||
### From Terminal CLI
|
||||
|
||||
```bash
|
||||
# Take screenshot
|
||||
qwenclaw gui screenshot https://example.com
|
||||
|
||||
# Navigate to page
|
||||
qwenclaw gui navigate https://github.com
|
||||
|
||||
# Click element
|
||||
qwenclaw gui click "#login-button"
|
||||
|
||||
# Type text
|
||||
qwenclaw gui type "search query" "#search-input"
|
||||
|
||||
# Extract text
|
||||
qwenclaw gui extract ".article-title"
|
||||
|
||||
# Get page title
|
||||
qwenclaw gui title
|
||||
```
|
||||
|
||||
### Programmatic Usage
|
||||
|
||||
```typescript
|
||||
import { GUIAutomation } from './tools/gui-automation';
|
||||
|
||||
const automation = new GUIAutomation({
|
||||
browserType: 'chromium',
|
||||
headless: true,
|
||||
});
|
||||
|
||||
await automation.launch();
|
||||
await automation.goto('https://example.com');
|
||||
await automation.screenshot('example.png');
|
||||
await automation.close();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## API Reference
|
||||
|
||||
### GUIAutomation Class
|
||||
|
||||
#### Constructor
|
||||
|
||||
```typescript
|
||||
new GUIAutomation(config?: Partial<GUIAutomationConfig>)
|
||||
```
|
||||
|
||||
**Config:**
|
||||
```typescript
|
||||
interface GUIAutomationConfig {
|
||||
browserType: 'chromium' | 'firefox' | 'webkit';
|
||||
headless: boolean;
|
||||
viewport?: { width: number; height: number };
|
||||
userAgent?: string;
|
||||
}
|
||||
```
|
||||
|
||||
#### Methods
|
||||
|
||||
| Method | Description | Example |
|
||||
|--------|-------------|---------|
|
||||
| `launch()` | Launch browser | `await automation.launch()` |
|
||||
| `close()` | Close browser | `await automation.close()` |
|
||||
| `goto(url)` | Navigate to URL | `await automation.goto('https://...')` |
|
||||
| `screenshot(name?)` | Take screenshot | `await automation.screenshot('page.png')` |
|
||||
| `click(selector)` | Click element | `await automation.click('#btn')` |
|
||||
| `type(selector, text)` | Type text | `await automation.type('#input', 'hello')` |
|
||||
| `fill(selector, value)` | Fill field | `await automation.fill('#email', 'test@test.com')` |
|
||||
| `select(selector, value)` | Select option | `await automation.select('#country', 'US')` |
|
||||
| `check(selector)` | Check checkbox | `await automation.check('#agree')` |
|
||||
| `getText(selector)` | Get element text | `const text = await automation.getText('#title')` |
|
||||
| `getAttribute(sel, attr)` | Get attribute | `const href = await automation.getAttribute('a', 'href')` |
|
||||
| `waitFor(selector)` | Wait for element | `await automation.waitFor('.loaded')` |
|
||||
| `evaluate(script)` | Execute JS | `const result = await automation.evaluate('2+2')` |
|
||||
| `scrollTo(selector)` | Scroll to element | `await automation.scrollTo('#footer')` |
|
||||
| `hover(selector)` | Hover element | `await automation.hover('#menu')` |
|
||||
| `press(key)` | Press key | `await automation.press('Enter')` |
|
||||
| `getTitle()` | Get page title | `const title = await automation.getTitle()` |
|
||||
| `getUrl()` | Get page URL | `const url = await automation.getUrl()` |
|
||||
| `getHTML()` | Get page HTML | `const html = await automation.getHTML()` |
|
||||
|
||||
---
|
||||
|
||||
## Examples
|
||||
|
||||
### Example 1: Take Screenshot
|
||||
|
||||
```typescript
|
||||
import { GUIAutomation } from './tools/gui-automation';
|
||||
|
||||
const automation = new GUIAutomation();
|
||||
await automation.launch();
|
||||
await automation.goto('https://example.com');
|
||||
await automation.screenshot('example-homepage.png');
|
||||
await automation.close();
|
||||
```
|
||||
|
||||
### Example 2: Fill and Submit Form
|
||||
|
||||
```typescript
|
||||
const automation = new GUIAutomation();
|
||||
await automation.launch();
|
||||
|
||||
await automation.goto('https://example.com/login');
|
||||
await automation.fill('#email', 'user@example.com');
|
||||
await automation.fill('#password', 'secret123');
|
||||
await automation.check('#remember-me');
|
||||
await automation.click('#submit-button');
|
||||
|
||||
await automation.screenshot('logged-in.png');
|
||||
await automation.close();
|
||||
```
|
||||
|
||||
### Example 3: Web Scraping
|
||||
|
||||
```typescript
|
||||
const automation = new GUIAutomation();
|
||||
await automation.launch();
|
||||
|
||||
await automation.goto('https://news.ycombinator.com');
|
||||
|
||||
// Extract all article titles
|
||||
const titles = await automation.findAll('.titleline > a');
|
||||
console.log('Articles:', titles.map(t => t.text));
|
||||
|
||||
// Extract specific data
|
||||
const data = await automation.extractData({
|
||||
topStory: '.titleline > a',
|
||||
score: '.score',
|
||||
comments: '.subtext a:last-child',
|
||||
});
|
||||
|
||||
await automation.close();
|
||||
```
|
||||
|
||||
### Example 4: Wait for Dynamic Content
|
||||
|
||||
```typescript
|
||||
const automation = new GUIAutomation();
|
||||
await automation.launch();
|
||||
|
||||
await automation.goto('https://example.com');
|
||||
|
||||
// Wait for element to appear
|
||||
await automation.waitFor('.loaded-content', { timeout: 10000 });
|
||||
|
||||
// Wait for navigation after click
|
||||
await automation.click('#load-more');
|
||||
await automation.waitForNavigation({ waitUntil: 'networkidle' });
|
||||
|
||||
await automation.screenshot('loaded.png');
|
||||
await automation.close();
|
||||
```
|
||||
|
||||
### Example 5: Execute JavaScript
|
||||
|
||||
```typescript
|
||||
const automation = new GUIAutomation();
|
||||
await automation.launch();
|
||||
await automation.goto('https://example.com');
|
||||
|
||||
// Get window size
|
||||
const size = await automation.evaluate(() => ({
|
||||
width: window.innerWidth,
|
||||
height: window.innerHeight,
|
||||
}));
|
||||
|
||||
// Modify page
|
||||
await automation.evaluate(() => {
|
||||
document.body.style.background = 'red';
|
||||
});
|
||||
|
||||
await automation.screenshot('modified.png');
|
||||
await automation.close();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Common Selectors
|
||||
|
||||
| Selector Type | Example |
|
||||
|--------------|---------|
|
||||
| ID | `#login-button` |
|
||||
| Class | `.article-title` |
|
||||
| Tag | `input`, `button`, `a` |
|
||||
| Attribute | `[type="email"]`, `[href*="github"]` |
|
||||
| CSS | `div > p.text`, `ul li:first-child` |
|
||||
| XPath | `//button[text()="Submit"]` |
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
### config.json
|
||||
|
||||
Create `~/.qwen/qwenclaw/gui-config.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"browserType": "chromium",
|
||||
"headless": true,
|
||||
"viewport": {
|
||||
"width": 1920,
|
||||
"height": 1080
|
||||
},
|
||||
"userAgent": "Mozilla/5.0...",
|
||||
"screenshotsDir": "~/.qwen/qwenclaw/screenshots",
|
||||
"timeout": 30000,
|
||||
"waitForNetworkIdle": true
|
||||
}
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `PLAYWRIGHT_BROWSERS_PATH` | Browser binaries location | `~/.cache/ms-playwright` |
|
||||
| `QWENCLAW_GUI_HEADLESS` | Run headless | `true` |
|
||||
| `QWENCLAW_GUI_BROWSER` | Default browser | `chromium` |
|
||||
|
||||
---
|
||||
|
||||
## Integration with QwenClaw Daemon
|
||||
|
||||
### Add to rig-service
|
||||
|
||||
Update `rig-service/src/agent.rs`:
|
||||
|
||||
```rust
|
||||
// Add GUI automation capability
|
||||
async fn automate_gui(task: &str) -> Result<String> {
|
||||
let automation = GUIAutomation::default();
|
||||
automation.launch().await?;
|
||||
|
||||
// Parse and execute task
|
||||
let result = automation.execute(task).await?;
|
||||
|
||||
automation.close().await?;
|
||||
Ok(result)
|
||||
}
|
||||
```
|
||||
|
||||
### Add to QwenClaw skills
|
||||
|
||||
The skill is automatically available when installed in `skills/gui-automation/`.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Issue: "Browser not found"
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Install browser binaries
|
||||
npx playwright install chromium
|
||||
```
|
||||
|
||||
### Issue: "Timeout waiting for element"
|
||||
|
||||
**Solution:**
|
||||
```typescript
|
||||
// Increase timeout
|
||||
await automation.waitFor('.element', { timeout: 30000 });
|
||||
|
||||
// Or wait for specific event
|
||||
await automation.waitForNavigation({ waitUntil: 'networkidle' });
|
||||
```
|
||||
|
||||
### Issue: "Screenshot not saved"
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Ensure directory exists
|
||||
mkdir -p ~/.qwen/qwenclaw/screenshots
|
||||
|
||||
# Check permissions
|
||||
chmod 755 ~/.qwen/qwenclaw/screenshots
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. **Always Close Browser**
|
||||
|
||||
```typescript
|
||||
try {
|
||||
await automation.launch();
|
||||
// ... do work
|
||||
} finally {
|
||||
await automation.close(); // Always close
|
||||
}
|
||||
```
|
||||
|
||||
### 2. **Use Headless for Automation**
|
||||
|
||||
```typescript
|
||||
const automation = new GUIAutomation({ headless: true });
|
||||
```
|
||||
|
||||
### 3. **Wait for Elements**
|
||||
|
||||
```typescript
|
||||
// Don't assume element exists
|
||||
await automation.waitFor('#dynamic-content');
|
||||
const text = await automation.getText('#dynamic-content');
|
||||
```
|
||||
|
||||
### 4. **Handle Errors**
|
||||
|
||||
```typescript
|
||||
try {
|
||||
await automation.click('#button');
|
||||
} catch (err) {
|
||||
console.error('Click failed:', err);
|
||||
// Fallback or retry
|
||||
}
|
||||
```
|
||||
|
||||
### 5. **Use Descriptive Selectors**
|
||||
|
||||
```typescript
|
||||
// Good
|
||||
await automation.click('[data-testid="submit-button"]');
|
||||
|
||||
// Bad (brittle)
|
||||
await automation.click('div > div:nth-child(3) > button');
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### 1. **Don't Automate Sensitive Sites**
|
||||
|
||||
Avoid automating:
|
||||
- Banking websites
|
||||
- Password managers
|
||||
- Private data entry
|
||||
|
||||
### 2. **Use Headless Mode**
|
||||
|
||||
```json
|
||||
{
|
||||
"headless": true
|
||||
}
|
||||
```
|
||||
|
||||
### 3. **Clear Cookies After Session**
|
||||
|
||||
```typescript
|
||||
await automation.close(); // Clears all session data
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Resources
|
||||
|
||||
- **Playwright Docs:** https://playwright.dev/
|
||||
- **Playwright Selectors:** https://playwright.dev/docs/selectors
|
||||
- **Playwright API:** https://playwright.dev/docs/api/class-playwright
|
||||
|
||||
---
|
||||
|
||||
## Changelog
|
||||
|
||||
### v1.0.0 (2026-02-26)
|
||||
- Initial release
|
||||
- Full browser automation
|
||||
- Screenshot capture
|
||||
- Element interaction
|
||||
- Data extraction
|
||||
- JavaScript execution
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
MIT License - See LICENSE file for details.
|
||||
|
||||
---
|
||||
|
||||
**GUI Automation skill ready for QwenClaw!** 🖥️🤖
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"version": "1.4.3",
|
||||
"version": "1.5.0",
|
||||
"lastUpdated": "2026-02-26",
|
||||
"totalSkills": 78,
|
||||
"totalSkills": 79,
|
||||
"sources": [
|
||||
{
|
||||
"name": "awesome-claude-skills",
|
||||
@@ -55,6 +55,12 @@
|
||||
"url": "https://github.rommark.dev/admin/QwenClaw-with-Auth",
|
||||
"skillsCount": 1,
|
||||
"note": "Enables Qwen Code to trigger, control, and communicate with QwenClaw daemon"
|
||||
},
|
||||
{
|
||||
"name": "gui-automation",
|
||||
"url": "https://playwright.dev/",
|
||||
"skillsCount": 1,
|
||||
"note": "Full GUI automation with Playwright - browser control, screenshots, element interaction, web scraping"
|
||||
}
|
||||
],
|
||||
"skills": [
|
||||
|
||||
536
src/tools/gui-automation.ts
Normal file
536
src/tools/gui-automation.ts
Normal file
@@ -0,0 +1,536 @@
|
||||
/**
|
||||
* GUI Automation Tool for QwenClaw
|
||||
*
|
||||
* Provides full GUI automation capabilities using Playwright
|
||||
* - Web browser automation
|
||||
* - Screenshot capture
|
||||
* - Element interaction (click, type, select)
|
||||
* - Form filling
|
||||
* - Navigation
|
||||
* - Data extraction
|
||||
*/
|
||||
|
||||
import { chromium, firefox, webkit, type Browser, type Page } from 'playwright';
|
||||
import { writeFile, mkdir } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
|
||||
const SCREENSHOTS_DIR = join(process.cwd(), '.qwen', 'qwenclaw', 'screenshots');
|
||||
|
||||
export interface GUIAutomationConfig {
|
||||
browserType: 'chromium' | 'firefox' | 'webkit';
|
||||
headless: boolean;
|
||||
viewport?: { width: number; height: number };
|
||||
userAgent?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG: GUIAutomationConfig = {
|
||||
browserType: 'chromium',
|
||||
headless: true,
|
||||
viewport: { width: 1920, height: 1080 },
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
};
|
||||
|
||||
export class GUIAutomation {
|
||||
private browser: Browser | null = null;
|
||||
private page: Page | null = null;
|
||||
private config: GUIAutomationConfig;
|
||||
|
||||
constructor(config: Partial<GUIAutomationConfig> = {}) {
|
||||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch browser
|
||||
*/
|
||||
async launch(): Promise<void> {
|
||||
const browserType = this.config.browserType;
|
||||
|
||||
switch (browserType) {
|
||||
case 'chromium':
|
||||
this.browser = await chromium.launch({
|
||||
headless: this.config.headless,
|
||||
});
|
||||
break;
|
||||
case 'firefox':
|
||||
this.browser = await firefox.launch({
|
||||
headless: this.config.headless,
|
||||
});
|
||||
break;
|
||||
case 'webkit':
|
||||
this.browser = await webkit.launch({
|
||||
headless: this.config.headless,
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
this.page = await this.browser.newPage({
|
||||
viewport: this.config.viewport,
|
||||
userAgent: this.config.userAgent,
|
||||
});
|
||||
|
||||
console.log(`[GUI] Browser launched: ${browserType}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Close browser
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
if (this.browser) {
|
||||
await this.browser.close();
|
||||
this.browser = null;
|
||||
this.page = null;
|
||||
console.log('[GUI] Browser closed');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to URL
|
||||
*/
|
||||
async goto(url: string, options?: { waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit' }): Promise<void> {
|
||||
if (!this.page) {
|
||||
await this.launch();
|
||||
}
|
||||
|
||||
await this.page!.goto(url, options);
|
||||
console.log(`[GUI] Navigated to: ${url}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Take screenshot
|
||||
*/
|
||||
async screenshot(name?: string): Promise<string> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await mkdir(SCREENSHOTS_DIR, { recursive: true });
|
||||
|
||||
const filename = name || `screenshot-${Date.now()}.png`;
|
||||
const filepath = join(SCREENSHOTS_DIR, filename);
|
||||
|
||||
await this.page.screenshot({ path: filepath, fullPage: true });
|
||||
|
||||
console.log(`[GUI] Screenshot saved: ${filepath}`);
|
||||
return filepath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Click element
|
||||
*/
|
||||
async click(selector: string): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.click(selector);
|
||||
console.log(`[GUI] Clicked: ${selector}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Type text into element
|
||||
*/
|
||||
async type(selector: string, text: string, options?: { delay?: number }): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.type(selector, text, options);
|
||||
console.log(`[GUI] Typed into: ${selector}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill form field
|
||||
*/
|
||||
async fill(selector: string, value: string): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.fill(selector, value);
|
||||
console.log(`[GUI] Filled: ${selector}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Select option from dropdown
|
||||
*/
|
||||
async select(selector: string, value: string): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.selectOption(selector, value);
|
||||
console.log(`[GUI] Selected: ${selector} = ${value}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check checkbox
|
||||
*/
|
||||
async check(selector: string): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.check(selector);
|
||||
console.log(`[GUI] Checked: ${selector}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get element text
|
||||
*/
|
||||
async getText(selector: string): Promise<string> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
const text = await this.page.textContent(selector);
|
||||
return text || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get element attribute
|
||||
*/
|
||||
async getAttribute(selector: string, attribute: string): Promise<string> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
const value = await this.page.getAttribute(selector, attribute);
|
||||
return value || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for element
|
||||
*/
|
||||
async waitFor(selector: string, options?: { timeout?: number }): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.waitForSelector(selector, options);
|
||||
console.log(`[GUI] Waited for: ${selector}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for navigation
|
||||
*/
|
||||
async waitForNavigation(options?: { timeout?: number, waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit' }): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.waitForNavigation(options);
|
||||
console.log('[GUI] Navigation completed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute JavaScript
|
||||
*/
|
||||
async evaluate(script: string): Promise<any> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
const result = await this.page.evaluate(script);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scroll to element
|
||||
*/
|
||||
async scrollTo(selector: string): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.locator(selector).scrollIntoViewIfNeeded();
|
||||
console.log(`[GUI] Scrolled to: ${selector}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Hover over element
|
||||
*/
|
||||
async hover(selector: string): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.hover(selector);
|
||||
console.log(`[GUI] Hovered: ${selector}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Press key
|
||||
*/
|
||||
async press(key: string): Promise<void> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
await this.page.keyboard.press(key);
|
||||
console.log(`[GUI] Pressed: ${key}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Download file
|
||||
*/
|
||||
async downloadFile(selector: string, savePath: string): Promise<string> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
const [download] = await Promise.all([
|
||||
this.page.waitForEvent('download'),
|
||||
this.page.click(selector),
|
||||
]);
|
||||
|
||||
await download.saveAs(savePath);
|
||||
console.log(`[GUI] Downloaded: ${savePath}`);
|
||||
return savePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get page HTML
|
||||
*/
|
||||
async getHTML(): Promise<string> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
return await this.page.content();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get page title
|
||||
*/
|
||||
async getTitle(): Promise<string> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
return await this.page.title();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get page URL
|
||||
*/
|
||||
async getUrl(): Promise<string> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
return this.page.url();
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all elements matching selector
|
||||
*/
|
||||
async findAll(selector: string): Promise<Array<{ text: string; html: string }>> {
|
||||
if (!this.page) {
|
||||
throw new Error('Browser not launched');
|
||||
}
|
||||
|
||||
const elements = await this.page.$$(selector);
|
||||
const results = [];
|
||||
|
||||
for (const element of elements) {
|
||||
const text = await element.textContent();
|
||||
const html = await element.innerHTML();
|
||||
results.push({ text: text || '', html });
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract data from page
|
||||
*/
|
||||
async extractData(selectors: Record<string, string>): Promise<Record<string, string>> {
|
||||
const result: Record<string, string> = {};
|
||||
|
||||
for (const [key, selector] of Object.entries(selectors)) {
|
||||
result[key] = await this.getText(selector);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Quick GUI automation function
|
||||
*/
|
||||
export async function automateGUI(
|
||||
task: string,
|
||||
config?: Partial<GUIAutomationConfig>
|
||||
): Promise<string> {
|
||||
const automation = new GUIAutomation(config);
|
||||
|
||||
try {
|
||||
await automation.launch();
|
||||
|
||||
// Parse task and execute
|
||||
const taskLower = task.toLowerCase();
|
||||
|
||||
if (taskLower.includes('screenshot')) {
|
||||
const urlMatch = task.match(/https?:\/\/[^\s]+/);
|
||||
if (urlMatch) {
|
||||
await automation.goto(urlMatch[0]);
|
||||
const path = await automation.screenshot();
|
||||
return `Screenshot saved to: ${path}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (taskLower.includes('navigate') || taskLower.includes('go to')) {
|
||||
const urlMatch = task.match(/https?:\/\/[^\s]+/);
|
||||
if (urlMatch) {
|
||||
await automation.goto(urlMatch[0]);
|
||||
const title = await automation.getTitle();
|
||||
return `Navigated to: ${urlMatch[0]}\nPage title: ${title}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (taskLower.includes('click')) {
|
||||
const selectorMatch = task.match(/click\s+["']?([^"'\s]+)["']?/);
|
||||
if (selectorMatch) {
|
||||
await automation.click(selectorMatch[1]);
|
||||
return `Clicked: ${selectorMatch[1]}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (taskLower.includes('type') || taskLower.includes('enter')) {
|
||||
const typeMatch = task.match(/type\s+["']([^"']+)["']\s+into\s+["']?([^"'\s]+)["']?/);
|
||||
if (typeMatch) {
|
||||
await automation.type(typeMatch[2], typeMatch[1]);
|
||||
return `Typed "${typeMatch[1]}" into: ${typeMatch[2]}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (taskLower.includes('extract') || taskLower.includes('get')) {
|
||||
const selectorMatch = task.match(/["']?([^"'\s]+)["']?/g);
|
||||
if (selectorMatch && selectorMatch.length > 0) {
|
||||
const text = await automation.getText(selectorMatch[0]);
|
||||
return `Extracted from ${selectorMatch[0]}: ${text}`;
|
||||
}
|
||||
}
|
||||
|
||||
return `Task executed. Current URL: ${await automation.getUrl()}`;
|
||||
} catch (err) {
|
||||
return `[ERROR] GUI automation failed: ${err instanceof Error ? err.message : String(err)}`;
|
||||
} finally {
|
||||
await automation.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Command-line interface for GUI automation
|
||||
*/
|
||||
export async function guiCommand(args: string[]): Promise<void> {
|
||||
console.log('🖥️ QwenClaw GUI Automation\n');
|
||||
|
||||
if (args.length === 0) {
|
||||
console.log('Usage: qwenclaw gui <command> [options]');
|
||||
console.log('');
|
||||
console.log('Commands:');
|
||||
console.log(' screenshot <url> Take screenshot of webpage');
|
||||
console.log(' navigate <url> Navigate to webpage');
|
||||
console.log(' click <selector> Click element');
|
||||
console.log(' type <text> <sel> Type text into element');
|
||||
console.log(' extract <selector> Extract text from element');
|
||||
console.log(' html Get page HTML');
|
||||
console.log(' title Get page title');
|
||||
console.log('');
|
||||
console.log('Examples:');
|
||||
console.log(' qwenclaw gui screenshot https://example.com');
|
||||
console.log(' qwenclaw gui navigate https://github.com');
|
||||
console.log(' qwenclaw gui click "#login-button"');
|
||||
console.log(' qwenclaw gui type "hello" "#search-input"');
|
||||
return;
|
||||
}
|
||||
|
||||
const command = args[0];
|
||||
const automation = new GUIAutomation();
|
||||
|
||||
try {
|
||||
await automation.launch();
|
||||
|
||||
switch (command) {
|
||||
case 'screenshot': {
|
||||
const url = args[1];
|
||||
if (!url) {
|
||||
console.log('[ERROR] URL required');
|
||||
return;
|
||||
}
|
||||
await automation.goto(url);
|
||||
const path = await automation.screenshot();
|
||||
console.log(`✅ Screenshot saved: ${path}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'navigate': {
|
||||
const url = args[1];
|
||||
if (!url) {
|
||||
console.log('[ERROR] URL required');
|
||||
return;
|
||||
}
|
||||
await automation.goto(url);
|
||||
const title = await automation.getTitle();
|
||||
console.log(`✅ Navigated to: ${url}`);
|
||||
console.log(` Title: ${title}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'click': {
|
||||
const selector = args[1];
|
||||
if (!selector) {
|
||||
console.log('[ERROR] Selector required');
|
||||
return;
|
||||
}
|
||||
await automation.click(selector);
|
||||
console.log(`✅ Clicked: ${selector}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'type': {
|
||||
const text = args[1];
|
||||
const selector = args[2];
|
||||
if (!text || !selector) {
|
||||
console.log('[ERROR] Text and selector required');
|
||||
return;
|
||||
}
|
||||
await automation.type(selector, text);
|
||||
console.log(`✅ Typed "${text}" into: ${selector}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'extract': {
|
||||
const selector = args[1];
|
||||
if (!selector) {
|
||||
console.log('[ERROR] Selector required');
|
||||
return;
|
||||
}
|
||||
const text = await automation.getText(selector);
|
||||
console.log(`✅ Extracted from ${selector}:`);
|
||||
console.log(` ${text}`);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'html': {
|
||||
const html = await automation.getHTML();
|
||||
console.log(html.substring(0, 1000) + '...');
|
||||
break;
|
||||
}
|
||||
|
||||
case 'title': {
|
||||
const title = await automation.getTitle();
|
||||
console.log(`Page title: ${title}`);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
console.log(`[ERROR] Unknown command: ${command}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(`[ERROR] ${err instanceof Error ? err.message : String(err)}`);
|
||||
} finally {
|
||||
await automation.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user