diff --git a/README.md b/README.md index 36ba3fe..af6bf66 100644 --- a/README.md +++ b/README.md @@ -631,6 +631,18 @@ data: [DONE] ## Changelog +### v3.1.0 (2026-05-20) +- **AutoGLM Device Control** — `AccessibilityService` for full device automation: tap, swipe, long press, type text, screenshot, UI tree +- **Navigation Controls** — press Back, Home, Recents, Notifications, Quick Settings, Power Dialog via global actions +- **App Control** — launch apps, get current foreground app, read full accessibility node tree with bounds/text/clickable state +- **Click by Text/ID** — `clickByText()` and `clickNode()` for semantic UI interaction without coordinates +- **Hermes Agent** — `installHermes()` creates Python venv and installs hermes-agent via pip; `hermesExec()` runs Hermes commands +- **AI Tool Tags** — `[DEVICE_*]` (17 tags) and `[HERMES_*]` (2 tags) parsed in both coding and agentic modes +- **AutoGLMPlugin** — Capacitor plugin exposing all device control methods to JavaScript +- **Accessibility Setup UI** — one-tap enable button in Dev Setup screen with live status indicator +- **Hermes Install UI** — install button in Dev Setup screen +- **v2 APK Signing** — release APK signed with APK Signature Scheme v2 (required for targetSdk 36) + ### v2.3.0 (2026-05-20) - **Java Virtual Environment** — `ecj.jar` (3.2MB) and `apksigner.jar` (1.1MB) bundled as APK assets, extracted at runtime - **app_process Wrappers** — wrapper scripts use `/system/bin/app_process` to run JARs (bypasses SELinux `execve` restrictions entirely) diff --git a/android/app/build.gradle b/android/app/build.gradle index b3fdf95..2d0ff6a 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -7,8 +7,8 @@ android { applicationId "ai.z.chat" minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion - versionCode 19 - versionName "2.3.0" + versionCode 20 + versionName "3.1.0" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" aaptOptions { ignoreAssetsPattern = '!.svn:!.git:!.ds_store:!*.scc:.*:!CVS:!thumbs.db:!picasa.ini:!*~' diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index aaaa5f5..ef879aa 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -7,6 +7,8 @@ + + + + + + + + + = Build.VERSION_CODES.R) { + takeScreenshot(android.view.Display.DEFAULT_DISPLAY, + getMainExecutor(), new TakeScreenshotCallback() { + @Override + public void onSuccess(ScreenshotResult screenshot) { + try { + Bitmap bitmap = Bitmap.wrapHardwareBuffer( + screenshot.getHardwareBuffer(), screenshot.getColorSpace()); + if (bitmap != null) { + Bitmap softwareBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, false); + FileOutputStream fos = new FileOutputStream(destPath); + softwareBitmap.compress(Bitmap.CompressFormat.PNG, 90, fos); + fos.close(); + softwareBitmap.recycle(); + } + screenshot.getHardwareBuffer().close(); + } catch (Exception e) { + Log.e(TAG, "Screenshot save failed", e); + } + } + @Override + public void onFailure(int errorCode) { + Log.e(TAG, "Screenshot failed: " + errorCode); + } + }); + } + } + + public String getUITree() { + try { + JSONObject root = new JSONObject(); + JSONArray windowsArr = new JSONArray(); + + List windows = getWindows(); + for (AccessibilityWindowInfo window : windows) { + JSONObject winObj = new JSONObject(); + winObj.put("id", window.getId()); + winObj.put("layer", window.getLayer()); + winObj.put("active", window.isActive()); + winObj.put("focused", window.isFocused()); + + AccessibilityNodeInfo rootNode = window.getRoot(); + if (rootNode != null) { + winObj.put("root", serializeNode(rootNode)); + rootNode.recycle(); + } + windowsArr.put(winObj); + } + + root.put("windows", windowsArr); + root.put("windowCount", windows.size()); + return root.toString(); + } catch (Exception e) { + Log.e(TAG, "getUITree failed", e); + return "{}"; + } + } + + public String getFocusedNodeInfo() { + try { + AccessibilityNodeInfo focusNode = findFocusNode(); + if (focusNode != null) { + JSONObject obj = serializeNode(focusNode); + focusNode.recycle(); + return obj.toString(); + } + } catch (Exception e) {} + return "{}"; + } + + public boolean clickNode(String viewId) { + List nodes = findNodesByViewId(viewId); + for (AccessibilityNodeInfo node : nodes) { + if (node.isClickable()) { + node.performAction(AccessibilityNodeInfo.ACTION_CLICK); + node.recycle(); + return true; + } + AccessibilityNodeInfo clickable = findClickableAncestor(node); + if (clickable != null) { + clickable.performAction(AccessibilityNodeInfo.ACTION_CLICK); + clickable.recycle(); + node.recycle(); + return true; + } + node.recycle(); + } + return false; + } + + public boolean clickNodeByText(String text) { + List nodes = findNodesByText(text); + for (AccessibilityNodeInfo node : nodes) { + if (node.isClickable()) { + node.performAction(AccessibilityNodeInfo.ACTION_CLICK); + node.recycle(); + return true; + } + AccessibilityNodeInfo clickable = findClickableAncestor(node); + if (clickable != null) { + clickable.performAction(AccessibilityNodeInfo.ACTION_CLICK); + clickable.recycle(); + node.recycle(); + return true; + } + node.recycle(); + } + return false; + } + + public boolean scrollNode(String viewId, int direction) { + List nodes = findNodesByViewId(viewId); + for (AccessibilityNodeInfo node : nodes) { + boolean result = node.performAction(direction); + node.recycle(); + if (result) return true; + } + return false; + } + + public String getCurrentApp() { + try { + List windows = getWindows(); + for (AccessibilityWindowInfo window : windows) { + if (window.isActive()) { + AccessibilityNodeInfo root = window.getRoot(); + if (root != null) { + CharSequence pkg = root.getPackageName(); + root.recycle(); + return pkg != null ? pkg.toString() : ""; + } + } + } + } catch (Exception e) {} + return ""; + } + + private AccessibilityNodeInfo findFocusNode() { + AccessibilityNodeInfo root = getRootInActiveWindow(); + if (root == null) return null; + + AccessibilityNodeInfo focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT); + root.recycle(); + return focused; + } + + private List findNodesByViewId(String viewId) { + AccessibilityNodeInfo root = getRootInActiveWindow(); + List result = new ArrayList<>(); + if (root != null) { + result = root.findAccessibilityNodeInfosByViewId(viewId); + root.recycle(); + } + return result; + } + + private List findNodesByText(String text) { + AccessibilityNodeInfo root = getRootInActiveWindow(); + List result = new ArrayList<>(); + if (root != null) { + result = root.findAccessibilityNodeInfosByText(text); + root.recycle(); + } + return result; + } + + private AccessibilityNodeInfo findClickableAncestor(AccessibilityNodeInfo node) { + AccessibilityNodeInfo parent = node.getParent(); + while (parent != null) { + if (parent.isClickable()) return parent; + AccessibilityNodeInfo grandParent = parent.getParent(); + if (grandParent == null) { + parent.recycle(); + return null; + } + parent.recycle(); + parent = grandParent; + } + return null; + } + + private JSONObject serializeNode(AccessibilityNodeInfo node) throws Exception { + JSONObject obj = new JSONObject(); + obj.put("className", safeStr(node.getClassName())); + obj.put("text", safeStr(node.getText())); + obj.put("contentDesc", safeStr(node.getContentDescription())); + obj.put("viewId", safeStr(node.getViewIdResourceName())); + obj.put("packageName", safeStr(node.getPackageName())); + obj.put("clickable", node.isClickable()); + obj.put("focusable", node.isFocusable()); + obj.put("editable", node.isEditable()); + obj.put("enabled", node.isEnabled()); + obj.put("checked", node.isChecked()); + obj.put("selected", node.isSelected()); + obj.put("scrollable", node.isScrollable()); + + Rect bounds = new Rect(); + node.getBoundsInScreen(bounds); + JSONObject boundsObj = new JSONObject(); + boundsObj.put("left", bounds.left); + boundsObj.put("top", bounds.top); + boundsObj.put("right", bounds.right); + boundsObj.put("bottom", bounds.bottom); + obj.put("bounds", boundsObj); + + int childCount = node.getChildCount(); + if (childCount > 0) { + JSONArray children = new JSONArray(); + for (int i = 0; i < Math.min(childCount, 50); i++) { + AccessibilityNodeInfo child = node.getChild(i); + if (child != null) { + children.put(serializeNode(child)); + child.recycle(); + } + } + obj.put("children", children); + } + + return obj; + } + + private String safeStr(CharSequence cs) { + return cs != null ? cs.toString() : ""; + } + + private GestureDescription buildClick(int x, int y, long duration) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return null; + Path path = new Path(); + path.moveTo(x, y); + GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0, duration); + return new GestureDescription.Builder().addStroke(stroke).build(); + } + + private GestureDescription buildSwipe(int startX, int startY, int endX, int endY, long duration) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return null; + Path path = new Path(); + path.moveTo(startX, startY); + path.lineTo(endX, endY); + GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0, duration); + return new GestureDescription.Builder().addStroke(stroke).build(); + } +} diff --git a/android/app/src/main/java/ai/z/chat/BootstrapPlugin.java b/android/app/src/main/java/ai/z/chat/BootstrapPlugin.java index e27c8f5..6ff6ea8 100644 --- a/android/app/src/main/java/ai/z/chat/BootstrapPlugin.java +++ b/android/app/src/main/java/ai/z/chat/BootstrapPlugin.java @@ -558,6 +558,130 @@ public class BootstrapPlugin extends Plugin { notifyListeners("bootstrap-progress", event); } + @PluginMethod + public void installHermes(PluginCall call) { + call.setKeepAlive(true); + new Thread(() -> { + try { + String prefix = call.getString("prefix", prefixDir); + String home = call.getString("home", homeDir + "/home"); + String hermesDir = home + "/.hermes"; + String venvDir = home + "/hermes-venv"; + + new File(hermesDir).mkdirs(); + + String pythonBin = prefix + "/bin/python"; + if (!new File(pythonBin).exists()) { + call.reject("Python not installed. Run Termux bootstrap first."); + return; + } + + String hermesLink = venvDir + "/bin/hermes"; + if (new File(hermesLink).exists()) { + call.resolve(new JSObject().put("installed", true).put("path", hermesLink).put("venv", venvDir)); + return; + } + + Log.i(TAG, "Creating Hermes Python venv..."); + ProcessBuilder pb = new ProcessBuilder(pythonBin, "-m", "venv", venvDir); + pb.environment().put("ANDROID_API_LEVEL", String.valueOf(android.os.Build.VERSION.SDK_INT)); + pb.environment().put("HOME", home); + pb.redirectErrorStream(true); + Process p = pb.start(); + drainProcess(p); + p.waitFor(); + + String pipBin = venvDir + "/bin/pip"; + if (!new File(pipBin).exists()) { + call.reject("Failed to create Python venv"); + return; + } + + Log.i(TAG, "Installing hermes-agent..."); + pb = new ProcessBuilder(pipBin, "install", "--upgrade", "pip", "setuptools", "wheel"); + pb.environment().put("ANDROID_API_LEVEL", String.valueOf(android.os.Build.VERSION.SDK_INT)); + pb.environment().put("HOME", home); + pb.redirectErrorStream(true); + p = pb.start(); + drainProcess(p); + p.waitFor(); + + pb = new ProcessBuilder(pipBin, "install", "hermes-agent"); + pb.environment().put("ANDROID_API_LEVEL", String.valueOf(android.os.Build.VERSION.SDK_INT)); + pb.environment().put("HOME", home); + pb.redirectErrorStream(true); + p = pb.start(); + drainProcess(p); + p.waitFor(); + + if (!new File(venvDir + "/bin/hermes").exists()) { + call.reject("hermes-agent installation failed"); + return; + } + + Log.i(TAG, "Hermes agent installed at " + venvDir + "/bin/hermes"); + call.resolve(new JSObject() + .put("installed", true) + .put("path", venvDir + "/bin/hermes") + .put("venv", venvDir) + .put("dir", hermesDir)); + } catch (Exception e) { + Log.e(TAG, "installHermes failed", e); + try { call.reject("installHermes failed: " + e.getMessage()); } catch (Exception ignored) {} + } + }).start(); + } + + @PluginMethod + public void hermesExec(PluginCall call) { + call.setKeepAlive(true); + new Thread(() -> { + try { + String command = call.getString("command", ""); + String home = call.getString("home", homeDir + "/home"); + String venvDir = call.getString("venv", home + "/hermes-venv"); + + if (command.isEmpty()) { + call.reject("command required"); + return; + } + + String hermesBin = venvDir + "/bin/hermes"; + if (!new File(hermesBin).exists()) { + call.reject("Hermes not installed"); + return; + } + + String fullCmd = hermesBin + " " + command; + ProcessBuilder pb = new ProcessBuilder("/system/bin/sh", "-c", fullCmd); + pb.environment().put("HOME", home); + pb.environment().put("PATH", venvDir + "/bin:" + prefixDir + "/bin:/system/bin"); + pb.environment().put("ANDROID_API_LEVEL", String.valueOf(android.os.Build.VERSION.SDK_INT)); + pb.environment().put("TERMUX_HOME", home); + pb.redirectErrorStream(true); + Process p = pb.start(); + String output = drainProcess(p); + int exitCode = p.waitFor(); + + call.resolve(new JSObject() + .put("output", output) + .put("exitCode", exitCode) + .put("ok", exitCode == 0)); + } catch (Exception e) { + try { call.reject("hermesExec failed: " + e.getMessage()); } catch (Exception ignored) {} + } + }).start(); + } + + private String drainProcess(Process p) throws Exception { + java.io.InputStream is = p.getInputStream(); + java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int r; + while ((r = is.read(buf)) > 0) baos.write(buf, 0, r); + return baos.toString("UTF-8"); + } + private String getArch() { String abi = android.os.Build.SUPPORTED_ABIS[0]; switch (abi) { diff --git a/android/app/src/main/java/ai/z/chat/MainActivity.java b/android/app/src/main/java/ai/z/chat/MainActivity.java index 2a8f7c0..cc589be 100644 --- a/android/app/src/main/java/ai/z/chat/MainActivity.java +++ b/android/app/src/main/java/ai/z/chat/MainActivity.java @@ -11,6 +11,7 @@ public class MainActivity extends BridgeActivity { registerPlugin(InstallerPlugin.class); registerPlugin(WakePlugin.class); registerPlugin(BootstrapPlugin.class); + registerPlugin(AutoGLMPlugin.class); super.onCreate(savedInstanceState); } } diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 7036bc8..adb6059 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -4,4 +4,6 @@ Z.AI Chat ai.z.chat ai.z.chat + Allows Z.AI Chat to control your device — tap, swipe, type, and read the screen — for AI-powered automation in coding and agentic modes. + AI device control for agentic automation diff --git a/android/app/src/main/res/xml/accessibility_service_config.xml b/android/app/src/main/res/xml/accessibility_service_config.xml new file mode 100644 index 0000000..95ecfad --- /dev/null +++ b/android/app/src/main/res/xml/accessibility_service_config.xml @@ -0,0 +1,11 @@ + + diff --git a/package.json b/package.json index 6bb2fcb..731893d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "zai-chat", - "version": "2.3.0", + "version": "3.1.0", "description": "Z.AI Chat - Full stack AI chat powered by GLM Coding Plan", "main": "index.js", "scripts": { diff --git a/www/index.html b/www/index.html index 0509e94..a6e4784 100644 --- a/www/index.html +++ b/www/index.html @@ -216,6 +216,17 @@ Install Dev Tools +
+

AutoGLM Device Control

+

Enable accessibility service to allow AI to control your device (tap, swipe, type, read screen).

+
+ +
+
+

Hermes Agent

+

Install Hermes agent for advanced AI capabilities: web search, terminal, skills, memory, browser automation.

+ +
@@ -327,13 +338,26 @@

About

-

Z.AI Chat v2.3.0

+

Z.AI Chat v3.1.0

Built with Z.AI SDK & GLM-5.1

Compatible with Android 15/16

Changelog

    +
  • + v3.1.0 + 2026-05-20 +
      +
    • AutoGLM Device Control — AccessibilityService for tap, swipe, type, long press, screenshot, UI tree, click by text/ID
      • +
    • Navigation Controls — press Back, Home, Recents, Notifications, Quick Settings, Power Dialog
      • +
    • App Control — launch apps, get current app, read full UI hierarchy
      • +
    • Hermes Agent — install hermes-agent in Python venv, execute Hermes commands from chat
      • +
    • Device Tools in AI — [DEVICE_*] and [HERMES_*] tags parsed in coding + agentic modes
      • +
    • Accessibility Setup — one-tap enable in Dev Setup screen with status indicator
      • +
    • v2 Signing — APK signed with v2 scheme (required for targetSdk 36)
      • +
    +
  • v2.3.0 2026-05-20 diff --git a/www/js/app.js b/www/js/app.js index 97e3886..6ac9db1 100644 --- a/www/js/app.js +++ b/www/js/app.js @@ -8,7 +8,7 @@ chat: 'You are a helpful, knowledgeable AI assistant. Be concise and accurate.', coding: 'You are an expert coding assistant. Write clean, efficient, well-documented code. Always use markdown code blocks with language tags. Explain your approach briefly before and after code. Handle edge cases and errors properly.', brainstorm: 'You are a creative brainstorming partner. Generate diverse ideas, explore unconventional angles, build on concepts, and help evaluate trade-offs. Think freely and expansively. Present ideas in organized lists or tables when appropriate.', - agentic: 'You are an autonomous coding agent with FULL control of an Android device. You have a real terminal with shell access, can read/write any file, build APKs, install apps, and execute any command.\n\n## Your Tools (use these EXACT formats):\n\n### Write a file:\n[CREATE_FILE path/to/file.ext]\nfile contents here\n[/CREATE_FILE]\n\n### Run a shell command:\n[RUN_COMMAND]\ncommand here\n[/RUN_COMMAND]\n\n### Build Android APK:\n[BUILD_APK project_name]\n\n### Install APK on device:\n[INSTALL_APK /path/to/file.apk]\n\n### List files:\n[RUN_COMMAND]\nfind . -type f | head -50\n[/RUN_COMMAND]\n\n## IMPORTANT RULES:\n1. ALWAYS use [CREATE_FILE] for EVERY file — the system auto-saves them to the device\n2. ALWAYS use [BUILD_APK] after writing files — the system auto-compiles\n3. ALWAYS use [INSTALL_APK] after building — the system auto-installs\n4. NEVER say "I installed it" unless you used [INSTALL_APK] — the system executes your tags automatically\n5. If a build fails, you will see the error output — FIX the code and try again\n6. Generate COMPLETE files — never use "// ... existing code ..."\n7. For Java: use package ai.z.app, target SDK 36, compile SDK 36\n8. You can run ANY shell command: ls, cat, mkdir, chmod, cp, grep, find, etc.\n9. If Termux tools are available, use: aapt2, d8, ecj, javac, apksigner\n10. Write ALL files first, THEN build, THEN install. Always in that order.\n11. When the user asks to build an app, generate EVERY file needed for a complete working app.\n12. CRITICAL: When you have FULLY completed the user\'s entire task (all files written, all builds done, all installs done), output [TASK_COMPLETE] on a line by itself. This is MANDATORY — the system uses it to know you are done.\n13. If your response is cut off or you haven\'t finished all work, do NOT output [TASK_COMPLETE]. The system will automatically continue you.\n14. NEVER output [TASK_COMPLETE] unless the ENTIRE task is truly done. If there are more files to write, commands to run, or builds to perform, keep working.' + agentic: 'You are an autonomous agent with FULL control of an Android device. You have a real terminal, can build APKs, control the device UI via AutoGLM, and use Hermes agent tools.\n\n## File Operations:\n[CREATE_FILE path/to/file.ext]\ncontents\n[/CREATE_FILE]\n\n## Shell:\n[RUN_COMMAND]\ncommand\n[/RUN_COMMAND]\n\n## Build:\n[BUILD_APK project_name]\n[INSTALL_APK /path/to/file.apk]\n\n## AutoGLM Device Control:\n[DEVICE_TAP x y]\n[DEVICE_LONG_PRESS x y]\n[DEVICE_SWIPE startX startY endX endY]\n[DEVICE_TYPE text]\n[DEVICE_PRESS_BACK]\n[DEVICE_PRESS_HOME]\n[DEVICE_PRESS_RECENTS]\n[DEVICE_SCREENSHOT]\n[DEVICE_UI_TREE]\n[DEVICE_CLICK_TEXT button text]\n[DEVICE_CLICK_ID com.example:id/viewId]\n[DEVICE_LAUNCH com.example.app]\n[DEVICE_CURRENT_APP]\n\n## Hermes Agent:\n[HERMES_INSTALL]\n[HERMES_EXEC command]\n\n## Rules:\n1. Use [CREATE_FILE] for files, [BUILD_APK] to compile, [INSTALL_APK] to install\n2. Use [DEVICE_*] for device control — first [DEVICE_UI_TREE] to see screen, then interact\n3. Use [HERMES_EXEC] for Hermes capabilities — web search, terminal, skills, memory\n4. Generate COMPLETE files, never stubs\n5. For Java: package ai.z.app, target SDK 36\n6. Output [TASK_COMPLETE] ONLY when ALL work is done\n7. Never say done unless all files written, builds done, installs done' }; var BUILD_SCRIPT = [ @@ -1175,6 +1175,9 @@ hasProot: false, prootPath: '', nativeLibDir: '', + hermesPath: '', + hermesVenv: '', + autoglmEnabled: false, commandQueue: [] }; @@ -1423,7 +1426,21 @@ var runCmdRegex = /\[RUN_COMMAND\]\n([\s\S]*?)\[\/RUN_COMMAND\]/gi; var buildApkRegex = /\[BUILD_APK\s+([^\]]+)\]/gi; var installApkRegex = /\[INSTALL_APK\s+([^\]]+)\]/gi; - var codeBlockFileRegex = /```(\w+)\s*\n([\s\S]*?)```/gi; + var deviceTapRegex = /\[DEVICE_TAP\s+(\d+)\s+(\d+)\]/gi; + var deviceLongPressRegex = /\[DEVICE_LONG_PRESS\s+(\d+)\s+(\d+)\]/gi; + var deviceSwipeRegex = /\[DEVICE_SWIPE\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\]/gi; + var deviceTypeRegex = /\[DEVICE_TYPE\s+([^\]]+)\]/gi; + var devicePressBackRegex = /\[DEVICE_PRESS_BACK\]/gi; + var devicePressHomeRegex = /\[DEVICE_PRESS_HOME\]/gi; + var devicePressRecentsRegex = /\[DEVICE_PRESS_RECENTS\]/gi; + var deviceScreenshotRegex = /\[DEVICE_SCREENSHOT\]/gi; + var deviceUiTreeRegex = /\[DEVICE_UI_TREE\]/gi; + var deviceClickTextRegex = /\[DEVICE_CLICK_TEXT\s+([^\]]+)\]/gi; + var deviceClickIdRegex = /\[DEVICE_CLICK_ID\s+([^\]]+)\]/gi; + var deviceLaunchRegex = /\[DEVICE_LAUNCH\s+([^\]]+)\]/gi; + var deviceCurrentAppRegex = /\[DEVICE_CURRENT_APP\]/gi; + var hermesInstallRegex = /\[HERMES_INSTALL\]/gi; + var hermesExecRegex = /\[HERMES_EXEC\s+([^\]]+)\]/gi; var match; while ((match = createActionRegex.exec(content)) !== null) { @@ -1438,6 +1455,51 @@ while ((match = installApkRegex.exec(content)) !== null) { actions.push({ type: 'install_apk', path: match[1].trim() }); } + while ((match = deviceTapRegex.exec(content)) !== null) { + actions.push({ type: 'device_tap', x: parseInt(match[1]), y: parseInt(match[2]) }); + } + while ((match = deviceLongPressRegex.exec(content)) !== null) { + actions.push({ type: 'device_long_press', x: parseInt(match[1]), y: parseInt(match[2]) }); + } + while ((match = deviceSwipeRegex.exec(content)) !== null) { + actions.push({ type: 'device_swipe', startX: parseInt(match[1]), startY: parseInt(match[2]), endX: parseInt(match[3]), endY: parseInt(match[4]) }); + } + while ((match = deviceTypeRegex.exec(content)) !== null) { + actions.push({ type: 'device_type', text: match[1].trim() }); + } + while ((match = devicePressBackRegex.exec(content)) !== null) { + actions.push({ type: 'device_press_back' }); + } + while ((match = devicePressHomeRegex.exec(content)) !== null) { + actions.push({ type: 'device_press_home' }); + } + while ((match = devicePressRecentsRegex.exec(content)) !== null) { + actions.push({ type: 'device_press_recents' }); + } + while ((match = deviceScreenshotRegex.exec(content)) !== null) { + actions.push({ type: 'device_screenshot' }); + } + while ((match = deviceUiTreeRegex.exec(content)) !== null) { + actions.push({ type: 'device_ui_tree' }); + } + while ((match = deviceClickTextRegex.exec(content)) !== null) { + actions.push({ type: 'device_click_text', text: match[1].trim() }); + } + while ((match = deviceClickIdRegex.exec(content)) !== null) { + actions.push({ type: 'device_click_id', viewId: match[1].trim() }); + } + while ((match = deviceLaunchRegex.exec(content)) !== null) { + actions.push({ type: 'device_launch', pkg: match[1].trim() }); + } + while ((match = deviceCurrentAppRegex.exec(content)) !== null) { + actions.push({ type: 'device_current_app' }); + } + while ((match = hermesInstallRegex.exec(content)) !== null) { + actions.push({ type: 'hermes_install' }); + } + while ((match = hermesExecRegex.exec(content)) !== null) { + actions.push({ type: 'hermes_exec', command: match[1].trim() }); + } while ((match = codeBlockFileRegex.exec(content)) !== null) { var lang = match[1]; var code = match[2]; @@ -1457,6 +1519,8 @@ var hasCommands = actions.some(function(a) { return a.type === 'run_command'; }); var hasBuild = actions.some(function(a) { return a.type === 'build_apk'; }); var hasInstall = actions.some(function(a) { return a.type === 'install_apk'; }); + var hasDevice = actions.some(function(a) { return a.type && a.type.indexOf('device_') === 0; }); + var hasHermes = actions.some(function(a) { return a.type && a.type.indexOf('hermes_') === 0; }); var actionBar = document.createElement('div'); actionBar.className = 'msg-actions'; @@ -1606,7 +1670,9 @@ var hasAction = content.indexOf('[CREATE_FILE') >= 0 || content.indexOf('[RUN_COMMAND]') >= 0 || content.indexOf('[BUILD_APK') >= 0 || - content.indexOf('[INSTALL_APK') >= 0; + content.indexOf('[INSTALL_APK') >= 0 || + content.indexOf('[DEVICE_') >= 0 || + content.indexOf('[HERMES_') >= 0; var hasCodeBlock = content.indexOf('```') >= 0; if (!hasCodeBlock && !hasAction && content.length < 300) return true; if ((content.match(/```/g) || []).length % 2 !== 0) return false; @@ -2129,6 +2195,24 @@ return false; } + async function checkAutoGLMStatus() { + var AutoGLM = window.Capacitor && window.Capacitor.Plugins && window.Capacitor.Plugins.AutoGLM; + var statusEl = $('#autoglm-status'); + var btn = $('#autoglm-enable-btn'); + if (!AutoGLM || !statusEl) return; + try { + var result = await AutoGLM.isEnabled(); + if (result.enabled) { + statusEl.innerHTML = 'Device control enabled'; + if (btn) { btn.textContent = 'Device Control Active'; btn.disabled = true; } + } else { + statusEl.innerHTML = 'Not enabled — tap to open Settings'; + } + } catch(e) { + statusEl.innerHTML = 'Checking status...'; + } + } + async function checkDevEnvironment() { if (state.currentMode !== 'coding' && state.currentMode !== 'agentic') return; @@ -2225,8 +2309,10 @@ var hasBuild = actions.some(function(a) { return a.type === 'build_apk'; }); var hasInstall = actions.some(function(a) { return a.type === 'install_apk'; }); var hasCommands = actions.some(function(a) { return a.type === 'run_command'; }); + var hasDevice = actions.some(function(a) { return a.type && a.type.indexOf('device_') === 0; }); + var hasHermes = actions.some(function(a) { return a.type && a.type.indexOf('hermes_') === 0; }); - if (!hasFiles && !hasBuild && !hasInstall && !hasCommands) return; + if (!hasFiles && !hasBuild && !hasInstall && !hasCommands && !hasDevice && !hasHermes) return; _agenticRetryCount = 0; var resultLog = []; @@ -2257,6 +2343,36 @@ } } + if (hasDevice) { + for (var d = 0; d < actions.length; d++) { + var act = actions[d]; + if (act.type.indexOf('device_') !== 0) continue; + try { + var devResult = await executeDeviceAction(act); + resultLog.push(devResult); + termPrint(devResult, ''); + } catch(e) { + resultLog.push('DEVICE_ERROR: ' + e.message); + termPrint('[!] Device: ' + e.message, 'err'); + } + } + } + + if (hasHermes) { + for (var h = 0; h < actions.length; h++) { + var hAct = actions[h]; + if (hAct.type.indexOf('hermes_') !== 0) continue; + try { + var hermesResult = await executeHermesAction(hAct); + resultLog.push(hermesResult); + termPrint(hermesResult, ''); + } catch(e) { + resultLog.push('HERMES_ERROR: ' + e.message); + termPrint('[!] Hermes: ' + e.message, 'err'); + } + } + } + if (hasBuild) { showStatusToast('Building APK...', 'info'); var toolsReady = await ensureBuildTools(); @@ -2285,6 +2401,87 @@ } } + async function executeDeviceAction(action) { + var AutoGLM = window.Capacitor && window.Capacitor.Plugins && window.Capacitor.Plugins.AutoGLM; + if (!AutoGLM) throw new Error('AutoGLM plugin not available'); + + switch (action.type) { + case 'device_tap': + await AutoGLM.tap({ x: action.x, y: action.y }); + return '[DEVICE] Tap(' + action.x + ', ' + action.y + ')'; + case 'device_long_press': + await AutoGLM.longPress({ x: action.x, y: action.y }); + return '[DEVICE] LongPress(' + action.x + ', ' + action.y + ')'; + case 'device_swipe': + await AutoGLM.swipe({ startX: action.startX, startY: action.startY, endX: action.endX, endY: action.endY }); + return '[DEVICE] Swipe(' + action.startX + ',' + action.startY + ' -> ' + action.endX + ',' + action.endY + ')'; + case 'device_type': + await AutoGLM.typeText({ text: action.text }); + return '[DEVICE] Type: "' + action.text + '"'; + case 'device_press_back': + await AutoGLM.pressBack(); + return '[DEVICE] Press Back'; + case 'device_press_home': + await AutoGLM.pressHome(); + return '[DEVICE] Press Home'; + case 'device_press_recents': + await AutoGLM.pressRecents(); + return '[DEVICE] Press Recents'; + case 'device_screenshot': + var ssResult = await AutoGLM.takeScreenshot({}); + return '[DEVICE] Screenshot: ' + (ssResult.path || 'saved'); + case 'device_ui_tree': + var treeResult = await AutoGLM.getUITree({}); + var tree = treeResult.tree || '{}'; + return '[DEVICE] UI Tree: ' + tree.substring(0, 2000); + case 'device_click_text': + var clickResult = await AutoGLM.clickByText({ text: action.text }); + return '[DEVICE] Click "' + action.text + '": ' + (clickResult.ok ? 'OK' : 'not found'); + case 'device_click_id': + var idResult = await AutoGLM.clickNode({ viewId: action.viewId }); + return '[DEVICE] Click ID "' + action.viewId + '": ' + (idResult.ok ? 'OK' : 'not found'); + case 'device_launch': + await AutoGLM.launchApp({ package: action.pkg }); + return '[DEVICE] Launch: ' + action.pkg; + case 'device_current_app': + var appResult = await AutoGLM.getCurrentApp({}); + return '[DEVICE] Current app: ' + (appResult.package || 'unknown'); + default: + return '[DEVICE] Unknown action: ' + action.type; + } + } + + async function executeHermesAction(action) { + var Bootstrap = window.Capacitor && window.Capacitor.Plugins && window.Capacitor.Plugins.Bootstrap; + if (!Bootstrap) throw new Error('Bootstrap plugin not available'); + + switch (action.type) { + case 'hermes_install': + showStatusToast('Installing Hermes agent...', 'info'); + var installResult = await Bootstrap.installHermes({}); + termState.hermesPath = installResult.path; + termState.hermesVenv = installResult.venv; + return '[HERMES] Installed: ' + installResult.path; + case 'hermes_exec': + if (!termState.hermesVenv) { + try { + var status = await Bootstrap.installHermes({}); + termState.hermesPath = status.path; + termState.hermesVenv = status.venv; + } catch(e) { + throw new Error('Hermes not installed: ' + e.message); + } + } + showStatusToast('Hermes: ' + action.command.substring(0, 30), 'info'); + var execResult = await Bootstrap.hermesExec({ command: action.command, venv: termState.hermesVenv }); + var hermesOutput = execResult.output || ''; + if (hermesOutput.length > 3000) hermesOutput = hermesOutput.substring(0, 1500) + '\n...truncated...\n' + hermesOutput.substring(hermesOutput.length - 1000); + return '[HERMES] ' + action.command + '\n' + hermesOutput; + default: + return '[HERMES] Unknown action: ' + action.type; + } + } + async function autoDeployFile(action) { var path = action.path; if (!path.startsWith('/')) { @@ -2644,6 +2841,39 @@ }); } + var autoglmBtn = $('#autoglm-enable-btn'); + if (autoglmBtn) { + checkAutoGLMStatus(); + autoglmBtn.addEventListener('click', function() { + var AutoGLM = window.Capacitor && window.Capacitor.Plugins && window.Capacitor.Plugins.AutoGLM; + if (AutoGLM) { + AutoGLM.openSettings(); + } + }); + } + + var hermesBtn = $('#hermes-install-btn'); + if (hermesBtn) { + hermesBtn.addEventListener('click', async function() { + var Bootstrap = window.Capacitor && window.Capacitor.Plugins && window.Capacitor.Plugins.Bootstrap; + if (!Bootstrap) return; + hermesBtn.disabled = true; + hermesBtn.textContent = 'Installing Hermes...'; + try { + var result = await Bootstrap.installHermes({}); + hermesBtn.textContent = 'Hermes Installed!'; + hermesBtn.disabled = true; + termState.hermesPath = result.path; + termState.hermesVenv = result.venv; + termPrint('[OK] Hermes installed: ' + result.path, 'success'); + } catch(e) { + hermesBtn.textContent = 'Install Failed - Retry'; + hermesBtn.disabled = false; + termPrint('[!] Hermes install failed: ' + e.message, 'err'); + } + }); + } + $$('.term-quick-btn').forEach(function(btn) { btn.addEventListener('click', function() { var cmd = this.dataset.cmd;