v3.1.0: AutoGLM device control + Hermes agent integration for coding/agentic modes
This commit is contained in:
admin
@@ -7,6 +7,8 @@
|
||||
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
|
||||
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
|
||||
<uses-permission android:name="android.permission.WAKE_LOCK" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PLAYBACK" />
|
||||
|
||||
<application
|
||||
android:allowBackup="true"
|
||||
@@ -36,6 +38,19 @@
|
||||
|
||||
</activity>
|
||||
|
||||
<service
|
||||
android:name=".AutoGLMService"
|
||||
android:exported="false"
|
||||
android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
|
||||
android:foregroundServiceType="mediaPlayback">
|
||||
<intent-filter>
|
||||
<action android:name="android.accessibilityservice.AccessibilityService" />
|
||||
</intent-filter>
|
||||
<meta-data
|
||||
android:name="android.accessibilityservice"
|
||||
android:resource="@xml/accessibility_service_config" />
|
||||
</service>
|
||||
|
||||
<provider
|
||||
android:name="androidx.core.content.FileProvider"
|
||||
android:authorities="${applicationId}.fileprovider"
|
||||
|
||||
221
android/app/src/main/java/ai/z/chat/AutoGLMPlugin.java
Normal file
221
android/app/src/main/java/ai/z/chat/AutoGLMPlugin.java
Normal file
@@ -0,0 +1,221 @@
|
||||
package ai.z.chat;
|
||||
|
||||
import com.getcapacitor.JSObject;
|
||||
import com.getcapacitor.Plugin;
|
||||
import com.getcapacitor.PluginCall;
|
||||
import com.getcapacitor.PluginMethod;
|
||||
import com.getcapacitor.annotation.CapacitorPlugin;
|
||||
import com.getcapacitor.annotation.Permission;
|
||||
|
||||
import android.content.Intent;
|
||||
import android.provider.Settings;
|
||||
import android.text.TextUtils;
|
||||
|
||||
@CapacitorPlugin(
|
||||
name = "AutoGLM",
|
||||
permissions = {}
|
||||
)
|
||||
public class AutoGLMPlugin extends Plugin {
|
||||
|
||||
@PluginMethod
|
||||
public void isEnabled(PluginCall call) {
|
||||
call.resolve(new JSObject().put("enabled", AutoGLMService.isEnabled()));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void openSettings(PluginCall call) {
|
||||
try {
|
||||
Intent intent = new Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS);
|
||||
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
|
||||
String label = getContext().getString(android.R.string.ok);
|
||||
intent.putExtra(":settings:fragment_args_key", "ai.z.chat/.AutoGLMService");
|
||||
getContext().startActivity(intent);
|
||||
call.resolve(new JSObject().put("opened", true));
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
Intent intent = new Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS);
|
||||
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
|
||||
getContext().startActivity(intent);
|
||||
call.resolve(new JSObject().put("opened", true));
|
||||
} catch (Exception e2) {
|
||||
call.reject("Cannot open accessibility settings: " + e2.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void tap(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
int x = call.getInt("x", 0);
|
||||
int y = call.getInt("y", 0);
|
||||
svc.tap(x, y);
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void longPress(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
int x = call.getInt("x", 0);
|
||||
int y = call.getInt("y", 0);
|
||||
svc.longPress(x, y);
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void swipe(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
int startX = call.getInt("startX", 0);
|
||||
int startY = call.getInt("startY", 0);
|
||||
int endX = call.getInt("endX", 0);
|
||||
int endY = call.getInt("endY", 0);
|
||||
int duration = call.getInt("duration", 300);
|
||||
svc.swipe(startX, startY, endX, endY, duration);
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void typeText(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
String text = call.getString("text", "");
|
||||
if (text.isEmpty()) { call.reject("text required"); return; }
|
||||
svc.typeText(text);
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void pressBack(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
svc.pressBack();
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void pressHome(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
svc.pressHome();
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void pressRecents(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
svc.pressRecents();
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void pressNotifications(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
svc.pressNotifications();
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void pressQuickSettings(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
svc.pressQuickSettings();
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void takeScreenshot(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
if (android.os.Build.VERSION.SDK_INT < android.os.Build.VERSION_CODES.R) {
|
||||
call.reject("Screenshot requires Android 11+");
|
||||
return;
|
||||
}
|
||||
String destPath = call.getString("dest", "");
|
||||
if (destPath.isEmpty()) {
|
||||
destPath = getContext().getCacheDir() + "/autoglm_screenshot.png";
|
||||
}
|
||||
svc.takeScreenshot(destPath);
|
||||
try { Thread.sleep(500); } catch (Exception e) {}
|
||||
call.resolve(new JSObject().put("path", destPath).put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void getUITree(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
String tree = svc.getUITree();
|
||||
call.resolve(new JSObject().put("tree", tree).put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void getFocusedNode(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
String info = svc.getFocusedNodeInfo();
|
||||
call.resolve(new JSObject().put("node", info).put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void clickNode(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
String viewId = call.getString("viewId", "");
|
||||
if (viewId.isEmpty()) { call.reject("viewId required"); return; }
|
||||
boolean result = svc.clickNode(viewId);
|
||||
call.resolve(new JSObject().put("ok", result));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void clickByText(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
String text = call.getString("text", "");
|
||||
if (text.isEmpty()) { call.reject("text required"); return; }
|
||||
boolean result = svc.clickNodeByText(text);
|
||||
call.resolve(new JSObject().put("ok", result));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void scrollNode(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
String viewId = call.getString("viewId", "");
|
||||
String direction = call.getString("direction", "forward");
|
||||
if (viewId.isEmpty()) { call.reject("viewId required"); return; }
|
||||
int action = direction.equals("forward")
|
||||
? android.view.accessibility.AccessibilityNodeInfo.ACTION_SCROLL_FORWARD
|
||||
: android.view.accessibility.AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD;
|
||||
boolean result = svc.scrollNode(viewId, action);
|
||||
call.resolve(new JSObject().put("ok", result));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void getCurrentApp(PluginCall call) {
|
||||
AutoGLMService svc = AutoGLMService.getInstance();
|
||||
if (svc == null) { call.reject("AutoGLM service not enabled"); return; }
|
||||
String pkg = svc.getCurrentApp();
|
||||
call.resolve(new JSObject().put("package", pkg).put("ok", true));
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void launchApp(PluginCall call) {
|
||||
String pkg = call.getString("package", "");
|
||||
if (pkg.isEmpty()) { call.reject("package required"); return; }
|
||||
try {
|
||||
Intent intent = getContext().getPackageManager().getLaunchIntentForPackage(pkg);
|
||||
if (intent != null) {
|
||||
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
|
||||
getContext().startActivity(intent);
|
||||
call.resolve(new JSObject().put("ok", true));
|
||||
} else {
|
||||
call.reject("Package not found: " + pkg);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
call.reject("Launch failed: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
374
android/app/src/main/java/ai/z/chat/AutoGLMService.java
Normal file
374
android/app/src/main/java/ai/z/chat/AutoGLMService.java
Normal file
@@ -0,0 +1,374 @@
|
||||
package ai.z.chat;
|
||||
|
||||
import android.accessibilityservice.AccessibilityService;
|
||||
import android.accessibilityservice.AccessibilityServiceInfo;
|
||||
import android.accessibilityservice.GestureDescription;
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.Path;
|
||||
import android.graphics.Rect;
|
||||
import android.os.Build;
|
||||
import android.os.Bundle;
|
||||
import android.util.Log;
|
||||
import android.view.accessibility.AccessibilityEvent;
|
||||
import android.view.accessibility.AccessibilityNodeInfo;
|
||||
import android.view.accessibility.AccessibilityWindowInfo;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import android.os.Bundle;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class AutoGLMService extends AccessibilityService {
|
||||
|
||||
private static final String TAG = "AutoGLMService";
|
||||
private static AutoGLMService instance;
|
||||
|
||||
@Override
|
||||
public void onCreate() {
|
||||
super.onCreate();
|
||||
instance = this;
|
||||
Log.i(TAG, "AutoGLM AccessibilityService created");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDestroy() {
|
||||
super.onDestroy();
|
||||
instance = null;
|
||||
Log.i(TAG, "AutoGLM AccessibilityService destroyed");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onAccessibilityEvent(AccessibilityEvent event) {}
|
||||
|
||||
@Override
|
||||
public void onInterrupt() {}
|
||||
|
||||
@Override
|
||||
protected void onServiceConnected() {
|
||||
super.onServiceConnected();
|
||||
instance = this;
|
||||
AccessibilityServiceInfo info = getServiceInfo();
|
||||
if (info == null) info = new AccessibilityServiceInfo();
|
||||
info.eventTypes = AccessibilityEvent.TYPES_ALL_MASK;
|
||||
info.feedbackType = AccessibilityServiceInfo.FEEDBACK_GENERIC;
|
||||
info.flags = AccessibilityServiceInfo.DEFAULT
|
||||
| AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
|
||||
| AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS
|
||||
| AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS;
|
||||
info.notificationTimeout = 100;
|
||||
setServiceInfo(info);
|
||||
Log.i(TAG, "AutoGLM service connected and configured");
|
||||
}
|
||||
|
||||
public static AutoGLMService getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
public static boolean isEnabled() {
|
||||
return instance != null;
|
||||
}
|
||||
|
||||
public void tap(int x, int y) {
|
||||
GestureDescription gesture = buildClick(x, y, 50);
|
||||
if (gesture != null) dispatchGesture(gesture, null, null);
|
||||
}
|
||||
|
||||
public void longPress(int x, int y) {
|
||||
GestureDescription gesture = buildClick(x, y, 500);
|
||||
if (gesture != null) dispatchGesture(gesture, null, null);
|
||||
}
|
||||
|
||||
public void swipe(int startX, int startY, int endX, int endY, int durationMs) {
|
||||
GestureDescription gesture = buildSwipe(startX, startY, endX, endY, durationMs);
|
||||
if (gesture != null) dispatchGesture(gesture, null, null);
|
||||
}
|
||||
|
||||
public void swipe(int startX, int startY, int endX, int endY) {
|
||||
swipe(startX, startY, endX, endY, 300);
|
||||
}
|
||||
|
||||
public void typeText(String text) {
|
||||
AccessibilityNodeInfo focusNode = findFocusNode();
|
||||
if (focusNode != null) {
|
||||
Bundle args = new Bundle();
|
||||
args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
|
||||
focusNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args);
|
||||
focusNode.recycle();
|
||||
}
|
||||
}
|
||||
|
||||
public void pressBack() {
|
||||
performGlobalAction(GLOBAL_ACTION_BACK);
|
||||
}
|
||||
|
||||
public void pressHome() {
|
||||
performGlobalAction(GLOBAL_ACTION_HOME);
|
||||
}
|
||||
|
||||
public void pressRecents() {
|
||||
performGlobalAction(GLOBAL_ACTION_RECENTS);
|
||||
}
|
||||
|
||||
public void pressNotifications() {
|
||||
performGlobalAction(GLOBAL_ACTION_NOTIFICATIONS);
|
||||
}
|
||||
|
||||
public void pressQuickSettings() {
|
||||
performGlobalAction(GLOBAL_ACTION_QUICK_SETTINGS);
|
||||
}
|
||||
|
||||
public void pressPowerDialog() {
|
||||
performGlobalAction(GLOBAL_ACTION_POWER_DIALOG);
|
||||
}
|
||||
|
||||
public void pressLockScreen() {
|
||||
performGlobalAction(GLOBAL_ACTION_LOCK_SCREEN);
|
||||
}
|
||||
|
||||
public void takeScreenshot(String destPath) {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
|
||||
takeScreenshot(android.view.Display.DEFAULT_DISPLAY,
|
||||
getMainExecutor(), new TakeScreenshotCallback() {
|
||||
@Override
|
||||
public void onSuccess(ScreenshotResult screenshot) {
|
||||
try {
|
||||
Bitmap bitmap = Bitmap.wrapHardwareBuffer(
|
||||
screenshot.getHardwareBuffer(), screenshot.getColorSpace());
|
||||
if (bitmap != null) {
|
||||
Bitmap softwareBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, false);
|
||||
FileOutputStream fos = new FileOutputStream(destPath);
|
||||
softwareBitmap.compress(Bitmap.CompressFormat.PNG, 90, fos);
|
||||
fos.close();
|
||||
softwareBitmap.recycle();
|
||||
}
|
||||
screenshot.getHardwareBuffer().close();
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Screenshot save failed", e);
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public void onFailure(int errorCode) {
|
||||
Log.e(TAG, "Screenshot failed: " + errorCode);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public String getUITree() {
|
||||
try {
|
||||
JSONObject root = new JSONObject();
|
||||
JSONArray windowsArr = new JSONArray();
|
||||
|
||||
List<AccessibilityWindowInfo> windows = getWindows();
|
||||
for (AccessibilityWindowInfo window : windows) {
|
||||
JSONObject winObj = new JSONObject();
|
||||
winObj.put("id", window.getId());
|
||||
winObj.put("layer", window.getLayer());
|
||||
winObj.put("active", window.isActive());
|
||||
winObj.put("focused", window.isFocused());
|
||||
|
||||
AccessibilityNodeInfo rootNode = window.getRoot();
|
||||
if (rootNode != null) {
|
||||
winObj.put("root", serializeNode(rootNode));
|
||||
rootNode.recycle();
|
||||
}
|
||||
windowsArr.put(winObj);
|
||||
}
|
||||
|
||||
root.put("windows", windowsArr);
|
||||
root.put("windowCount", windows.size());
|
||||
return root.toString();
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "getUITree failed", e);
|
||||
return "{}";
|
||||
}
|
||||
}
|
||||
|
||||
public String getFocusedNodeInfo() {
|
||||
try {
|
||||
AccessibilityNodeInfo focusNode = findFocusNode();
|
||||
if (focusNode != null) {
|
||||
JSONObject obj = serializeNode(focusNode);
|
||||
focusNode.recycle();
|
||||
return obj.toString();
|
||||
}
|
||||
} catch (Exception e) {}
|
||||
return "{}";
|
||||
}
|
||||
|
||||
public boolean clickNode(String viewId) {
|
||||
List<AccessibilityNodeInfo> nodes = findNodesByViewId(viewId);
|
||||
for (AccessibilityNodeInfo node : nodes) {
|
||||
if (node.isClickable()) {
|
||||
node.performAction(AccessibilityNodeInfo.ACTION_CLICK);
|
||||
node.recycle();
|
||||
return true;
|
||||
}
|
||||
AccessibilityNodeInfo clickable = findClickableAncestor(node);
|
||||
if (clickable != null) {
|
||||
clickable.performAction(AccessibilityNodeInfo.ACTION_CLICK);
|
||||
clickable.recycle();
|
||||
node.recycle();
|
||||
return true;
|
||||
}
|
||||
node.recycle();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean clickNodeByText(String text) {
|
||||
List<AccessibilityNodeInfo> nodes = findNodesByText(text);
|
||||
for (AccessibilityNodeInfo node : nodes) {
|
||||
if (node.isClickable()) {
|
||||
node.performAction(AccessibilityNodeInfo.ACTION_CLICK);
|
||||
node.recycle();
|
||||
return true;
|
||||
}
|
||||
AccessibilityNodeInfo clickable = findClickableAncestor(node);
|
||||
if (clickable != null) {
|
||||
clickable.performAction(AccessibilityNodeInfo.ACTION_CLICK);
|
||||
clickable.recycle();
|
||||
node.recycle();
|
||||
return true;
|
||||
}
|
||||
node.recycle();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean scrollNode(String viewId, int direction) {
|
||||
List<AccessibilityNodeInfo> nodes = findNodesByViewId(viewId);
|
||||
for (AccessibilityNodeInfo node : nodes) {
|
||||
boolean result = node.performAction(direction);
|
||||
node.recycle();
|
||||
if (result) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public String getCurrentApp() {
|
||||
try {
|
||||
List<AccessibilityWindowInfo> windows = getWindows();
|
||||
for (AccessibilityWindowInfo window : windows) {
|
||||
if (window.isActive()) {
|
||||
AccessibilityNodeInfo root = window.getRoot();
|
||||
if (root != null) {
|
||||
CharSequence pkg = root.getPackageName();
|
||||
root.recycle();
|
||||
return pkg != null ? pkg.toString() : "";
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {}
|
||||
return "";
|
||||
}
|
||||
|
||||
private AccessibilityNodeInfo findFocusNode() {
|
||||
AccessibilityNodeInfo root = getRootInActiveWindow();
|
||||
if (root == null) return null;
|
||||
|
||||
AccessibilityNodeInfo focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT);
|
||||
root.recycle();
|
||||
return focused;
|
||||
}
|
||||
|
||||
private List<AccessibilityNodeInfo> findNodesByViewId(String viewId) {
|
||||
AccessibilityNodeInfo root = getRootInActiveWindow();
|
||||
List<AccessibilityNodeInfo> result = new ArrayList<>();
|
||||
if (root != null) {
|
||||
result = root.findAccessibilityNodeInfosByViewId(viewId);
|
||||
root.recycle();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<AccessibilityNodeInfo> findNodesByText(String text) {
|
||||
AccessibilityNodeInfo root = getRootInActiveWindow();
|
||||
List<AccessibilityNodeInfo> result = new ArrayList<>();
|
||||
if (root != null) {
|
||||
result = root.findAccessibilityNodeInfosByText(text);
|
||||
root.recycle();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private AccessibilityNodeInfo findClickableAncestor(AccessibilityNodeInfo node) {
|
||||
AccessibilityNodeInfo parent = node.getParent();
|
||||
while (parent != null) {
|
||||
if (parent.isClickable()) return parent;
|
||||
AccessibilityNodeInfo grandParent = parent.getParent();
|
||||
if (grandParent == null) {
|
||||
parent.recycle();
|
||||
return null;
|
||||
}
|
||||
parent.recycle();
|
||||
parent = grandParent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private JSONObject serializeNode(AccessibilityNodeInfo node) throws Exception {
|
||||
JSONObject obj = new JSONObject();
|
||||
obj.put("className", safeStr(node.getClassName()));
|
||||
obj.put("text", safeStr(node.getText()));
|
||||
obj.put("contentDesc", safeStr(node.getContentDescription()));
|
||||
obj.put("viewId", safeStr(node.getViewIdResourceName()));
|
||||
obj.put("packageName", safeStr(node.getPackageName()));
|
||||
obj.put("clickable", node.isClickable());
|
||||
obj.put("focusable", node.isFocusable());
|
||||
obj.put("editable", node.isEditable());
|
||||
obj.put("enabled", node.isEnabled());
|
||||
obj.put("checked", node.isChecked());
|
||||
obj.put("selected", node.isSelected());
|
||||
obj.put("scrollable", node.isScrollable());
|
||||
|
||||
Rect bounds = new Rect();
|
||||
node.getBoundsInScreen(bounds);
|
||||
JSONObject boundsObj = new JSONObject();
|
||||
boundsObj.put("left", bounds.left);
|
||||
boundsObj.put("top", bounds.top);
|
||||
boundsObj.put("right", bounds.right);
|
||||
boundsObj.put("bottom", bounds.bottom);
|
||||
obj.put("bounds", boundsObj);
|
||||
|
||||
int childCount = node.getChildCount();
|
||||
if (childCount > 0) {
|
||||
JSONArray children = new JSONArray();
|
||||
for (int i = 0; i < Math.min(childCount, 50); i++) {
|
||||
AccessibilityNodeInfo child = node.getChild(i);
|
||||
if (child != null) {
|
||||
children.put(serializeNode(child));
|
||||
child.recycle();
|
||||
}
|
||||
}
|
||||
obj.put("children", children);
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
private String safeStr(CharSequence cs) {
|
||||
return cs != null ? cs.toString() : "";
|
||||
}
|
||||
|
||||
private GestureDescription buildClick(int x, int y, long duration) {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return null;
|
||||
Path path = new Path();
|
||||
path.moveTo(x, y);
|
||||
GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0, duration);
|
||||
return new GestureDescription.Builder().addStroke(stroke).build();
|
||||
}
|
||||
|
||||
private GestureDescription buildSwipe(int startX, int startY, int endX, int endY, long duration) {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return null;
|
||||
Path path = new Path();
|
||||
path.moveTo(startX, startY);
|
||||
path.lineTo(endX, endY);
|
||||
GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0, duration);
|
||||
return new GestureDescription.Builder().addStroke(stroke).build();
|
||||
}
|
||||
}
|
||||
@@ -558,6 +558,130 @@ public class BootstrapPlugin extends Plugin {
|
||||
notifyListeners("bootstrap-progress", event);
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void installHermes(PluginCall call) {
|
||||
call.setKeepAlive(true);
|
||||
new Thread(() -> {
|
||||
try {
|
||||
String prefix = call.getString("prefix", prefixDir);
|
||||
String home = call.getString("home", homeDir + "/home");
|
||||
String hermesDir = home + "/.hermes";
|
||||
String venvDir = home + "/hermes-venv";
|
||||
|
||||
new File(hermesDir).mkdirs();
|
||||
|
||||
String pythonBin = prefix + "/bin/python";
|
||||
if (!new File(pythonBin).exists()) {
|
||||
call.reject("Python not installed. Run Termux bootstrap first.");
|
||||
return;
|
||||
}
|
||||
|
||||
String hermesLink = venvDir + "/bin/hermes";
|
||||
if (new File(hermesLink).exists()) {
|
||||
call.resolve(new JSObject().put("installed", true).put("path", hermesLink).put("venv", venvDir));
|
||||
return;
|
||||
}
|
||||
|
||||
Log.i(TAG, "Creating Hermes Python venv...");
|
||||
ProcessBuilder pb = new ProcessBuilder(pythonBin, "-m", "venv", venvDir);
|
||||
pb.environment().put("ANDROID_API_LEVEL", String.valueOf(android.os.Build.VERSION.SDK_INT));
|
||||
pb.environment().put("HOME", home);
|
||||
pb.redirectErrorStream(true);
|
||||
Process p = pb.start();
|
||||
drainProcess(p);
|
||||
p.waitFor();
|
||||
|
||||
String pipBin = venvDir + "/bin/pip";
|
||||
if (!new File(pipBin).exists()) {
|
||||
call.reject("Failed to create Python venv");
|
||||
return;
|
||||
}
|
||||
|
||||
Log.i(TAG, "Installing hermes-agent...");
|
||||
pb = new ProcessBuilder(pipBin, "install", "--upgrade", "pip", "setuptools", "wheel");
|
||||
pb.environment().put("ANDROID_API_LEVEL", String.valueOf(android.os.Build.VERSION.SDK_INT));
|
||||
pb.environment().put("HOME", home);
|
||||
pb.redirectErrorStream(true);
|
||||
p = pb.start();
|
||||
drainProcess(p);
|
||||
p.waitFor();
|
||||
|
||||
pb = new ProcessBuilder(pipBin, "install", "hermes-agent");
|
||||
pb.environment().put("ANDROID_API_LEVEL", String.valueOf(android.os.Build.VERSION.SDK_INT));
|
||||
pb.environment().put("HOME", home);
|
||||
pb.redirectErrorStream(true);
|
||||
p = pb.start();
|
||||
drainProcess(p);
|
||||
p.waitFor();
|
||||
|
||||
if (!new File(venvDir + "/bin/hermes").exists()) {
|
||||
call.reject("hermes-agent installation failed");
|
||||
return;
|
||||
}
|
||||
|
||||
Log.i(TAG, "Hermes agent installed at " + venvDir + "/bin/hermes");
|
||||
call.resolve(new JSObject()
|
||||
.put("installed", true)
|
||||
.put("path", venvDir + "/bin/hermes")
|
||||
.put("venv", venvDir)
|
||||
.put("dir", hermesDir));
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "installHermes failed", e);
|
||||
try { call.reject("installHermes failed: " + e.getMessage()); } catch (Exception ignored) {}
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void hermesExec(PluginCall call) {
|
||||
call.setKeepAlive(true);
|
||||
new Thread(() -> {
|
||||
try {
|
||||
String command = call.getString("command", "");
|
||||
String home = call.getString("home", homeDir + "/home");
|
||||
String venvDir = call.getString("venv", home + "/hermes-venv");
|
||||
|
||||
if (command.isEmpty()) {
|
||||
call.reject("command required");
|
||||
return;
|
||||
}
|
||||
|
||||
String hermesBin = venvDir + "/bin/hermes";
|
||||
if (!new File(hermesBin).exists()) {
|
||||
call.reject("Hermes not installed");
|
||||
return;
|
||||
}
|
||||
|
||||
String fullCmd = hermesBin + " " + command;
|
||||
ProcessBuilder pb = new ProcessBuilder("/system/bin/sh", "-c", fullCmd);
|
||||
pb.environment().put("HOME", home);
|
||||
pb.environment().put("PATH", venvDir + "/bin:" + prefixDir + "/bin:/system/bin");
|
||||
pb.environment().put("ANDROID_API_LEVEL", String.valueOf(android.os.Build.VERSION.SDK_INT));
|
||||
pb.environment().put("TERMUX_HOME", home);
|
||||
pb.redirectErrorStream(true);
|
||||
Process p = pb.start();
|
||||
String output = drainProcess(p);
|
||||
int exitCode = p.waitFor();
|
||||
|
||||
call.resolve(new JSObject()
|
||||
.put("output", output)
|
||||
.put("exitCode", exitCode)
|
||||
.put("ok", exitCode == 0));
|
||||
} catch (Exception e) {
|
||||
try { call.reject("hermesExec failed: " + e.getMessage()); } catch (Exception ignored) {}
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
|
||||
private String drainProcess(Process p) throws Exception {
|
||||
java.io.InputStream is = p.getInputStream();
|
||||
java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream();
|
||||
byte[] buf = new byte[8192];
|
||||
int r;
|
||||
while ((r = is.read(buf)) > 0) baos.write(buf, 0, r);
|
||||
return baos.toString("UTF-8");
|
||||
}
|
||||
|
||||
private String getArch() {
|
||||
String abi = android.os.Build.SUPPORTED_ABIS[0];
|
||||
switch (abi) {
|
||||
|
||||
@@ -11,6 +11,7 @@ public class MainActivity extends BridgeActivity {
|
||||
registerPlugin(InstallerPlugin.class);
|
||||
registerPlugin(WakePlugin.class);
|
||||
registerPlugin(BootstrapPlugin.class);
|
||||
registerPlugin(AutoGLMPlugin.class);
|
||||
super.onCreate(savedInstanceState);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,4 +4,6 @@
|
||||
<string name="title_activity_main">Z.AI Chat</string>
|
||||
<string name="package_name">ai.z.chat</string>
|
||||
<string name="custom_url_scheme">ai.z.chat</string>
|
||||
<string name="accessibility_service_desc">Allows Z.AI Chat to control your device — tap, swipe, type, and read the screen — for AI-powered automation in coding and agentic modes.</string>
|
||||
<string name="accessibility_service_summary">AI device control for agentic automation</string>
|
||||
</resources>
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:accessibilityEventTypes="typeAllMask"
|
||||
android:accessibilityFeedbackType="feedbackGeneric"
|
||||
android:accessibilityFlags="flagDefault|flagRetrieveInteractiveWindows|flagReportViewIds|flagIncludeNotImportantViews"
|
||||
android:canPerformGestures="true"
|
||||
android:canRetrieveWindowContent="true"
|
||||
android:notificationTimeout="100"
|
||||
android:description="@string/accessibility_service_desc"
|
||||
android:settingsActivity="ai.z.chat.MainActivity"
|
||||
android:summary="@string/accessibility_service_summary" />
|
||||
Reference in New Issue
Block a user