From 089ca56d937ee64ad247e71dd09809a2c66391c4 Mon Sep 17 00:00:00 2001 From: gmegidish Date: Wed, 26 Aug 2026 21:49:59 +0200 Subject: [PATCH 1/2] feat: dump Flutter UI on Android via the Dart VM service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flutter renders its whole UI into a single opaque native view, so the accessibility-based dumps (uiautomator / DeviceKit) only see what Flutter publishes to the a11y layer: merged, typeless, and missing anything without semantics (e.g. a CustomPaint chart). For a debuggable Flutter app we now read the live render tree directly from the Dart VM service. Pipeline (all with no `flutter run`, on a normally-launched debug app): - Agent RPC `device.flutter.vmServiceUri` reflects FlutterJNI.getVMServiceUri() in the host app's classloader to return the service URI + auth token. A ClassNotFoundException is the clean "not a Flutter app" signal. No log reads. - DumpSource gates on a debuggable foreground app, forwards a host port to the VM service, and walks the render tree over the VM-service WebSocket using `invoke` (reflective method call — needs no Dart expression compiler, unlike `evaluate`). Children come from RenderObject.debugDescribeChildren() (type- agnostic); global bounds from localToGlobal(Offset.zero) + size, scaled to physical px. Any failure falls through to the existing accessibility dump. This surfaces elements the accessibility path cannot: the icon-only "+" button (mobile-next/mobilewright#234), real widget types (Paragraph/Image/CustomPaint/ Editable) instead of android.view.View, and non-semantic widgets like a CustomPaint chart that is entirely absent from uiautomator. Also: keep unlabeled-but-clickable nodes in collectDeviceKitElements (they were dropped when they had no text/desc/hint/resource-id), mirroring the uiautomator XML path — fixes Flutter icon-only controls disappearing from the dump. Perf: ~0.85s for the Flutter render-tree dump (getClassList returns class names inline in one call; the tree walk and Offset.zero scan run concurrently under a call-throttling semaphore). Known gaps (follow-ups): no semantic labels/identifiers/tap-actions yet (needs a semantics-tree merge); off-screen nodes emitted with negative y; per-isolate caching of class ids / Offset.zero would cut repeat-dump bootstrap. Claude-Session: https://claude.ai/code/session_01CTSXjZJjYbaLDTyeVAh1Rp --- agents/android/java/JsonRpcDispatcher.java | 31 + devices/android.go | 14 +- devices/android_elements_test.go | 33 ++ devices/android_flutter.go | 632 +++++++++++++++++++++ devices/android_flutter_test.go | 39 ++ 5 files changed, 748 insertions(+), 1 deletion(-) create mode 100644 devices/android_flutter.go create mode 100644 devices/android_flutter_test.go diff --git a/agents/android/java/JsonRpcDispatcher.java b/agents/android/java/JsonRpcDispatcher.java index 97a6a633..9accf8de 100644 --- a/agents/android/java/JsonRpcDispatcher.java +++ b/agents/android/java/JsonRpcDispatcher.java @@ -46,6 +46,37 @@ static String dispatch(String json) { case "device.dump.ui": return result(id, WebViewAgent.dumpUi()); + case "device.flutter.vmServiceUri": { + // Returns the running Flutter app's Dart VM service URI + // (including the auth token) by reflecting + // FlutterJNI.getVMServiceUri(). FlutterJNI lives in the host + // app's PathClassLoader — reached via Context.getClassLoader() + // on ActivityThread.currentApplication(); the agent dex's own + // loader (parented to the system loader) cannot see it, and + // app.getClass().getClassLoader() is the boot loader when the + // app uses the base Application class. + try { + Object app = Class.forName("android.app.ActivityThread") + .getMethod("currentApplication").invoke(null); + ClassLoader appLoader = app == null + ? Thread.currentThread().getContextClassLoader() + : (ClassLoader) app.getClass().getMethod("getClassLoader").invoke(app); + Class jni = Class.forName( + "io.flutter.embedding.engine.FlutterJNI", false, appLoader); + Object v; + try { + v = jni.getMethod("getVMServiceUri").invoke(null); + } catch (NoSuchMethodException e) { + v = jni.getMethod("getObservatoryUri").invoke(null); // pre-2022 engines + } + return result(id, new JSONObject().put("uri", v == null ? "" : v.toString())); + } catch (ClassNotFoundException e) { + throw new RpcException(RpcException.INVALID_PARAMS, "not a flutter app"); + } catch (Exception e) { + throw new RpcException(RpcException.INTERNAL_ERROR, "vmServiceUri failed: " + e.getMessage()); + } + } + case "device.webview.list": return result(id, WebViewAgent.listWebViews()); diff --git a/devices/android.go b/devices/android.go index 89af4a86..aa8c9aca 100644 --- a/devices/android.go +++ b/devices/android.go @@ -1441,6 +1441,8 @@ type deviceKitNode struct { Focused bool `json:"focused"` Enabled bool `json:"enabled"` Checked bool `json:"checked"` + Checkable bool `json:"checkable"` + Clickable bool `json:"clickable"` Selected bool `json:"selected"` Visible bool `json:"visible"` Rect deviceKitRect `json:"rect"` @@ -1563,7 +1565,9 @@ func collectDeviceKitElements(nodes []deviceKitNode) []types.ScreenElement { for _, node := range nodes { childElements := collectDeviceKitElements(node.Children) - if node.Text == "" && node.ContentDesc == "" && node.Hint == "" && node.ResourceID == "" { + // keep interactable nodes even when unlabeled (e.g. Flutter icon-only + // buttons), mirroring the uiautomator XML path in collectElements + if node.Text == "" && node.ContentDesc == "" && node.Hint == "" && node.ResourceID == "" && !node.Clickable && !node.Checkable { elements = append(elements, childElements...) continue } @@ -1723,6 +1727,14 @@ func (d *AndroidDevice) DumpSourceRaw() (any, error) { } func (d *AndroidDevice) DumpSource() ([]ScreenElement, error) { + // Flutter apps render into an opaque native view, so the accessibility-based + // dumps below miss typed/unlabeled/non-semantic widgets. When the foreground + // app is a debuggable Flutter app, read its live render tree from the Dart VM + // service instead. Any failure falls through to the accessibility dump. + if elements, ok := d.tryDumpFlutterSource(); ok { + return elements, nil + } + if nodes, err := d.getDeviceKitNodes(); err == nil { return collectDeviceKitElements(nodes), nil } else { diff --git a/devices/android_elements_test.go b/devices/android_elements_test.go index f6439f4b..520b4418 100644 --- a/devices/android_elements_test.go +++ b/devices/android_elements_test.go @@ -547,3 +547,36 @@ func TestCollectDeviceKitElementsMarksSelectedNode(t *testing.T) { t.Errorf("expected Selected to be true, got %+v", output[0].Selected) } } + +// Flutter icon-only controls (e.g. a "+" button) are clickable views with no +// text, content-desc, hint, or resource-id. They must not be dropped from the +// tree — see https://github.com/mobile-next/mobilewright/issues/234. +func TestCollectDeviceKitElementsKeepsUnlabeledClickableNode(t *testing.T) { + nodes := []deviceKitNode{ + { + Class: "android.view.View", + ContentDesc: "Premium package (qty: 0)", + Clickable: true, + Rect: deviceKitRect{X: 48, Y: 1563, Width: 1184, Height: 120}, + Children: []deviceKitNode{ + { + Class: "android.view.View", + Clickable: true, + Rect: deviceKitRect{X: 1112, Y: 1563, Width: 120, Height: 120}, + }, + }, + }, + } + + output := collectDeviceKitElements(nodes) + + if len(output) != 1 { + t.Fatalf("expected 1 top-level element, got %d: %+v", len(output), output) + } + if len(output[0].Children) != 1 { + t.Fatalf("expected the unlabeled clickable child to be kept, got %+v", output[0].Children) + } + if output[0].Children[0].Rect.X != 1112 { + t.Errorf("expected child rect x=1112, got %+v", output[0].Children[0].Rect) + } +} diff --git a/devices/android_flutter.go b/devices/android_flutter.go new file mode 100644 index 00000000..448ae32e --- /dev/null +++ b/devices/android_flutter.go @@ -0,0 +1,632 @@ +package devices + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/gorilla/websocket" + "github.com/mobile-next/mobilecli/types" + "github.com/mobile-next/mobilecli/utils" +) + +// Flutter renders its whole UI into a single opaque native view, so uiautomator +// and the accessibility tree only ever see what Flutter publishes to the a11y +// layer — merged, typeless, and missing anything without semantics (e.g. a +// CustomPaint chart). For a debuggable Flutter app we instead read the live +// render tree from the Dart VM service: the JVMTI agent hands us the service +// URI (with its auth token) via FlutterJNI.getVMServiceUri(), and here we walk +// the render objects over the VM-service WebSocket using `invoke` — a reflective +// method call that, unlike `evaluate`, needs no Dart expression compiler and so +// works on a normally-launched app (no `flutter run`). Each render object's +// global position comes from invoking RenderBox.localToGlobal(Offset.zero). + +const flutterVMCallTimeout = 15 * time.Second + +// vmServiceURIPattern extracts port and token from http://127.0.0.1:PORT/TOKEN/ +var vmServiceURIPattern = regexp.MustCompile(`^http://127\.0\.0\.1:(\d+)/([^/]*)/?$`) + +// flutterVMServiceURI attaches the agent to pkg and asks it for the running +// Flutter app's Dart VM service URI. An empty string (no error) or an error +// both mean "treat as not-Flutter and fall back to the accessibility dump". +func (d *AndroidDevice) flutterVMServiceURI(pkg string) string { + port, err := d.ensureAgentReady(pkg) + if err != nil { + utils.Verbose("flutter: agent not ready for %s: %v", pkg, err) + return "" + } + raw, err := agentRequest(port, "device.flutter.vmServiceUri", nil) + if err != nil { + // "not a flutter app" is the expected negative — the FlutterJNI class is + // absent from a non-Flutter app's classloader. + utils.Verbose("flutter: vmServiceUri for %s: %v", pkg, err) + return "" + } + var r struct { + URI string `json:"uri"` + } + if err := json.Unmarshal(raw, &r); err != nil { + return "" + } + return r.URI +} + +// tryDumpFlutterSource returns the Flutter render tree for the foreground app, +// or ok=false to signal the caller should use the accessibility dump. Detection +// requires a debuggable app (the JVMTI agent only attaches to those), which is +// also the only case where a Dart VM service exists. +func (d *AndroidDevice) tryDumpFlutterSource() ([]types.ScreenElement, bool) { + foreground, err := d.GetForegroundApp() + if err != nil { + return nil, false + } + pkg := foreground.PackageName + if !d.isAppDebuggable(pkg) { + return nil, false + } + uri := d.flutterVMServiceURI(pkg) + if uri == "" { + return nil, false + } + start := time.Now() + elements, err := d.dumpFlutterSource(uri) + if err != nil { + utils.Verbose("flutter: render-tree dump failed, falling back: %v", err) + return nil, false + } + utils.Verbose("flutter: render-tree dump produced %d elements in %s", len(elements), time.Since(start)) + return elements, true +} + +// dumpFlutterSource reads the Flutter render tree over the VM service and +// converts it to ScreenElements. uri is the on-device service URI; we forward a +// host port to its device port and speak the VM-service protocol over WebSocket. +func (d *AndroidDevice) dumpFlutterSource(uri string) ([]types.ScreenElement, error) { + m := vmServiceURIPattern.FindStringSubmatch(strings.TrimSpace(uri)) + if m == nil { + return nil, fmt.Errorf("unexpected Dart VM service URI: %q", uri) + } + devicePort, token := m[1], m[2] + + out, err := d.runAdbCommand("forward", "tcp:0", "tcp:"+devicePort) + if err != nil { + return nil, fmt.Errorf("adb forward to VM service: %s: %w", strings.TrimSpace(string(out)), err) + } + localPort, err := strconv.Atoi(strings.TrimSpace(string(out))) + if err != nil { + return nil, fmt.Errorf("unexpected adb forward output %q: %w", strings.TrimSpace(string(out)), err) + } + defer d.runAdbCommand("forward", "--remove", fmt.Sprintf("tcp:%d", localPort)) + + wsURL := fmt.Sprintf("ws://127.0.0.1:%d/%s/ws", localPort, token) + vm, err := dialFlutterVM(wsURL) + if err != nil { + return nil, err + } + defer vm.close() + + if err := vm.resolveIsolate(); err != nil { + return nil, err + } + return vm.dumpRenderTree(d.devicePixelRatio()) +} + +// devicePixelRatio maps Flutter's logical pixels to the physical pixels that +// uiautomator (and taps) use. `wm density` reports dpi; dpr = dpi / 160. +func (d *AndroidDevice) devicePixelRatio() float64 { + out, err := d.runAdbCommand("shell", "wm", "density") + if err == nil { + if m := regexp.MustCompile(`(\d+)`).FindStringSubmatch(string(out)); m != nil { + if dpi, err := strconv.Atoi(m[1]); err == nil && dpi > 0 { + return float64(dpi) / 160.0 + } + } + } + return 1.0 +} + +// --------------------------------------------------------------------------- +// VM-service WebSocket client (JSON-RPC 2.0, concurrent request/response) +// --------------------------------------------------------------------------- + +type flutterVM struct { + conn *websocket.Conn + writeMu sync.Mutex + mu sync.Mutex + seq int + pending map[int]chan vmResp + isolateID string + dpr float64 + zeroOffsetID string // objectId of Offset.zero, the localToGlobal argument + callSem chan struct{} // bounds concurrent in-flight VM-service calls +} + +// maxConcurrentVMCalls caps in-flight VM-service requests. The walk fans out one +// goroutine per render child (cheap), but the underlying calls are throttled +// here so we pipeline aggressively without flooding the VM service. Structural +// goroutines wait on their children *after* their own calls return, so they +// never hold a call slot across a wait — no deadlock regardless of tree shape. +const maxConcurrentVMCalls = 48 + +type vmResp struct { + result json.RawMessage + err error +} + +func dialFlutterVM(wsURL string) (*flutterVM, error) { + conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil) + if err != nil { + return nil, fmt.Errorf("connect to Dart VM service: %w", err) + } + vm := &flutterVM{ + conn: conn, + pending: make(map[int]chan vmResp), + callSem: make(chan struct{}, maxConcurrentVMCalls), + } + go vm.readLoop() + return vm, nil +} + +func (vm *flutterVM) close() { + vm.conn.Close() +} + +func (vm *flutterVM) readLoop() { + for { + _, data, err := vm.conn.ReadMessage() + if err != nil { + vm.failAll(err) + return + } + var msg struct { + ID *int `json:"id"` + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(data, &msg); err != nil || msg.ID == nil { + continue + } + vm.mu.Lock() + ch := vm.pending[*msg.ID] + delete(vm.pending, *msg.ID) + vm.mu.Unlock() + if ch == nil { + continue + } + if msg.Error != nil { + ch <- vmResp{err: fmt.Errorf("vm error %d: %s", msg.Error.Code, msg.Error.Message)} + } else { + ch <- vmResp{result: msg.Result} + } + } +} + +func (vm *flutterVM) failAll(err error) { + vm.mu.Lock() + defer vm.mu.Unlock() + for id, ch := range vm.pending { + ch <- vmResp{err: err} + delete(vm.pending, id) + } +} + +func (vm *flutterVM) call(method string, params map[string]any) (json.RawMessage, error) { + vm.callSem <- struct{}{} + defer func() { <-vm.callSem }() + + vm.mu.Lock() + vm.seq++ + id := vm.seq + ch := make(chan vmResp, 1) + vm.pending[id] = ch + vm.mu.Unlock() + + req := map[string]any{"jsonrpc": "2.0", "id": id, "method": method} + if params != nil { + req["params"] = params + } + payload, _ := json.Marshal(req) + + vm.writeMu.Lock() + err := vm.conn.WriteMessage(websocket.TextMessage, payload) + vm.writeMu.Unlock() + if err != nil { + vm.mu.Lock() + delete(vm.pending, id) + vm.mu.Unlock() + return nil, err + } + + select { + case r := <-ch: + return r.result, r.err + case <-time.After(flutterVMCallTimeout): + vm.mu.Lock() + delete(vm.pending, id) + vm.mu.Unlock() + return nil, fmt.Errorf("vm service call %q timed out", method) + } +} + +// vmInstanceRef / vmInstance model the subset of the VM-service Instance shape +// we read: an object id, its class name, a scalar string value, and its fields. +type vmInstanceRef struct { + Type string `json:"type"` + ID string `json:"id"` + Kind string `json:"kind"` + Name string `json:"name"` // populated for Class objects (the class name) + ValueAsStr string `json:"valueAsString"` + ClassRef *struct { + Name string `json:"name"` + } `json:"class"` +} + +func (r *vmInstanceRef) className() string { + if r != nil && r.ClassRef != nil { + return r.ClassRef.Name + } + return "" +} + +type vmInstance struct { + vmInstanceRef + Fields []struct { + Decl *struct { + Name string `json:"name"` + } `json:"decl"` + Name string `json:"name"` + Value *vmInstanceRef `json:"value"` + } `json:"fields"` + Elements []vmInstanceRef `json:"elements"` // populated for List objects +} + +func (o *vmInstance) field(name string) *vmInstanceRef { + for _, f := range o.Fields { + fn := f.Name + if f.Decl != nil { + fn = f.Decl.Name + } + if fn == name { + return f.Value + } + } + return nil +} + +func (vm *flutterVM) getObject(objectID string) (*vmInstance, error) { + raw, err := vm.call("getObject", map[string]any{"isolateId": vm.isolateID, "objectId": objectID}) + if err != nil { + return nil, err + } + var o vmInstance + if err := json.Unmarshal(raw, &o); err != nil { + return nil, err + } + return &o, nil +} + +// invoke calls a method reflectively on a live object (no expression compiler). +func (vm *flutterVM) invoke(target, selector string, args []string) (*vmInstanceRef, error) { + if args == nil { + args = []string{} + } + raw, err := vm.call("invoke", map[string]any{ + "isolateId": vm.isolateID, + "targetId": target, + "selector": selector, + "argumentIds": args, + }) + if err != nil { + return nil, err + } + var r vmInstanceRef + if err := json.Unmarshal(raw, &r); err != nil { + return nil, err + } + return &r, nil +} + +func (vm *flutterVM) resolveIsolate() error { + raw, err := vm.call("getVM", nil) + if err != nil { + return err + } + var v struct { + Isolates []struct { + ID string `json:"id"` + } `json:"isolates"` + } + if err := json.Unmarshal(raw, &v); err != nil { + return err + } + if len(v.Isolates) == 0 { + return fmt.Errorf("no Dart isolate available") + } + vm.isolateID = v.Isolates[0].ID + return nil +} + +// findClassIDs resolves class ids by exact name. getClassList already returns +// every class ref with its name inline, so this is a single round trip — no +// per-class getObject. +func (vm *flutterVM) findClassIDs(names map[string]string) error { + raw, err := vm.call("getClassList", map[string]any{"isolateId": vm.isolateID}) + if err != nil { + return err + } + var cl struct { + Classes []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"classes"` + } + if err := json.Unmarshal(raw, &cl); err != nil { + return err + } + for _, c := range cl.Classes { + if _, want := names[c.Name]; want && names[c.Name] == "" { + names[c.Name] = c.ID + } + } + return nil +} + +// --------------------------------------------------------------------------- +// Render-tree walk +// --------------------------------------------------------------------------- + +const ( + rootRenderClass = "_ReusableRenderView" + offsetClass = "Offset" +) + +var offsetSizePattern = regexp.MustCompile(`\(([-\d.]+),\s*([-\d.]+)\)`) + +func (vm *flutterVM) dumpRenderTree(dpr float64) ([]types.ScreenElement, error) { + vm.dpr = dpr + t0 := time.Now() + classes := map[string]string{rootRenderClass: "", offsetClass: ""} + if err := vm.findClassIDs(classes); err != nil { + return nil, err + } + if classes[rootRenderClass] == "" { + return nil, fmt.Errorf("could not locate the Flutter root render object (%s)", rootRenderClass) + } + + rootID, err := vm.firstInstance(classes[rootRenderClass]) + if err != nil { + return nil, fmt.Errorf("no root render object instance: %w", err) + } + zeroID, err := vm.offsetZeroID(classes[offsetClass]) + if err != nil { + return nil, fmt.Errorf("could not obtain Offset.zero: %w", err) + } + vm.zeroOffsetID = zeroID + utils.Verbose("flutter: bootstrap (classes+root+offset.zero) took %s", time.Since(t0)) + + t1 := time.Now() + els, err := vm.visit(rootID, rootRenderClass) + utils.Verbose("flutter: tree walk took %s", time.Since(t1)) + return els, err +} + +// renderChild is a child render object plus its class name (both come from the +// parent's debugDescribeChildren, so no extra getObject is needed per node). +type renderChild struct { + id string + class string +} + +func (vm *flutterVM) firstInstance(classID string) (string, error) { + raw, err := vm.call("getInstances", map[string]any{"isolateId": vm.isolateID, "objectId": classID, "limit": 1}) + if err != nil { + return "", err + } + var r struct { + Instances []struct { + ID string `json:"id"` + } `json:"instances"` + } + if err := json.Unmarshal(raw, &r); err != nil { + return "", err + } + if len(r.Instances) == 0 { + return "", fmt.Errorf("class %s has no live instances", classID) + } + return r.Instances[0].ID, nil +} + +// offsetZeroID finds the const Offset.zero on the heap (fields _dx/_dy == 0.0), +// used as the point argument to RenderBox.localToGlobal. +func (vm *flutterVM) offsetZeroID(offsetClassID string) (string, error) { + raw, err := vm.call("getInstances", map[string]any{"isolateId": vm.isolateID, "objectId": offsetClassID, "limit": 500}) + if err != nil { + return "", err + } + var r struct { + Instances []struct { + ID string `json:"id"` + } `json:"instances"` + } + if err := json.Unmarshal(raw, &r); err != nil { + return "", err + } + // Inspect the instances concurrently — call() throttles the fan-out. + found := make([]string, len(r.Instances)) + var wg sync.WaitGroup + for i, inst := range r.Instances { + wg.Add(1) + go func(i int, id string) { + defer wg.Done() + o, err := vm.getObject(id) + if err != nil { + return + } + dx, dy := o.field("_dx"), o.field("_dy") + if dx != nil && dy != nil && dx.ValueAsStr == "0.0" && dy.ValueAsStr == "0.0" { + found[i] = id + } + }(i, inst.ID) + } + wg.Wait() + for _, id := range found { + if id != "" { + return id, nil + } + } + return "", fmt.Errorf("Offset.zero not found among live instances") +} + +// visit walks a render object subtree, returning the meaningful ScreenElements +// it contains. Pure-wrapper render objects (those with render children) are +// hoisted — only leaf render objects with a non-zero size are emitted, which is +// exactly the visible content (paragraphs, images, custom-painted widgets like +// charts), each with its real Flutter type and global bounds. +func (vm *flutterVM) visit(nodeID, className string) ([]types.ScreenElement, error) { + kids := vm.childrenOf(nodeID) + + // Wrapper node (has render children): visit children concurrently and hoist + // their results to our level, preserving order. + if len(kids) > 0 { + results := make([][]types.ScreenElement, len(kids)) + var wg sync.WaitGroup + for i, k := range kids { + wg.Add(1) + go func(i int, k renderChild) { + defer wg.Done() + results[i], _ = vm.visit(k.id, k.class) + }(i, k) + } + wg.Wait() + var children []types.ScreenElement + for _, r := range results { + children = append(children, r...) + } + return children, nil + } + + // Leaf: emit it if it has a real, on-screen size. + rect, ok := vm.globalRect(nodeID) + if !ok || rect.Width <= 0 || rect.Height <= 0 { + return nil, nil + } + + el := types.ScreenElement{ + Type: friendlyRenderType(className), + Rect: rect, + } + if text := vm.extractText(nodeID, className); text != "" { + el.Text = &text + } + return []types.ScreenElement{el}, nil +} + +// childrenOf returns a render object's render children type-agnostically via +// RenderObject.debugDescribeChildren() — every RenderObject implements it, +// regardless of how it stores children (single-child, container, or sliver). +// Each returned DiagnosticsNode's `value` is the child render object. +func (vm *flutterVM) childrenOf(nodeID string) []renderChild { + list, err := vm.invoke(nodeID, "debugDescribeChildren", nil) + if err != nil || list.ID == "" { + return nil + } + obj, err := vm.getObject(list.ID) + if err != nil { + return nil + } + var kids []renderChild + for _, node := range obj.Elements { + if node.ID == "" { + continue + } + val, err := vm.invoke(node.ID, "get:value", nil) + if err != nil || val.ID == "" || val.Kind == "Null" { + continue + } + // debugDescribeChildren only lists children, but guard anyway. + if !strings.Contains(val.className(), "Render") { + continue + } + kids = append(kids, renderChild{id: val.ID, class: val.className()}) + } + return kids +} + +// globalRect returns a render object's global bounds in physical pixels via +// RenderBox.localToGlobal(Offset.zero) and RenderBox.size. +func (vm *flutterVM) globalRect(nodeID string) (types.ScreenElementRect, bool) { + off, err := vm.invoke(nodeID, "localToGlobal", []string{vm.zeroOffsetID}) + if err != nil || off.ID == "" { + return types.ScreenElementRect{}, false + } + ox, oy, ok := vm.offsetPair(off.ID) + if !ok { + return types.ScreenElementRect{}, false + } + size, err := vm.invoke(nodeID, "get:size", nil) + if err != nil || size.ID == "" { + return types.ScreenElementRect{}, false + } + sw, sh, ok := vm.offsetPair(size.ID) + if !ok { + return types.ScreenElementRect{}, false + } + return types.ScreenElementRect{ + X: int(ox*vm.dpr + 0.5), + Y: int(oy*vm.dpr + 0.5), + Width: int(sw*vm.dpr + 0.5), + Height: int(sh*vm.dpr + 0.5), + }, true +} + +// offsetPair reads the two doubles of an Offset or Size (both store _dx/_dy). +func (vm *flutterVM) offsetPair(objectID string) (float64, float64, bool) { + o, err := vm.getObject(objectID) + if err != nil { + return 0, 0, false + } + dx, dy := o.field("_dx"), o.field("_dy") + if dx == nil || dy == nil { + return 0, 0, false + } + x, err1 := strconv.ParseFloat(dx.ValueAsStr, 64) + y, err2 := strconv.ParseFloat(dy.ValueAsStr, 64) + if err1 != nil || err2 != nil { + return 0, 0, false + } + return x, y, true +} + +// extractText reads the plain text of a paragraph/editable render object. +func (vm *flutterVM) extractText(nodeID, className string) string { + if !strings.Contains(className, "Paragraph") && !strings.Contains(className, "Editable") { + return "" + } + span, err := vm.invoke(nodeID, "get:text", nil) + if err != nil || span.ID == "" { + return "" + } + plain, err := vm.invoke(span.ID, "toPlainText", nil) + if err != nil { + return "" + } + return plain.ValueAsStr +} + +// friendlyRenderType turns a render class name into a widget-ish type, e.g. +// RenderParagraph -> Paragraph, _RenderColoredBox -> ColoredBox. +func friendlyRenderType(name string) string { + name = strings.TrimPrefix(name, "_") + name = strings.TrimPrefix(name, "Render") + if name == "" { + return "FlutterWidget" + } + return name +} diff --git a/devices/android_flutter_test.go b/devices/android_flutter_test.go new file mode 100644 index 00000000..37da7c0e --- /dev/null +++ b/devices/android_flutter_test.go @@ -0,0 +1,39 @@ +package devices + +import "testing" + +func TestFriendlyRenderTypeStripsRenderAndUnderscorePrefixes(t *testing.T) { + cases := map[string]string{ + "RenderParagraph": "Paragraph", + "RenderCustomPaint": "CustomPaint", + "RenderImage": "Image", + "_RenderColoredBox": "ColoredBox", + "RenderEditable": "Editable", + "": "FlutterWidget", + "Render": "FlutterWidget", + } + for in, want := range cases { + if got := friendlyRenderType(in); got != want { + t.Errorf("friendlyRenderType(%q) = %q, want %q", in, got, want) + } + } +} + +func TestVMServiceURIPatternExtractsPortAndToken(t *testing.T) { + m := vmServiceURIPattern.FindStringSubmatch("http://127.0.0.1:46809/fEuKPZ9fvrk=/") + if m == nil { + t.Fatal("expected the Dart VM service URI to match") + } + if m[1] != "46809" { + t.Errorf("port = %q, want 46809", m[1]) + } + if m[2] != "fEuKPZ9fvrk=" { + t.Errorf("token = %q, want fEuKPZ9fvrk=", m[2]) + } +} + +func TestVMServiceURIPatternRejectsNonLoopback(t *testing.T) { + if vmServiceURIPattern.MatchString("http://10.0.0.5:46809/token/") { + t.Error("expected a non-loopback VM service URI to be rejected") + } +} From f602212bd7200c0ea7e9b13a1f90dcfbf21e9854 Mon Sep 17 00:00:00 2001 From: gmegidish Date: Wed, 26 Aug 2026 22:08:07 +0200 Subject: [PATCH 2/2] feat: dump Flutter UI on the iOS simulator via the Dart VM service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The render-tree walk is identical Dart to Android (same _ReusableRenderView root, same invoke/localToGlobal), so SimulatorDevice.DumpSource reuses the whole flutterVM client and dumpRenderTree. Only detection and URI acquisition are iOS-specific: - Detect Flutter: the app bundle embeds Frameworks/Flutter.framework (via `simctl get_app_container`). - Get the Dart VM service URI: primarily over mDNS — the engine advertises `_dartVmService._tcp` (instance name = bundle id) with the port and auth code in its TXT record, live for the app's lifetime. This is what `flutter attach` uses, resolves in ~5ms, and survives log rotation. The simulator log is kept as a fallback (a one-shot launch line that rotates out of the buffer, and `log show` costs ~1.25s), and finally the accessibility dump. Simulator processes run natively on the Mac, so the VM service listens on the Mac's own 127.0.0.1 — we connect directly, no port forwarding and no code injection (no LLDB). iOS reports layout in logical points, so it passes dpr=1.0 (Android scales by the device pixel ratio); both share dumpFlutterTreeOverWS. Verified: the demo app's iOS dump went from 15 nodes (missing the app-bar title, the Premium row and its "+", the CustomPaint chart, and everything below the first few list items) to 51 typed nodes with all of them, in ~230ms. Claude-Session: https://claude.ai/code/session_01CTSXjZJjYbaLDTyeVAh1Rp --- devices/android_flutter.go | 12 +-- devices/simulator.go | 7 ++ devices/simulator_flutter.go | 179 +++++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 10 deletions(-) create mode 100644 devices/simulator_flutter.go diff --git a/devices/android_flutter.go b/devices/android_flutter.go index 448ae32e..48cbbe42 100644 --- a/devices/android_flutter.go +++ b/devices/android_flutter.go @@ -102,17 +102,9 @@ func (d *AndroidDevice) dumpFlutterSource(uri string) ([]types.ScreenElement, er } defer d.runAdbCommand("forward", "--remove", fmt.Sprintf("tcp:%d", localPort)) + // The device port is now reachable at the forwarded local port. wsURL := fmt.Sprintf("ws://127.0.0.1:%d/%s/ws", localPort, token) - vm, err := dialFlutterVM(wsURL) - if err != nil { - return nil, err - } - defer vm.close() - - if err := vm.resolveIsolate(); err != nil { - return nil, err - } - return vm.dumpRenderTree(d.devicePixelRatio()) + return dumpFlutterTreeOverWS(wsURL, d.devicePixelRatio()) } // devicePixelRatio maps Flutter's logical pixels to the physical pixels that diff --git a/devices/simulator.go b/devices/simulator.go index a5beebf7..63c7945a 100644 --- a/devices/simulator.go +++ b/devices/simulator.go @@ -889,6 +889,13 @@ func (s *SimulatorDevice) getDeviceKitEnvPort(envVar string) (int, error) { } func (s SimulatorDevice) DumpSource() ([]ScreenElement, error) { + // Flutter apps render into an opaque native view, so the accessibility dump + // misses typed/unlabeled/non-semantic widgets. When the foreground app is a + // Flutter app with a live Dart VM service, read its render tree instead. Any + // failure falls through to the accessibility dump. + if elements, ok := s.tryDumpFlutterSource(); ok { + return elements, nil + } return s.deviceKitClient.GetSourceElements() } diff --git a/devices/simulator_flutter.go b/devices/simulator_flutter.go new file mode 100644 index 00000000..dac6d3a1 --- /dev/null +++ b/devices/simulator_flutter.go @@ -0,0 +1,179 @@ +package devices + +import ( + "bufio" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/mobile-next/mobilecli/types" + "github.com/mobile-next/mobilecli/utils" +) + +// Flutter support for the iOS simulator. The render-tree walk is identical Dart +// to Android (same `_ReusableRenderView` root, same `invoke`/localToGlobal), so +// the whole flutterVM client and dumpRenderTree are reused verbatim. Only two +// things are iOS-specific: detecting Flutter (the app bundle embeds +// Flutter.framework) and obtaining the Dart VM service URI. On the simulator the +// app runs natively on the Mac, so its VM service listens on the Mac's own +// 127.0.0.1 — we connect directly, no port forwarding and no code injection. +// +// iOS reports layout in logical points (matching the existing DeviceKit dump), +// so unlike Android we do not scale by the device pixel ratio (dpr = 1.0). + +// vmServiceLineURL pulls the service URL out of the engine's log line +// "The Dart VM service is listening on http://127.0.0.1:PORT/TOKEN/". +var vmServiceLineURL = regexp.MustCompile(`http://127\.0\.0\.1:\d+/[A-Za-z0-9_=-]*/`) + +// tryDumpFlutterSource returns the Flutter render tree for the foreground app, +// or ok=false to signal the caller should use the accessibility dump. +func (s *SimulatorDevice) tryDumpFlutterSource() ([]types.ScreenElement, bool) { + foreground, err := s.GetForegroundApp() + if err != nil { + return nil, false + } + bundleID := foreground.PackageName + if !s.isFlutterAppBundle(bundleID) { + return nil, false + } + uri := s.flutterVMServiceURI(bundleID) + if uri == "" { + utils.Verbose("flutter: no Dart VM service URI found for %s (release build, or the launch log rotated out)", bundleID) + return nil, false + } + start := time.Now() + elements, err := dumpFlutterSourceFromURI(uri, 1.0) + if err != nil { + utils.Verbose("flutter: render-tree dump failed, falling back: %v", err) + return nil, false + } + utils.Verbose("flutter: render-tree dump produced %d elements in %s", len(elements), time.Since(start)) + return elements, true +} + +// isFlutterAppBundle reports whether the installed app embeds Flutter.framework. +func (s *SimulatorDevice) isFlutterAppBundle(bundleID string) bool { + out, err := runSimctl("get_app_container", s.UDID, bundleID, "app") + if err != nil { + return false + } + appPath := strings.TrimSpace(string(out)) + if appPath == "" { + return false + } + info, err := os.Stat(filepath.Join(appPath, "Frameworks", "Flutter.framework")) + return err == nil && info.IsDir() +} + +// flutterVMServiceURI recovers the running app's Dart VM service URI (with its +// auth token). Primary source is mDNS: the Flutter engine advertises +// `_dartVmService._tcp` (instance name = bundle id) with the port and auth code +// in its TXT record, for as long as the app runs — this is what `flutter attach` +// uses, ~5ms, and survives log rotation. The simulator log is a fallback for the +// rare case where VM-service publication is disabled. +func (s *SimulatorDevice) flutterVMServiceURI(bundleID string) string { + if uri := resolveDartVMServiceMDNS(bundleID, 3*time.Second); uri != "" { + return uri + } + utils.Verbose("flutter: mDNS lookup failed for %s, trying simulator log", bundleID) + return s.flutterVMServiceURIFromLog() +} + +var ( + mdnsPortLine = regexp.MustCompile(`can be reached at \S+?:(\d+)`) + mdnsAuthCode = regexp.MustCompile(`authCode=([A-Za-z0-9_=+/\-]+)`) +) + +// resolveDartVMServiceMDNS resolves the app's Dart VM service via Bonjour and +// returns http://127.0.0.1://, or "" if not found in time. +func resolveDartVMServiceMDNS(bundleID string, timeout time.Duration) string { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // -L resolves a specific instance (the bundle id) to host:port + TXT record. + // dns-sd streams until killed, so we read until we have both fields. Use the + // absolute system path rather than relying on PATH. + cmd := exec.CommandContext(ctx, "/usr/bin/dns-sd", "-L", bundleID, "_dartVmService._tcp", "local.") + stdout, err := cmd.StdoutPipe() + if err != nil { + return "" + } + if err := cmd.Start(); err != nil { + return "" + } + defer func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }() + + var port, auth string + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + if m := mdnsPortLine.FindStringSubmatch(line); m != nil { + port = m[1] + } + if m := mdnsAuthCode.FindStringSubmatch(line); m != nil { + auth = m[1] + } + if port != "" && auth != "" { + return fmt.Sprintf("http://127.0.0.1:%s/%s/", port, auth) + } + } + return "" +} + +// flutterVMServiceURIFromLog scrapes the URI (with token) from the simulator log, +// where the engine prints it once at launch. Fragile — the line rotates out of +// the log buffer over time — so it is only a fallback for mDNS. +func (s *SimulatorDevice) flutterVMServiceURIFromLog() string { + out, err := runSimctl("spawn", s.UDID, "log", "show", "--last", "30m", + "--style", "compact", "--predicate", `eventMessage CONTAINS "Dart VM service is listening"`) + if err != nil { + utils.Verbose("flutter: reading simulator log failed: %v", err) + return "" + } + matches := vmServiceLineURL.FindAllString(string(out), -1) + if len(matches) == 0 { + return "" + } + return matches[len(matches)-1] // newest listener wins +} + +// dumpFlutterSourceFromURI connects to a Dart VM service already reachable at +// uri's host:port (the iOS-simulator case — the app runs on the Mac) and walks +// the render tree. dpr is 1.0 for iOS (points) and the device pixel ratio for +// Android (physical pixels). +func dumpFlutterSourceFromURI(uri string, dpr float64) ([]types.ScreenElement, error) { + m := vmServiceURIPattern.FindStringSubmatch(strings.TrimSpace(uri)) + if m == nil { + return nil, fmt.Errorf("unexpected Dart VM service URI: %q", uri) + } + // m[2] is the auth token; it is empty when the app was launched with + // --disable-service-auth-codes (the no-log variant), which changes the path. + wsURL := fmt.Sprintf("ws://127.0.0.1:%s/ws", m[1]) + if m[2] != "" { + wsURL = fmt.Sprintf("ws://127.0.0.1:%s/%s/ws", m[1], m[2]) + } + return dumpFlutterTreeOverWS(wsURL, dpr) +} + +// dumpFlutterTreeOverWS is the platform-neutral core: dial the Dart VM service +// WebSocket, bind the isolate, and walk the render tree. Android reaches it +// through an adb-forwarded local port; iOS connects to the Mac directly. +func dumpFlutterTreeOverWS(wsURL string, dpr float64) ([]types.ScreenElement, error) { + vm, err := dialFlutterVM(wsURL) + if err != nil { + return nil, err + } + defer vm.close() + if err := vm.resolveIsolate(); err != nil { + return nil, err + } + return vm.dumpRenderTree(dpr) +}