Skip to content

Commit eb9c30f

Browse files
authored
improvement(computer-use): keep an element's id stable across observations (#10046)
* improvement(computer-use): keep an element's id stable across observations * fix the snapshot key and make the stable-id fixture insert ahead of the field * assert stable ids on an element after the inserted row * keep element ids across actions, which consume the actionable snapshot * keep each app's id counter apart from the evictable history and carry the index only when a full read fits the cap
1 parent dfe5904 commit eb9c30f

3 files changed

Lines changed: 49 additions & 11 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,9 @@ func attributes(_ element: AXUIElement, _ names: [String]) -> [String: CFTypeRef
109109
/// Elements bucketed by CFHash, so a walk's membership and parent lookups stay linear in its size.
110110
struct ElementIndex {
111111
private var buckets: [CFHashCode: [(element: AXUIElement, id: String)]] = [:]
112+
private(set) var count = 0
112113
func id(of element: AXUIElement) -> String? { buckets[CFHash(element)]?.first { CFEqual($0.element, element) }?.id }
113-
mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)) }
114+
mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)); count += 1 }
114115
}
115116
let NODE_ATTRIBUTES = [kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXDescriptionAttribute, kAXPlaceholderValueAttribute,
116117
kAXHelpAttribute, kAXValueAttribute, kAXFocusedAttribute, kAXEnabledAttribute, kAXPositionAttribute, kAXSizeAttribute]
@@ -304,7 +305,18 @@ final class Snapshot {
304305
var selectedWindowId: String?
305306
var nodes: [[String: Any]] = []
306307
var truncated = false
307-
init(app: NSRunningApplication) { pid = app.processIdentifier; launchDate = app.launchDate }
308+
/// Every element this app session has been given an id, so the same element keeps its id across observations.
309+
private(set) var index = ElementIndex()
310+
private(set) var nextOrdinal: Int
311+
/// Reuses `previous`'s ids when it observed the same process instance and new ids continue from
312+
/// `nextOrdinal`, which the driver keeps per process so ids never repeat while it runs.
313+
init(app: NSRunningApplication, previous: Snapshot? = nil, nextOrdinal: Int = 0) {
314+
pid = app.processIdentifier; launchDate = app.launchDate; self.nextOrdinal = nextOrdinal
315+
// One read adds at most 2,000 elements, so the carried index stays within 20,000; past that the next read starts fresh.
316+
if let previous, previous.pid == pid, previous.launchDate == launchDate, previous.index.count <= 18_000 {
317+
index = previous.index
318+
}
319+
}
308320
func read(preferredWindowFrame: CGRect? = nil) async throws {
309321
try requireAccessibility()
310322
let root = AXUIElementCreateApplication(pid)
@@ -342,12 +354,12 @@ final class Snapshot {
342354
queue.append(AXTraversalEntry(element: unsafeBitCast(raw, to: AXUIElement.self), parent: root, depth: 1, menu: false, priority: true))
343355
}
344356
var menuCount = 0
345-
var index = ElementIndex()
357+
var seen = ElementIndex()
346358
while nodes.count < 2000 && Date().timeIntervalSince(created) < 8, let entry = queue.next() {
347359
try checkCancellation()
348360
let element = entry.element
349361
AXUIElementSetMessagingTimeout(element, 0.25)
350-
if index.id(of: element) != nil { continue }
362+
if seen.id(of: element) != nil { continue }
351363
let values = attributes(element, NODE_ATTRIBUTES)
352364
let role = values[kAXRoleAttribute] as? String ?? "AXUnknown"
353365
let isMenu = entry.menu || [kAXMenuBarRole, kAXMenuRole, kAXMenuItemRole, kAXMenuBarItemRole].contains(role)
@@ -359,9 +371,13 @@ final class Snapshot {
359371
guard menuCount < 80 else { continue }
360372
menuCount += 1
361373
}
362-
let id = "e\(nodes.count)"; elements[id] = element; index.insert(element, id: id)
374+
let id: String
375+
if let known = index.id(of: element) { id = known } else {
376+
id = "e\(nextOrdinal)"; nextOrdinal += 1; index.insert(element, id: id)
377+
}
378+
elements[id] = element; seen.insert(element, id: id)
363379
var node: [String: Any] = ["elementId": id, "role": role, "actions": [String]()]
364-
if let parent = entry.parent, let parentID = index.id(of: parent) { node["parentId"] = parentID }
380+
if let parent = entry.parent, let parentID = seen.id(of: parent) { node["parentId"] = parentID }
365381
let isSecure = values[kAXSubroleAttribute] as? String == kAXSecureTextFieldSubrole
366382
var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty }
367383
if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } }
@@ -688,6 +704,10 @@ func discoverApps() throws -> [[String: Any]] {
688704
@MainActor
689705
final class Driver {
690706
var snapshots: [String: Snapshot] = [:]
707+
/// Each app's latest observation, kept after actions consume `snapshots` so the next read reuses its element ids.
708+
var lastObserved: [String: Snapshot] = [:]
709+
/// Each app's next element id, kept apart from the evictable history so ids never repeat while that process runs.
710+
var nextOrdinals: [String: (pid: pid_t, launchDate: Date?, next: Int)] = [:]
691711
func point(_ p: Parameters, app: NSRunningApplication, snapshot: Snapshot, end: Bool = false) throws -> CGPoint {
692712
if let elementId = p.elementId, !end {
693713
guard let frame = elementRect(try snapshot.element(elementId)) else { throw ComputerError("element_unavailable", "Element has no usable frame.") }
@@ -741,7 +761,9 @@ final class Driver {
741761
return ["kind": "action", "action": "activate_app", "bundleId": app.bundleIdentifier!, "dispatched": true, "verified": true]
742762
case "get_app_state":
743763
try requireAccessibility()
744-
let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app)
764+
let app = try await appFor(p.bundleId, launch: true); let bundle = app.bundleIdentifier!
765+
let ordinal = nextOrdinals[bundle].flatMap { $0.pid == app.processIdentifier && $0.launchDate == app.launchDate ? $0.next : nil } ?? 0
766+
let snapshot = Snapshot(app: app, previous: lastObserved[bundle], nextOrdinal: ordinal)
745767
let preferredFrame = try p.windowId.map { try windowRect($0, pid: app.processIdentifier) }
746768
try await snapshot.read(preferredWindowFrame: preferredFrame)
747769
let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole }
@@ -765,7 +787,10 @@ final class Driver {
765787
result.merge(rectJSON(rect)) { _, new in new }; return result
766788
}
767789
if snapshots.count >= 16 { snapshots.removeAll() }
768-
snapshots[app.bundleIdentifier!] = snapshot
790+
if lastObserved[bundle] == nil, lastObserved.count >= 16, let evicted = lastObserved.keys.first { lastObserved.removeValue(forKey: evicted) }
791+
snapshots[bundle] = snapshot
792+
lastObserved[bundle] = snapshot
793+
nextOrdinals[bundle] = (app.processIdentifier, app.launchDate, snapshot.nextOrdinal)
769794
let root = AXUIElementCreateApplication(app.processIdentifier)
770795
let focusedID = attribute(root, kAXFocusedWindowAttribute).flatMap { CFGetTypeID($0) == AXUIElementGetTypeID() ? snapshot.windowId(holding: unsafeBitCast($0, to: AXUIElement.self)) : nil }
771796
for index in snapshot.nodes.indices {

‎apps/desktop/native/computer-use/tests/Fixture.swift‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ final class InfiniteFrameView: NSView {
2525
}
2626
final class FixtureDelegate: NSObject, NSApplicationDelegate {
2727
var window: NSWindow!
28+
var stack: NSStackView!
2829
var count = 0
2930
let counter = NSTextField(labelWithString: "Count: 0")
3031
let scrollCounter = NSTextField(labelWithString: "Scroll: 0")
@@ -47,14 +48,15 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate {
4748
NSApplication.shared.mainMenu = menu
4849
window = NSWindow(contentRect: NSRect(x: 100, y: 100, width: 600, height: 720), styleMask: [.titled, .closable, .resizable], backing: .buffered, defer: false)
4950
window.title = "Mothership Computer Use Test Fixture"
50-
let stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16
51+
stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16
5152
stack.translatesAutoresizingMaskIntoConstraints = false
5253
let field = NSTextField(string: "Fixture text"); field.setAccessibilityIdentifier("fixture-text"); field.setAccessibilityLabel("Fixture editable text")
5354
let secure = NSSecureTextField(string: "fixture-only-secret"); secure.setAccessibilityIdentifier("fixture-password"); secure.setAccessibilityLabel("Fixture password")
5455
let button = NSButton(title: "Increment", target: self, action: #selector(increment)); button.setAccessibilityIdentifier("fixture-increment")
5556
let clickField = ClickFocusField(string: ""); clickField.setAccessibilityLabel("Fixture click-focus field")
5657
let caretField = FixedCaretField(string: String(repeating: "keep ", count: 40)); caretField.setAccessibilityLabel("Fixture fixed-caret field")
5758
let sheetButton = NSButton(title: "Show sheet", target: self, action: #selector(showSheet)); sheetButton.setAccessibilityIdentifier("fixture-show-sheet")
59+
let addRowButton = NSButton(title: "Add row", target: self, action: #selector(addRow))
5860
let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider")
5961
let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time")
6062
time.dateValue = Calendar.current.date(bySettingHour: 10, minute: 15, second: 0, of: Date()) ?? Date(); time.target = self; time.action = #selector(timeChanged(_:))
@@ -68,14 +70,19 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate {
6870
scroll.documentView = document
6971
scroll.contentView.postsBoundsChangedNotifications = true
7072
NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView)
71-
for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) }
73+
for view in [addRowButton, field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) }
7274
window.contentView!.addSubview(stack)
7375
NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), clickField.widthAnchor.constraint(equalTo: stack.widthAnchor), caretField.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)])
7476
window.makeKeyAndOrderFront(nil)
7577
window.makeFirstResponder(field)
7678
NSApplication.shared.activate()
7779
print("fixture-ready"); fflush(stdout)
7880
}
81+
/// Adds a label ahead of every control in Accessibility order, so positional ids after it would all shift.
82+
@objc func addRow() {
83+
let row = NSTextField(labelWithString: "Added row"); row.frame = NSRect(x: 4, y: 4, width: 120, height: 18)
84+
window.contentView!.addSubview(row, positioned: .below, relativeTo: stack)
85+
}
7986
@objc func increment() { count += 1; counter.stringValue = "Count: \(count)" }
8087
@objc func showSheet() { let alert = NSAlert(); alert.messageText = "Fixture sheet"; alert.addButton(withTitle: "Dismiss"); alert.beginSheetModal(for: window) }
8188
@objc func timeChanged(_ picker: NSDatePicker) {

‎apps/desktop/native/computer-use/tests/live.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,12 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5):
9191
else:
9292
assert 'screen_capture_permission_required' in snapshot.get('screenshotError', '')
9393
print('BLOCKED: screenshot verification skipped because Screen Recording grant is missing', flush=True)
94+
# Increment follows the inserted row and is never the focused editor, which traversal always lists first.
95+
increment_id = find(snapshot, role='AXButton', label='Increment')['elementId']
96+
mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Add row')['elementId'])
97+
snapshot = state()
98+
assert any(node.get('value', node.get('label')) == 'Added row' for node in snapshot['nodes']), 'Add row did not insert its label'
99+
assert find(snapshot, role='AXButton', label='Increment')['elementId'] == increment_id, 'An element changed id when another was added above it'
94100
baseline = int(next(node.get('value', node.get('label', '')) for node in snapshot['nodes'] if node.get('value', node.get('label', '')).startswith('Count: ')).split(': ')[1])
95101
button = find(snapshot, role='AXButton', label='Increment')
96102
snapshot = mutate('click', snapshot, elementId=button['elementId'])
@@ -212,7 +218,7 @@ def batch(snapshot, steps):
212218
assert error['dispatchState'] == 'not_started', error
213219
assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'replaced', 'A background app was clicked or typed into'
214220
call('activate_app', bundleId=bundle)
215-
print('PASS: real fixture state, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
221+
print('PASS: real fixture state, element ids stable across observations, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
216222
finally:
217223
if fixture:
218224
fixture.terminate()

0 commit comments

Comments
 (0)