Skip to content

Commit 0156813

Browse files
authored
fix(computer-use): stop reporting closed menu items past the cap as a truncated state (#10043)
1 parent 77c9e26 commit 0156813

3 files changed

Lines changed: 11 additions & 3 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,8 +354,9 @@ final class Snapshot {
354354
if role == kAXMenuItemRole && !entry.menu && !entry.priority {
355355
queue.append(AXTraversalEntry(element: element, parent: entry.parent, depth: entry.depth, menu: true, priority: false)); continue
356356
}
357+
// Closed menus past the cap hide no content: opening a menu lists its items as priority entries.
357358
if isMenu && !entry.priority {
358-
guard menuCount < 80 else { truncated = true; continue }
359+
guard menuCount < 80 else { continue }
359360
menuCount += 1
360361
}
361362
let id = "e\(nodes.count)"; elements[id] = element; index.insert(element, id: id)

‎apps/desktop/native/computer-use/tests/Fixture.swift‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,13 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate {
3838
let editItem = NSMenuItem(title: "Edit", action: nil, keyEquivalent: "")
3939
let editMenu = NSMenu(title: "Edit")
4040
editMenu.addItem(withTitle: "Select All", action: #selector(NSText.selectAll(_:)), keyEquivalent: "a")
41-
editItem.submenu = editMenu; menu.addItem(editItem); NSApplication.shared.mainMenu = menu
41+
editItem.submenu = editMenu; menu.addItem(editItem)
42+
// More closed menu items than a snapshot lists, as most real apps have.
43+
let manyItem = NSMenuItem(title: "Many", action: nil, keyEquivalent: "")
44+
let manyMenu = NSMenu(title: "Many")
45+
for index in 1...120 { manyMenu.addItem(withTitle: "Fixture command \(index)", action: nil, keyEquivalent: "") }
46+
manyItem.submenu = manyMenu; menu.addItem(manyItem)
47+
NSApplication.shared.mainMenu = menu
4248
window = NSWindow(contentRect: NSRect(x: 100, y: 100, width: 600, height: 720), styleMask: [.titled, .closable, .resizable], backing: .buffered, defer: false)
4349
window.title = "Mothership Computer Use Test Fixture"
4450
let stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16

‎apps/desktop/native/computer-use/tests/live.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5):
8181
time.sleep(1)
8282
snapshot = state(screenshot=True)
8383
assert 'fixture-only-secret' not in json.dumps(snapshot), 'Secure value leaked'
84+
assert snapshot['truncated'] is False, 'Closed menu items past the snapshot cap marked the state truncated'
8485
infinite = find(snapshot, label='Fixture infinite frame')
8586
assert not {'x', 'y', 'width', 'height'} & infinite.keys(), f'An infinite frame reached the reply: {infinite}'
8687
image = snapshot.get('screenshot')
@@ -211,7 +212,7 @@ def batch(snapshot, steps):
211212
assert error['dispatchState'] == 'not_started', error
212213
assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'replaced', 'A background app was clicked or typed into'
213214
call('activate_app', bundleId=bundle)
214-
print('PASS: real fixture state, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
215+
print('PASS: real fixture state, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
215216
finally:
216217
if fixture:
217218
fixture.terminate()

0 commit comments

Comments
 (0)