Skip to content

Commit 876fc72

Browse files
authored
improvement(desktop-browser): click through a nested control that fills the whole target (#9956)
* improvement(desktop-browser): click through a nested control that fills the whole target * fix(desktop-browser): click through only a native top-page click onto a control that covers the whole target * test(desktop-browser): exercise the form-field click-through guard directly
1 parent 9866b65 commit 876fc72

3 files changed

Lines changed: 104 additions & 8 deletions

File tree

‎apps/desktop/src/main/browser-agent/driver.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3631,6 +3631,9 @@ async function executeToolInner(
36313631
activation,
36323632
element: prepared.element,
36333633
refRecovered: prepared.refRecovered === true,
3634+
...(typeof prepared.clickedThrough === 'string'
3635+
? { clickedThrough: prepared.clickedThrough }
3636+
: {}),
36343637
effectObserved,
36353638
possibleEffectObserved,
36363639
effect,

‎apps/desktop/src/main/browser-agent/page-functions.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -693,6 +693,57 @@ describe('collectSnapshot', () => {
693693
expect(focusElementForTyping(ref)).toMatchObject({ error: 'stale' })
694694
})
695695

696+
it('clicks through a nested control that fills the whole target', () => {
697+
document.body.innerHTML = '<a href="#slot"><span role="button">3:00 PM</span></a>'
698+
const link = visible(document.querySelector('a') as HTMLAnchorElement)
699+
const slot = visible(document.querySelector('span') as HTMLSpanElement)
700+
register(link)
701+
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => slot })
702+
703+
expect(clickElement(0, false)).toMatchObject({ clickedThrough: '3:00 PM' })
704+
})
705+
706+
it('refuses a nested control that leaves the target edges uncovered', () => {
707+
document.body.innerHTML = '<a href="#card"><span role="button">Open</span></a>'
708+
const link = visible(document.querySelector('a') as HTMLAnchorElement)
709+
const inner = document.querySelector('span') as HTMLSpanElement
710+
inner.getBoundingClientRect = () =>
711+
({ x: 15, y: 3, width: 70, height: 14, left: 15, top: 3, right: 85, bottom: 17 }) as DOMRect
712+
register(link)
713+
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => inner })
714+
715+
expect(clickElement(0, false)).toMatchObject({ error: 'nested-control' })
716+
})
717+
718+
it('refuses to click through to a form field or through a synthetic fallback', () => {
719+
document.body.innerHTML = '<div id="wrap" role="button"><input type="text" /></div>'
720+
const wrap = visible(document.querySelector('#wrap') as HTMLDivElement)
721+
const input = visible(document.querySelector('input') as HTMLInputElement)
722+
register(wrap)
723+
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => input })
724+
expect(clickElement(0, false)).toMatchObject({ error: 'nested-control' })
725+
726+
document.body.innerHTML = '<a href="#slot"><span role="button">3:00 PM</span></a>'
727+
const link = visible(document.querySelector('a') as HTMLAnchorElement)
728+
const slot = visible(document.querySelector('span') as HTMLSpanElement)
729+
register(link)
730+
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => slot })
731+
expect(clickElement(0)).toMatchObject({ error: 'nested-control' })
732+
})
733+
734+
it('still refuses a nested control that covers only part of the target', () => {
735+
document.body.innerHTML = '<a href="#card">Card title <button>Save</button></a>'
736+
const link = visible(document.querySelector('a') as HTMLAnchorElement)
737+
const save = visible(document.querySelector('button') as HTMLButtonElement)
738+
register(link)
739+
Object.defineProperty(document, 'elementFromPoint', {
740+
configurable: true,
741+
value: (x: number) => (x <= 50 ? save : document.body),
742+
})
743+
744+
expect(clickElement(0)).toMatchObject({ error: 'nested-control' })
745+
})
746+
696747
it('refuses to recover a ref when replacement is ambiguous', () => {
697748
document.body.innerHTML = '<button>Close</button>'
698749
const original = visible(document.querySelector('button') as HTMLButtonElement)

‎apps/desktop/src/main/browser-agent/page-functions.ts‎

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1382,6 +1382,8 @@ export function clickElement(
13821382
let clientY = 0
13831383
let blocker: Element | null = null
13841384
const blockedHits: Array<Element | null> = []
1385+
let firstBlockedPoint: { x: number; y: number } | null = null
1386+
let filledByControl: Element | null = null
13851387
let foundPoint = false
13861388
const blockerLabel = (element: Element | null): string =>
13871389
(
@@ -1409,6 +1411,7 @@ export function clickElement(
14091411
break
14101412
}
14111413
blocker ??= hit
1414+
firstBlockedPoint ??= { x, y }
14121415
blockedHits.push(hit)
14131416
}
14141417
if (foundPoint) break
@@ -1511,21 +1514,59 @@ export function clickElement(
15111514
break
15121515
}
15131516
}
1514-
if (nested) {
1515-
let control = blocker
1517+
const nestedControlOf = (hit: Element | null): Element | null => {
1518+
let control = hit
15161519
while (control && control !== el && !isIndependentInteractive(control)) {
15171520
control = composedParent(control)
15181521
}
1522+
return control && control !== el ? control : null
1523+
}
1524+
const control = nested ? nestedControlOf(blocker) : null
1525+
// One nested control filling the whole target (a time-slot label inside
1526+
// its link) is what a person clicking the target hits, so a native click
1527+
// on the top page goes there. Form fields and disabled controls keep
1528+
// their own refusals, and any uncovered edge leaves the target clickable
1529+
// in its own right, so both refuse.
1530+
const fillsTarget = (node: Element): boolean => {
1531+
const bounds = node.getBoundingClientRect()
1532+
return rects.every(
1533+
(rect) =>
1534+
bounds.left <= rect.left + 1 &&
1535+
bounds.top <= rect.top + 1 &&
1536+
bounds.right >= rect.right - 1 &&
1537+
bounds.bottom >= rect.bottom - 1
1538+
)
1539+
}
1540+
const controlTag = control ? String(control.tagName || '').toUpperCase() : ''
1541+
if (
1542+
control &&
1543+
firstBlockedPoint &&
1544+
!dispatchSynthetic &&
1545+
!focusForKeyboard &&
1546+
el.ownerDocument.defaultView === window &&
1547+
window.top === window &&
1548+
!['INPUT', 'TEXTAREA', 'SELECT'].includes(controlTag) &&
1549+
!(control as HTMLElement).isContentEditable &&
1550+
!(control as HTMLButtonElement).disabled &&
1551+
control.getAttribute('aria-disabled') !== 'true' &&
1552+
fillsTarget(control) &&
1553+
blockedHits.every((hit) => hit !== null && nestedControlOf(hit) === control)
1554+
) {
1555+
clientX = firstBlockedPoint.x
1556+
clientY = firstBlockedPoint.y
1557+
filledByControl = control
1558+
} else if (nested) {
15191559
return {
15201560
error: 'nested-control',
15211561
blocker: blockerLabel(blocker),
1522-
controlId: control && control !== el ? window.__simAgentRefOf(control) : null,
1562+
controlId: control ? window.__simAgentRefOf(control) : null,
1563+
}
1564+
} else {
1565+
return {
1566+
error: 'obstructed',
1567+
blocker: blockerLabel(blocker),
1568+
blockerControls: window.__simAgentOverlayControls(blocker),
15231569
}
1524-
}
1525-
return {
1526-
error: 'obstructed',
1527-
blocker: blockerLabel(blocker),
1528-
blockerControls: window.__simAgentOverlayControls(blocker),
15291570
}
15301571
}
15311572

@@ -1636,6 +1677,7 @@ export function clickElement(
16361677
frameDepth,
16371678
focusSucceeded: focusForKeyboard && el.ownerDocument.activeElement === el,
16381679
refRecovered: resolved?.recovered === true,
1680+
...(filledByControl ? { clickedThrough: blockerLabel(filledByControl) } : {}),
16391681
}
16401682
}
16411683

0 commit comments

Comments
 (0)