Skip to content

Commit 82f8c35

Browse files
authored
fix(desktop-browser): refuse reissued earlier refs and recover from uncommitted ref reuse (#9959)
* fix(desktop-browser): refuse reissued earlier refs and recover from uncommitted ref reuse * fix(desktop-browser): keep new markers and frame checks across a recapture
1 parent 4388745 commit 82f8c35

4 files changed

Lines changed: 152 additions & 42 deletions

File tree

‎apps/desktop/src/main/browser-agent/driver.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3152,6 +3152,33 @@ describe('credential protection', () => {
31523152
})
31533153
})
31543154

3155+
it('recaptures with fresh ids when the page reuses an id this scope never committed', async () => {
3156+
const contents = await openPage()
3157+
const snapshot = (refIds: number[], nextElementId: number) => ({
3158+
url: 'https://example.com/',
3159+
title: 'Example',
3160+
outline: refIds.map((ref) => `- button "B${ref}" [ref=${ref}]`).join('\n'),
3161+
truncated: false,
3162+
refIds,
3163+
refLineIndexes: Object.fromEntries(refIds.map((ref, index) => [ref, index])),
3164+
nextElementId,
3165+
})
3166+
respondWith(contents, { collectSnapshot: snapshot([4, 5], 6) })
3167+
await driver.executeTool('chat-test', 'browser_snapshot', {})
3168+
vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
3169+
if (!isPageCall(expression, 'collectSnapshot')) return Promise.resolve(undefined)
3170+
const forgets = expression.includes(',true])')
3171+
return Promise.resolve(forgets ? snapshot([6, 7], 8) : snapshot([2, 6], 7))
3172+
})
3173+
3174+
const result = await driver.executeTool('chat-test', 'browser_snapshot', {})
3175+
3176+
expect(result).toMatchObject({
3177+
ok: true,
3178+
result: { outline: expect.stringContaining('[ref=7]') },
3179+
})
3180+
})
3181+
31553182
it('acts on a ref an earlier snapshot of the same page issued', async () => {
31563183
const contents = await openPage()
31573184
respondWith(contents, {

‎apps/desktop/src/main/browser-agent/driver.ts‎

Lines changed: 55 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2496,28 +2496,36 @@ async function captureSnapshot(
24962496
}
24972497

24982498
const mainStartingElementId = state.nextElementRefId
2499-
const mainSnapshot = unwrapPageResult(
2500-
await execInPage(
2501-
contents,
2502-
collectSnapshot,
2503-
[mainStartingElementId, elementId ?? null, markNew, state.generation],
2504-
false,
2505-
notAfter
2499+
// A page reuses ids it issued in a capture this scope never committed (an
2500+
// aborted or timed-out snapshot); when validation cannot vouch for one, the
2501+
// page forgets its issued refs and the capture runs once more.
2502+
const captureMain = async (forgetIssuedRefs: boolean) => {
2503+
const snapshot = unwrapPageResult(
2504+
await execInPage(
2505+
contents,
2506+
collectSnapshot,
2507+
[mainStartingElementId, elementId ?? null, markNew, state.generation, forgetIssuedRefs],
2508+
false,
2509+
notAfter
2510+
)
25062511
)
2507-
)
2508-
if (!stillCurrent()) {
2509-
throw new ToolError('The tab changed while its snapshot was being captured. Try again.')
2510-
}
2511-
if (
2512-
!isRecordLike(mainSnapshot) ||
2513-
typeof mainSnapshot.outline !== 'string' ||
2514-
!Array.isArray(mainSnapshot.refIds)
2515-
) {
2516-
throw new ToolError('The page returned an incomplete snapshot. Try browser_snapshot again.')
2512+
if (!stillCurrent()) {
2513+
throw new ToolError('The tab changed while its snapshot was being captured. Try again.')
2514+
}
2515+
if (
2516+
!isRecordLike(snapshot) ||
2517+
typeof snapshot.outline !== 'string' ||
2518+
!Array.isArray(snapshot.refIds)
2519+
) {
2520+
throw new ToolError('The page returned an incomplete snapshot. Try browser_snapshot again.')
2521+
}
2522+
const refs = validateSnapshotRefs(snapshot, mainStartingElementId, (ref) =>
2523+
samePageTarget(state.issuedRefTargets.get(ref), contents)
2524+
)
2525+
return { snapshot, refs }
25172526
}
2518-
const mainRefs = validateSnapshotRefs(mainSnapshot, mainStartingElementId, (ref) =>
2519-
samePageTarget(state.issuedRefTargets.get(ref), contents)
2520-
)
2527+
let { snapshot: mainSnapshot, refs: mainRefs } = await captureMain(false)
2528+
if (!mainRefs) ({ snapshot: mainSnapshot, refs: mainRefs } = await captureMain(true))
25212529
if (!mainRefs) {
25222530
throw new ToolError('The page returned invalid element ids. Try browser_snapshot again.')
25232531
}
@@ -2556,13 +2564,15 @@ async function captureSnapshot(
25562564
}
25572565
const frameUrl = frame.url
25582566
const frameStartingElementId = nextElementId
2559-
const frameSnapshot = await execInPage(
2560-
frame,
2561-
collectSnapshot,
2562-
[frameStartingElementId, null, markNew, state.generation],
2563-
false,
2564-
notAfter
2565-
)
2567+
const captureFrame = (forgetIssuedRefs: boolean) =>
2568+
execInPage(
2569+
frame,
2570+
collectSnapshot,
2571+
[frameStartingElementId, null, markNew, state.generation, forgetIssuedRefs],
2572+
false,
2573+
notAfter
2574+
)
2575+
let frameSnapshot = await captureFrame(false)
25662576
if (!stillCurrent()) {
25672577
throw new ToolError('The tab changed while its frames were being captured. Try again.')
25682578
}
@@ -2578,10 +2588,25 @@ async function captureSnapshot(
25782588
unreadableCrossOriginFrames++
25792589
continue
25802590
}
2581-
const frameRefs = validateSnapshotRefs(frameSnapshot, frameStartingElementId, (ref) =>
2582-
samePageTarget(state.issuedRefTargets.get(ref), frame)
2583-
)
2591+
const validateFrame = (snapshot: Record<string, unknown>) =>
2592+
validateSnapshotRefs(snapshot, frameStartingElementId, (ref) =>
2593+
samePageTarget(state.issuedRefTargets.get(ref), frame)
2594+
)
2595+
let frameRefs = validateFrame(frameSnapshot)
25842596
if (!frameRefs) {
2597+
frameSnapshot = await captureFrame(true)
2598+
if (!stillCurrent()) {
2599+
throw new ToolError('The tab changed while its frames were being captured. Try again.')
2600+
}
2601+
if (frame.isDestroyed() || frame.detached || frame.url !== frameUrl) {
2602+
unreadableCrossOriginFrames++
2603+
continue
2604+
}
2605+
if (isRecordLike(frameSnapshot) && Array.isArray(frameSnapshot.refIds)) {
2606+
frameRefs = validateFrame(frameSnapshot)
2607+
}
2608+
}
2609+
if (!frameRefs || !isRecordLike(frameSnapshot) || typeof frameSnapshot.outline !== 'string') {
25852610
unreadableCrossOriginFrames++
25862611
continue
25872612
}

‎apps/desktop/src/main/browser-agent/page-functions.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ beforeEach(() => {
117117
window.__simAgentMutationStates = undefined
118118
window.__simAgentNextElementId = 0
119119
window.__simAgentShownElements = undefined
120+
window.__simAgentShownPending = undefined
120121
installPageHelpers()
121122
window.__simAgentResolveElement = undefined
122123
installDomShims()
@@ -542,6 +543,19 @@ describe('collectSnapshot', () => {
542543
expect(outlineOf(collectSnapshot())).not.toContain(' new')
543544
})
544545

546+
it('keeps new markers for a recapture that replaces a discarded capture', () => {
547+
document.body.innerHTML = '<button>Compose</button>'
548+
visible(document.querySelector('button') as HTMLButtonElement)
549+
collectSnapshot()
550+
const added = document.createElement('button')
551+
added.textContent = 'Send'
552+
document.body.append(visible(added))
553+
collectSnapshot()
554+
555+
const lines = outlineOf(collectSnapshot(0, null, true, 0, true)).split('\n')
556+
expect(lines.find((line) => line.includes('"Send"'))).toMatch(/ new$/)
557+
})
558+
545559
it('leaves new markers out of an unmarked read without consuming them', () => {
546560
document.body.innerHTML = '<button>Compose</button>'
547561
visible(document.querySelector('button') as HTMLButtonElement)
@@ -814,6 +828,28 @@ describe('collectSnapshot', () => {
814828
expect(refFor(outlineOf(collectSnapshot()), 'Delete')).not.toBe(first)
815829
})
816830

831+
it('refuses an earlier ref once a later snapshot reissued its recycled element', () => {
832+
document.body.innerHTML = '<div role="row"><span>Order 1</span><button>Archive</button></div>'
833+
for (const el of Array.from(document.querySelectorAll('span, button'))) visible(el)
834+
const first = refFor(outlineOf(collectSnapshot()), 'Archive')
835+
;(document.querySelector('span') as HTMLSpanElement).textContent = 'Order 2'
836+
const second = refFor(outlineOf(collectSnapshot()), 'Archive')
837+
collectSnapshot(500, second)
838+
839+
expect(second).not.toBe(first)
840+
expect(clickElement(first)).toMatchObject({ error: 'stale' })
841+
})
842+
843+
it('does not take an icon-only control as revealing content when its label grows', () => {
844+
document.body.innerHTML = '<button aria-label="Remove"></button>'
845+
const button = visible(document.querySelector('button') as HTMLButtonElement)
846+
const ref = refFor(outlineOf(collectSnapshot()), 'Remove')
847+
848+
button.setAttribute('aria-label', 'Remove member')
849+
850+
expect(clickElement(ref)).toMatchObject({ error: 'stale' })
851+
})
852+
817853
it('invalidates a connected ref whose name changed to a longer, different word', () => {
818854
document.body.innerHTML = '<div role="listitem">Item 1</div>'
819855
const row = visible(document.querySelector('[role="listitem"]') as HTMLDivElement)

‎apps/desktop/src/main/browser-agent/page-functions.ts‎

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ declare global {
6565
__simAgentRefOfElement?: WeakMap<Element, number>
6666
/** Elements an earlier snapshot of this document listed; a later snapshot marks the rest `new`. */
6767
__simAgentShownElements?: WeakSet<Element>
68+
/** Elements the latest capture listed, committed to the shown set by the next capture that keeps them. */
69+
__simAgentShownPending?: Element[]
6870
/** Installed by {@link installPageHelpers} before every page function; cached per call. */
6971
__simAgentIsExemptModal: (element: Element) => boolean
7072
/** Installed by {@link installPageHelpers}: the element's id in the current snapshot. */
@@ -231,7 +233,8 @@ export function collectSnapshot(
231233
startingElementId = 0,
232234
elementId: number | null = null,
233235
markNew = true,
234-
refNamespace = 0
236+
refNamespace = 0,
237+
forgetIssuedRefs = false
235238
): unknown {
236239
const resolver = window.__simAgentResolveElement
237240
const scopedRoot =
@@ -331,14 +334,22 @@ export function collectSnapshot(
331334
const refLineIndexes: Record<number, number> = {}
332335
const locators: SnapshotRefLocator[] = []
333336
window.__simAgentElements = registry
334-
const previouslyShown = window.__simAgentShownElements
335-
const shown = previouslyShown ?? new WeakSet<Element>()
336-
if (markNew) window.__simAgentShownElements = shown
337+
// A capture the driver discards and recaptures must not use up `new`
338+
// markers, so its listed elements commit only when the next capture keeps them.
339+
const pendingShown = window.__simAgentShownPending
340+
if (markNew && pendingShown && !forgetIssuedRefs) {
341+
const committed = window.__simAgentShownElements ?? new WeakSet<Element>()
342+
for (const el of pendingShown) committed.add(el)
343+
window.__simAgentShownElements = committed
344+
}
345+
const shown = window.__simAgentShownElements
346+
const listed: Element[] = []
347+
if (markNew) window.__simAgentShownPending = listed
337348
/** Whether no earlier snapshot of this document listed the element; the first snapshot marks nothing. */
338-
const isNew = (el: Element): boolean => markNew && previouslyShown !== undefined && !shown.has(el)
349+
const isNew = (el: Element): boolean => markNew && shown !== undefined && !shown.has(el)
339350
/** Records an element whose line made it into the outline, so the next snapshot knows it. */
340351
const recordShown = (el: Element): void => {
341-
if (markNew) shown.add(el)
352+
if (markNew) listed.push(el)
342353
}
343354
const lines: string[] = []
344355
let truncated = false
@@ -603,9 +614,11 @@ export function collectSnapshot(
603614

604615
// An unchanged element keeps its ref, and a ref stays resolvable after a
605616
// later snapshot leaves it out; a changed element gets a new ref.
606-
const issuedRefs = window.__simAgentIssuedRefs ?? new Map()
617+
// The driver asks the page to forget when it cannot vouch for a reused id
618+
// (a capture it never committed handed it out), so every element gets a new one.
619+
const issuedRefs = (!forgetIssuedRefs && window.__simAgentIssuedRefs) || new Map()
607620
window.__simAgentIssuedRefs = issuedRefs
608-
const refOfElement = window.__simAgentRefOfElement ?? new WeakMap()
621+
const refOfElement = (!forgetIssuedRefs && window.__simAgentRefOfElement) || new WeakMap()
609622
window.__simAgentRefOfElement = refOfElement
610623
const issuedRefCap = 5000
611624
const textHashOf = (el: Element): number => {
@@ -958,8 +971,10 @@ export function collectSnapshot(
958971
// was revealed (its DOM text is unchanged) or it declares a popup. A
959972
// recycled row changes its DOM text with neither.
960973
const name = nameFor(candidate)
961-
const opensPopup =
962-
textHashOf(candidate) === locator.textHash ||
974+
const revealedContent = (): boolean =>
975+
Boolean(candidate.textContent?.trim()) && textHashOf(candidate) === locator.textHash
976+
const opensPopup = (): boolean =>
977+
revealedContent() ||
963978
['aria-haspopup', 'aria-expanded', 'aria-controls'].some(
964979
(attribute) => (candidate.getAttribute(attribute) || 'false') !== 'false'
965980
) ||
@@ -968,10 +983,11 @@ export function collectSnapshot(
968983
) !== null
969984
const nameGrewInPlace =
970985
connected &&
971-
opensPopup &&
986+
name !== locator.name &&
972987
locator.name.length > 0 &&
973988
name.startsWith(locator.name) &&
974-
(locator.name.length >= 120 || /\s/.test(name.charAt(locator.name.length)))
989+
(locator.name.length >= 120 || /\s/.test(name.charAt(locator.name.length))) &&
990+
opensPopup()
975991
if (name !== locator.name && !nameGrewInPlace) return false
976992
const candidateAttributes = locatorAttributes(candidate)
977993
const attributes = connected ? connectedIdentityAttributes : stableAttributes
@@ -1052,6 +1068,12 @@ export function collectSnapshot(
10521068
}
10531069

10541070
const current = registry[id] ?? issued?.element.deref()
1071+
// An earlier ref whose element has since been issued a new ref (it changed
1072+
// identity in between, e.g. a recycled row) no longer names it.
1073+
if (!inLatestSnapshot && current && refOfElement.get(current) !== id) {
1074+
window.__simAgentStaleReason = 'the element was given a new ref by a later snapshot'
1075+
return null
1076+
}
10551077
if (current?.isConnected) {
10561078
if (!identityMatches(current, true)) {
10571079
window.__simAgentStaleReason = `the node is still mounted but its identity drifted (now <${String(current.tagName || '').toLowerCase()}> role=${roleFor(current) || 'none'} name="${nameFor(current).slice(0, 60)}")`

0 commit comments

Comments
 (0)