diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8499b5b..fcef8478 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -242,6 +242,12 @@ jobs: BSK_CLICK_CHROME: google-chrome run: pnpm exec vitest run src/tools/__tests__/emulation-lifecycle.browser.test.ts + - name: Run fill browser regression + working-directory: apps/extension + env: + BSK_FILL_CHROME: google-chrome + run: pnpm exec vitest run src/tools/__tests__/fill.browser.test.ts + node-scripts: name: Node script tests runs-on: ubuntu-latest diff --git a/apps/extension/src/tools/__tests__/fill.browser.test.ts b/apps/extension/src/tools/__tests__/fill.browser.test.ts new file mode 100644 index 00000000..10e98eb7 --- /dev/null +++ b/apps/extension/src/tools/__tests__/fill.browser.test.ts @@ -0,0 +1,508 @@ +// @vitest-environment node +// Opt in with BSK_FILL_CHROME=/path/to/chrome; the suite owns its browser/profile and each test gets a fresh tab. +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { CdpFrame } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import { handleFill } from "../interaction"; +import { handleObserve } from "../observation"; +import type { CdpRunner } from "../shared"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; + +let browserSend: Send; +let closeBrowser: (() => void) | undefined; +let browserRun: Promise | undefined; + +if (process.env.BSK_FILL_CHROME) { + beforeAll(async () => { + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + const ready = Promise.withResolvers(); + const closed = Promise.withResolvers(); + closeBrowser = () => closed.resolve(); + browserRun = withChrome( + { executable: process.env.BSK_FILL_CHROME, deviceScale: 1, zoom: 1 }, + async (send: Send) => { + ready.resolve(send); + await closed.promise; + }, + ); + browserRun!.catch(ready.reject); + browserSend = await ready.promise; + }, 30_000); + afterAll(async () => { + closeBrowser?.(); + await browserRun; + }, 30_000); +} + +async function withFillBrowser( + { background, markup }: { background: boolean; markup: string }, + run: (h: { + evaluate: (expression: string) => Promise; + ref: (expression: string) => Promise; + fill: ( + value: string, + clearBefore?: boolean, + signal?: AbortSignal, + selector?: string, + ref?: string, + ) => ReturnType; + afterCommand: (hook: (method: string, params?: object) => void | Promise) => void; + calls: string[]; + observe: () => ReturnType; + }) => Promise, +) { + const send = browserSend; + // Closing each target restores the launcher's blank foreground page. + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + background, + }); + try { + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + if (!background) await send("Page.bringToFront", {}, sessionId); + const evaluate = async (expression: string) => { + const reply = await send<{ result: { value: unknown }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { expression, returnByValue: true }, + sessionId, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + await evaluate(`document.body.innerHTML = ${JSON.stringify(markup)}`); + expect(await evaluate("document.hasFocus()")).toBe(!background); + const manager = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [] }), + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + const calls: string[] = []; + let afterCommand: ((method: string, params?: object) => void | Promise) | undefined; + const cdp: CdpRunner = { + send: async (_tabId, method, params) => { + calls.push(method); + const result = await send(method, params, sessionId); + await afterCommand?.(method, params); + return result as never; + }, + }; + cdp.sendToTarget = (_target, method, params) => cdp.send(4, method, params); + cdp.getAttachmentId = () => sessionId; + cdp.getFrameGraph = async () => { + type FrameTree = { frame: { id: string }; childFrames?: FrameTree[] }; + const { frameTree } = await send<{ frameTree: FrameTree }>( + "Page.getFrameTree", + {}, + sessionId, + ); + const frames: CdpFrame[] = []; + const visit = async (tree: FrameTree, parentFrameId?: string) => { + const owner = parentFrameId + ? await send<{ backendNodeId: number }>( + "DOM.getFrameOwner", + { frameId: tree.frame.id }, + sessionId, + ) + : undefined; + frames.push({ + frameId: tree.frame.id, + parentFrameId, + target: { tabId: 4 }, + ownerBackendNodeId: owner?.backendNodeId, + }); + for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id); + }; + await visit(frameTree); + return { rootFrameId: frameTree.frame.id, frames }; + }; + const deps = { + cdp, + tabsApi: { + get: async (id: number) => ({ id, windowId: 100, active: !background }) as chrome.tabs.Tab, + query: async () => [{ id: 4, windowId: 100, active: !background } as chrome.tabs.Tab], + }, + }; + await run({ + evaluate, + calls, + observe: () => + handleObserve(manager, { session_id: "aa11" }, { ...deps, conditionalSurfaceProbe: false }), + afterCommand: (hook) => { + afterCommand = hook; + }, + ref: async (expression) => { + const { result } = await send<{ result: { objectId: string } }>( + "Runtime.evaluate", + { expression }, + sessionId, + ); + const { node } = await send<{ node: { backendNodeId: number } }>( + "DOM.describeNode", + { objectId: result.objectId }, + sessionId, + ); + await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId); + ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 }); + }, + fill: (value, clearBefore, signal, selector, ref = "e1") => + handleFill( + manager, + { + session_id: "aa11", + ...(selector ? { selector } : { ref }), + value, + clear_before: clearBefore, + }, + { ...deps, signal }, + ), + }); + } finally { + await send("Target.closeTarget", { targetId }); + } +} + +const { acceptedTargets, rejectedTargets, editorMarkup } = await import( + new URL( + "../../../../../evals/browser/cases/regression/fill-editor-roots/fill-editor-roots.fixture.mjs", + import.meta.url, + ).href +); + +type AcceptedTarget = { name: string; markup: string; steps: [string, boolean, string, string?][] }; +type RejectedTarget = { name: string; markup: string }; +const guard = '

outside text

'; +const outsideTarget = `(() => { + const body = document.body.cloneNode(true); + body.querySelector('#target').replaceWith(document.createComment('fill target')); + return body.innerHTML; +})()`; + +const unchangedState = `JSON.stringify({ + html: document.body.innerHTML, + focus: document.activeElement.id, + selection: [document.querySelector('#guard').selectionStart, document.querySelector('#guard').selectionEnd], + scroll: [scrollX, scrollY], events: window.fillEvents +})`; + +describe.skipIf(!process.env.BSK_FILL_CHROME)("native controls and editing hosts", () => { + describe.each([ + ["foreground", false], + ["background", true], + ] as const)("%s", (_name, background) => { + it.each(acceptedTargets as AcceptedTarget[])("fills $name", async ({ markup, steps }) => { + await withFillBrowser({ background, markup: guard + markup }, async (h) => { + await h.ref("document.querySelector('#target')"); + const outsideBefore = await h.evaluate(outsideTarget); + for (const [value, clearBefore, expected, domText] of steps) { + const result = await h.fill(value, clearBefore); + expect( + result, + JSON.stringify({ + value, + state: await h.evaluate( + "({ html: document.querySelector('#target').innerHTML, text: document.querySelector('#target').innerText, content: document.querySelector('#target').textContent })", + ), + }), + ).toMatchObject({ value_length: expected.length }); + expect(await h.evaluate(outsideTarget)).toBe(outsideBefore); + // innerText exposes line breaks introduced by browser editing, while + // textContent preserves invisible characters that cleanup must not erase. + const actual = await h.evaluate(`(() => { + const t = document.querySelector('#target'); + if (t.matches('input,textarea')) return t.value; + const selection = document.getSelection(); + selection.selectAllChildren(t); + const selected = selection.toString(); + selection.removeAllRanges(); + return { text: t.innerText, content: t.textContent, selected }; + })()`); + if (typeof actual === "string") expect(actual).toBe(expected); + else { + const { content, selected } = actual as { content: string; selected: string }; + if (domText !== undefined) expect(content).toBe(domText); + else expect(selected).toBe(expected); + if (expected === "\u200b") expect(content).toBe(expected); + } + expect(await h.evaluate("document.querySelector('#guard').value")).toBe("untouched"); + expect(await h.evaluate("document.querySelector('#outside').textContent")).toBe( + "outside text", + ); + } + }); + }, 30_000); + + it.each(rejectedTargets as RejectedTarget[])("rejects $name without side effects", async ({ + markup, + }) => { + await withFillBrowser( + { background, markup: guard + '
' + markup }, + async (h) => { + await h.ref("document.querySelector('#target')"); + await h.evaluate(`document.querySelector('#guard').focus(); document.querySelector('#guard').setSelectionRange(1, 3); + window.fillEvents = []; for (const type of ['focus', 'blur', 'input', 'change']) document.addEventListener(type, e => window.fillEvents.push([type, e.target.id]), true);`); + const before = await h.evaluate(unchangedState); + for (const [value, clearBefore] of [ + ["hello", true], + ["", true], + ["hello", false], + ] as const) { + expect(await h.fill(value, clearBefore)).toMatchObject({ + code: "invalid_params", + data: { reason: "target_not_fillable" }, + }); + expect(await h.evaluate(unchangedState)).toBe(before); + } + expect(h.calls).not.toContain("DOM.focus"); + expect(h.calls).not.toContain("DOM.scrollIntoViewIfNeeded"); + expect(h.calls).not.toContain("Input.insertText"); + }, + ); + }, 30_000); + + it.each([ + ['', "invalid"], + ['', "longer"], + ])( + "rejects invalid values before focus: %s", + async (markup, value) => { + await withFillBrowser({ background, markup: guard + markup }, async (h) => { + await h.ref("document.querySelector('#target')"); + await h.evaluate("document.querySelector('#guard').focus(); window.fillEvents = []"); + const before = await h.evaluate(unchangedState); + expect(await h.fill(value)).toMatchObject({ + code: "invalid_params", + data: { reason: "fill_value_invalid" }, + }); + expect(await h.evaluate(unchangedState)).toBe(before); + expect(h.calls).not.toContain("DOM.focus"); + }); + }, + 30_000, + ); + + it("uses the same contract for selectors", async () => { + await withFillBrowser({ background, markup: editorMarkup }, async (h) => { + expect(await h.fill("hello", true, undefined, "#paragraph")).toMatchObject({ + data: { reason: "target_not_fillable" }, + }); + expect(await h.fill("hello", true, undefined, "#editor")).toMatchObject({ + value_length: 5, + }); + expect(await h.evaluate("document.querySelector('#editor').textContent")).toBe("hello"); + }); + }, 30_000); + }); +}); + +describe.skipIf(!process.env.BSK_FILL_CHROME)("fill lifecycle and discoverability", () => { + it.each([ + "", + "true", + "plaintext-only", + ])("fills the editor ref from observation: %s", async (attribute) => { + await withFillBrowser( + { + background: false, + markup: editorMarkup.replace( + "contenteditable aria-label", + `contenteditable="${attribute}" aria-label`, + ), + }, + async (h) => { + const observation = await h.observe(); + expect(observation).not.toHaveProperty("code"); + if (!("text" in observation)) throw new Error("observation failed"); + const ref = observation.text.match(/(@e\d+) textbox "Message editor"/)?.[1]; + expect(ref, observation.text).toBeDefined(); + expect(await h.fill("from observation", true, undefined, undefined, ref)).toMatchObject({ + value_length: 16, + }); + expect(await h.evaluate("document.querySelector('#editor').textContent")).toBe( + "from observation", + ); + }, + ); + }, 30_000); + + it.each(["iframe", "shadow"])("fills roots and rejects descendants in %s", async (kind) => { + await withFillBrowser( + { background: false, markup: guard + '
' }, + async (h) => { + const markup = + '
old

keep

'; + const root = + kind === "iframe" + ? "document.querySelector('iframe').contentDocument" + : "document.querySelector('#container').shadowRoot"; + await h.evaluate( + kind === "iframe" + ? `(() => { const f = document.createElement('iframe'); document.body.append(f); f.contentDocument.body.innerHTML = ${JSON.stringify(markup)}; })()` + : `document.querySelector('#container').attachShadow({mode:'open'}).innerHTML = ${JSON.stringify(markup)}`, + ); + await h.ref(`${root}.querySelector('#paragraph')`); + const before = await h.evaluate(`${root}.querySelector('#editor').innerHTML`); + expect(await h.fill("wrong")).toMatchObject({ data: { reason: "target_not_fillable" } }); + expect(await h.evaluate(`${root}.querySelector('#editor').innerHTML`)).toBe(before); + await h.ref(`${root}.querySelector('#editor')`); + expect(await h.fill("hello")).toMatchObject({ value_length: 5 }); + expect(await h.fill("!", false)).toMatchObject({ value_length: 6 }); + expect(await h.evaluate(`${root}.querySelector('#editor').textContent`)).toBe("hello!"); + }, + ); + }, 30_000); + + it.each([ + ["DOM.resolveNode", "old"], + ["DOM.scrollIntoViewIfNeeded", "old"], + ["DOM.focus", "old"], + ["cleared", ""], + ["Input.insertText", "hello"], + ["notified", "hello"], + ["verified", "hello"], + ])( + "cancels after %s without leaving temporary content", + async (phase, expected) => { + await withFillBrowser( + { background: false, markup: guard + '
old
' }, + async (h) => { + await h.ref("document.querySelector('#target')"); + const controller = new AbortController(); + h.afterCommand((method, params) => { + const declaration = + (params as { functionDeclaration?: string })?.functionDeclaration ?? ""; + const matched = + method === phase || + (phase === "cleared" && declaration.includes("this.textContent = ''")) || + (phase === "notified" && declaration.includes("new Event('change'")) || + (phase === "verified" && declaration.startsWith("function(expected)")); + if (matched) controller.abort(); + }); + expect(await h.fill("hello", true, controller.signal)).toMatchObject({ + code: "cancelled", + }); + expect(await h.evaluate("document.querySelector('#target').textContent")).toBe(expected); + expect(await h.evaluate("document.querySelector('#guard').value")).toBe("untouched"); + expect(h.calls.filter((method) => method === "Runtime.releaseObject")).toHaveLength(1); + if (expected !== "hello") expect(h.calls).not.toContain("Input.insertText"); + }, + ); + }, + 30_000, + ); + + it.each([ + "focus", + "input", + "change", + ])("does not report success when a %s handler replaces the editor", async (event) => { + await withFillBrowser( + { background: false, markup: guard + '
old
' }, + async (h) => { + await h.ref("document.querySelector('#target')"); + await h.evaluate(`document.querySelector('#target').addEventListener(${JSON.stringify(event)}, event => { + const replacement = event.target.cloneNode(true); replacement.textContent = 'replacement'; event.target.replaceWith(replacement); + }, {once:true})`); + expect(await h.fill("hello")).toHaveProperty("code"); + expect(await h.evaluate("document.querySelector('#target').textContent")).toBe( + "replacement", + ); + expect(await h.evaluate("document.querySelector('#guard').value")).toBe("untouched"); + if (event !== "change") expect(h.calls).not.toContain("Input.insertText"); + }, + ); + }, 30_000); + + it("rechecks a root that becomes a descendant during focus", async () => { + await withFillBrowser( + { + background: false, + markup: guard + '
old
', + }, + async (h) => { + await h.ref("document.querySelector('#target')"); + await h.evaluate( + "document.querySelector('#target').addEventListener('focus', () => document.querySelector('#wrapper').contentEditable = 'true', {once:true})", + ); + expect(await h.fill("hello")).toMatchObject({ data: { reason: "target_not_fillable" } }); + expect(await h.evaluate("document.querySelector('#target').textContent")).toBe("old"); + expect(h.calls).not.toContain("Input.insertText"); + }, + ); + }, 30_000); +}); + +describe.skipIf(!process.env.BSK_FILL_CHROME)("fill focus changes", () => { + it.each([ + false, + true, + ])("stops when focus changes after root caret placement (background=%s)", async (background) => { + await withFillBrowser( + { background, markup: guard + '
old
' }, + async (h) => { + await h.ref("document.querySelector('#target')"); + h.afterCommand(async (method, params) => { + const script = params as { + functionDeclaration?: string; + arguments?: { value: unknown }[]; + }; + if ( + method === "Runtime.callFunctionOn" && + script.functionDeclaration?.startsWith("function(before, placeCaret") && + script.arguments?.[1].value === true + ) { + await h.evaluate("document.querySelector('#guard').focus()"); + } + }); + expect(await h.fill("hello", false)).toMatchObject({ data: { reason: "fill_focus_lost" } }); + expect(await h.evaluate("document.querySelector('#guard').value")).toBe("untouched"); + expect(await h.evaluate("document.querySelector('#target').textContent")).toBe("old"); + expect(h.calls).not.toContain("Input.insertText"); + }, + ); + }, 30_000); +}); + +describe.skipIf(!process.env.BSK_FILL_CHROME)("fill caret validation", () => { + it("stops if page code moves the caret within the same editor", async () => { + await withFillBrowser( + { background: false, markup: guard + '
old
' }, + async (h) => { + await h.ref("document.querySelector('#target')"); + h.afterCommand(async (method, params) => { + const script = params as { + functionDeclaration?: string; + arguments?: { value: unknown }[]; + }; + if ( + method === "Runtime.callFunctionOn" && + script.functionDeclaration?.startsWith("function(before, placeCaret") && + script.arguments?.[1].value === true + ) { + await h.evaluate( + "document.getSelection().collapse(document.querySelector('#target'), 0)", + ); + } + }); + expect(await h.fill("hello", false)).toMatchObject({ data: { reason: "fill_focus_lost" } }); + expect(await h.evaluate("document.querySelector('#target').textContent")).toBe("old"); + expect(h.calls).not.toContain("Input.insertText"); + }, + ); + }, 30_000); +}); diff --git a/apps/extension/src/tools/__tests__/fill.test.ts b/apps/extension/src/tools/__tests__/fill.test.ts index 19f95503..112cfe5f 100644 --- a/apps/extension/src/tools/__tests__/fill.test.ts +++ b/apps/extension/src/tools/__tests__/fill.test.ts @@ -321,11 +321,11 @@ describe("fill result verification", () => { const result = await h.fill(); expect(result).toMatchObject({ code: "cdp_failed", data: { reason: "fill_failed" } }); expect(JSON.stringify(result)).not.toContain("page secret"); - if (phase <= 3) expect(h.insert).not.toHaveBeenCalled(); + if (phase <= 4) expect(h.insert).not.toHaveBeenCalled(); expect(h.release).toHaveBeenCalledOnce(); }); - it.each([1, 2, 3, 5])("rejects missing results at phase %s", async (phase) => { + it.each([1, 2, 3, 4, 6])("rejects missing results at phase %s", async (phase) => { const h = await setup(); vi.spyOn(document, "hasFocus").mockReturnValue(false); const run = h.script.getMockImplementation()!; @@ -334,7 +334,7 @@ describe("fill result verification", () => { ++count === phase ? { result: { type: "undefined" } } : run(params), ); expect(await h.fill()).toMatchObject({ code: "cdp_failed" }); - if (phase <= 3) expect(h.insert).not.toHaveBeenCalled(); + if (phase <= 4) expect(h.insert).not.toHaveBeenCalled(); }); it.each(["input", "change"])("checks state after nested microtasks from %s", async (event) => { @@ -394,6 +394,7 @@ describe("fill result verification", () => { h.element.value = normalized; }); expect(await h.fill(requested)).toMatchObject({ value_length: normalized.length }); + expect(h.insert).toHaveBeenCalledWith(requested.replace(/\r\n?/g, "\n")); }); it("does not count an editable padding break as an extra typed character", async () => { @@ -538,7 +539,7 @@ describe("fill result verification", () => { const controller = new AbortController(); h.insert.mockImplementation(async () => controller.abort()); expect(await h.fill("hello", true, controller.signal)).toMatchObject({ code: "cancelled" }); - expect(h.script).toHaveBeenCalledTimes(2); + expect(h.script).toHaveBeenCalledTimes(3); expect(h.release).toHaveBeenCalledOnce(); }); diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index 5d04d202..986a99d4 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -1292,12 +1292,11 @@ function successfulFillScript(params: unknown) { const args = script.arguments ?? []; return { result: { - value: - args.length === 2 - ? { before: "", expected: args[0].value } - : script.functionDeclaration.startsWith("function(expected)") - ? { connected: true, matches: true, valueLength: String(args[0].value).length } - : "ready", + value: script.functionDeclaration.startsWith("function(value, clearBefore") + ? { before: "", expected: args[0].value } + : script.functionDeclaration.startsWith("function(expected)") + ? { connected: true, matches: true, valueLength: String(args[0].value).length } + : "ready", }, }; } @@ -1322,7 +1321,7 @@ describe("handleFill", () => { ctx.refStore.set("e1", 100, { tabId: 4 }); const fake = makeFakeCdp({ "DOM.describeNode": () => ({ - node: { backendNodeId: 100, nodeName: "DIV", attributes: [] }, + node: { backendNodeId: 100, nodeName: "BUTTON", attributes: [] }, }), }); const res = await handleFill( @@ -1365,7 +1364,7 @@ describe("handleFill", () => { expect(insert?.params).toEqual({ text: "hello" }); // Foreground replacement needs no extra caret-positioning round trip. const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn"); - expect(callFns).toHaveLength(4); + expect(callFns).toHaveLength(5); }); it("passes clear_before=false to preparation and verifies the result", async () => { @@ -1380,8 +1379,10 @@ describe("handleFill", () => { "DOM.focus": () => ({}), "DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }), "Runtime.callFunctionOn": (p) => { - const args = (p as { arguments?: Array<{ value: unknown }> }).arguments ?? []; - if (args.length === 2) expect(args[1].value).toBe(false); + const script = p as { arguments?: Array<{ value: unknown }>; functionDeclaration: string }; + if (script.functionDeclaration.startsWith("function(value, clearBefore")) { + expect(script.arguments?.[1].value).toBe(false); + } return successfulFillScript(p); }, "Input.dispatchKeyEvent": () => ({}), @@ -1467,6 +1468,9 @@ describe("handleFill", () => { return {}; }, "DOM.focus": () => ({}), + "DOM.resolveNode": () => ({ object: { objectId: "fill-target" } }), + "Runtime.callFunctionOn": successfulFillScript, + "Runtime.releaseObject": () => ({}), }); const res = await handleFill( diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 0a1af329..a1889d98 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -901,8 +901,8 @@ interface DescribedNode { } /** - * Decide whether a node can receive `tool.fill`: native `` / - * ` +

old paragraph

second paragraph

+
outside text
`; + +export const acceptedTargets = [ + { + name: "line breaks and whitespace", + markup: '
', + steps: [ + ["\n", true, "\n"], + ["\n\n", true, "\n\n"], + ["a\r\nb", true, "a\nb"], + [" ", true, " "], + ], + }, + { + name: "normal whitespace style", + markup: '
', + steps: [ + [" a b ", true, " a b ", "\u00a0a b\u00a0"], + ["!", false, " a b !", "\u00a0a b !"], + ], + }, + { + name: "trailing line break", + markup: '
hello
', + steps: [ + ["!", false, "hello!", "hello!"], + ["\n", false, "hello!\n"], + ], + }, + { + name: "formatted line wrappers", + markup: '
\n
a
\n
b
\n
', + steps: [["!", false, "a\nb!"]], + }, + { + name: "append to existing paragraphs", + markup: '

first

second

', + steps: [["!", false, "first\n\nsecond!"]], + }, + ...["text", "search", "tel", "url", "password"].map((type) => ({ + name: `${type} input`, + markup: ``, + steps: [ + ["你好🙂", true, "你好🙂"], + ["!", false, "你好🙂!"], + ["", false, "你好🙂!"], + ["", true, ""], + ], + })), + { + name: "email input", + markup: '', + steps: [ + ["x@y.com", true, "x@y.com"], + [".cn", false, "x@y.com.cn"], + ], + }, + { + name: "number input", + markup: '', + steps: [ + ["34", true, "34"], + ["5", false, "345"], + ["", true, ""], + ], + }, + { + name: "textarea", + markup: '', + steps: [ + ["a\r\nb", true, "a\nb"], + ["\n你好🙂", false, "a\nb\n你好🙂"], + ["", true, ""], + ], + }, + ...["", "true", "plaintext-only"].map((attribute) => ({ + name: `editable=${JSON.stringify(attribute)}`, + markup: `

old

second

`, + steps: [ + ["你好🙂", true, "你好🙂"], + ["\nnext\n", false, "你好🙂\nnext\n"], + ["", false, "你好🙂\nnext\n"], + ["", true, ""], + ["again", false, "again"], + ], + })), + ...["", "
", " ", "\n", "


"].map((initial) => ({ + name: `empty root ${JSON.stringify(initial)}`, + markup: `
${initial}
`, + steps: [ + ["hello", false, "hello"], + ["", true, ""], + ["\u200b", false, "\u200b"], + ], + })), + { + name: "preserved whitespace", + markup: '
', + steps: [ + ["hello", false, " hello"], + [" \n ", true, " \n "], + ], + }, + { + name: "native input inside editor", + markup: '
', + steps: [ + ["hello", true, "hello"], + ["!", false, "hello!"], + ], + }, + { + name: "independent root inside noneditable island", + markup: + '
before
old
after
', + steps: [ + ["hello", true, "hello"], + ["!", false, "hello!"], + ], + }, +]; + +export const rejectedTargets = [ + ...[ + '

old

', + '

', + '

', + '


', + '

old

', + '

a old b

', + '

old

', + '

old

', + '

old

', + ].map((target) => ({ + name: target, + markup: `

before

${target}

keep

`, + })), + { name: "noneditable textbox", markup: '
old
' }, + { name: "readonly input", markup: '' }, + { + name: "disabled fieldset", + markup: '
', + }, + ...["checkbox", "radio", "range", "date", "file"].map((type) => ({ + name: `${type} input`, + markup: ``, + })), +]; + +export default { + id: "fill-editor-roots", + routes: ["/fill-editor-roots"], + render() { + return page({ + title: "Fill editor roots", + body: `

Fill editor roots

${editorMarkup}

`, + script: `document.querySelector('#check').addEventListener('click', () => { + const data = { + notes: document.querySelector('#notes').value, + editor: document.querySelector('#editor').innerText, + guard: document.querySelector('#guard').value, + outside: document.querySelector('#outside').textContent, + }; + browserEval.send('fill.checked', data); + document.querySelector('#result').textContent = 'FILL-ROOTS-OK'; + });`, + }); + }, +}; diff --git a/evals/browser/lib/bsk-runner.mjs b/evals/browser/lib/bsk-runner.mjs index 25a40b5c..4a780f8b 100644 --- a/evals/browser/lib/bsk-runner.mjs +++ b/evals/browser/lib/bsk-runner.mjs @@ -74,6 +74,7 @@ export async function runBskSmokeTask({ runId, seed = task.seed, timeoutMs = 60_000, + executeProcess = runProcess, }) { const steps = []; const evidence = {}; @@ -87,14 +88,32 @@ export async function runBskSmokeTask({ let executionError; await mkdir(outputDirectory, { recursive: true }); - async function bsk(args, { timeout = timeoutMs } = {}) { - const execution = await runProcess(bskCommand, [...args, "--json"], { timeoutMs: timeout }); + async function bsk(args, { timeout = timeoutMs, expectError } = {}) { + const execution = await executeProcess(bskCommand, [...args, "--json"], { timeoutMs: timeout }); steps.push(execution); + if (expectError && !execution.timedOut && !execution.error && execution.exitCode > 0) { + let failure; + try { + failure = JSON.parse(execution.stderr.trim() || execution.stdout.trim()); + } catch { + throw new Error( + `bsk did not return the expected JSON error: ${renderStepError(execution)}`, + ); + } + if (failure.code !== expectError.code || failure.data?.reason !== expectError.reason) { + throw new Error( + `expected ${expectError.code}/${expectError.reason}, received ${failure.code}/${failure.data?.reason}`, + ); + } + return failure; + } if (execution.exitCode !== 0 || execution.timedOut || execution.error) { throw new Error( `bsk ${args.join(" ")} failed: ${execution.error ?? renderStepError(execution)}`, ); } + if (expectError) + throw new Error(`expected ${expectError.code}/${expectError.reason}, but bsk succeeded`); return parseJsonOutput(execution); } @@ -169,14 +188,18 @@ export async function runBskSmokeTask({ ]); break; case "fill": - result = await bsk([ - "fill", - ...targetArgs(step, variables), - "--value", - String(resolveValue(step.value, variables)), - ...session, - ...tabArgs(step, variables), - ]); + result = await bsk( + [ + "fill", + ...targetArgs(step, variables), + "--value", + String(resolveValue(step.value, variables)), + ...(step.noClear ? ["--no-clear"] : []), + ...session, + ...tabArgs(step, variables), + ], + { expectError: step.expectError }, + ); break; case "select": result = await bsk([ diff --git a/evals/browser/lib/case-loader.mjs b/evals/browser/lib/case-loader.mjs index d3b98dd5..522741f0 100644 --- a/evals/browser/lib/case-loader.mjs +++ b/evals/browser/lib/case-loader.mjs @@ -69,7 +69,24 @@ function validateWorkflowStep(errors, step, path) { if (["click", "hover", "fill", "select"].includes(step.action) && !step.selector && !step.ref) { errors.push(`${path} requires selector or ref`); } - if (step.action === "fill") checkString(errors, step.value, `${path}.value`); + if (step.action === "fill") { + if (typeof step.value !== "string") errors.push(`${path}.value must be a string`); + if (step.noClear !== undefined && typeof step.noClear !== "boolean") { + errors.push(`${path}.noClear must be a boolean`); + } + } else if (step.noClear !== undefined || step.expectError !== undefined) { + errors.push(`${path}.noClear and expectError are only supported for fill`); + } + if (step.expectError !== undefined) { + if (!isObject(step.expectError)) errors.push(`${path}.expectError must be an object`); + else { + if (Object.keys(step.expectError).some((key) => !["code", "reason"].includes(key))) { + errors.push(`${path}.expectError only supports code and reason`); + } + checkString(errors, step.expectError.code, `${path}.expectError.code`); + checkString(errors, step.expectError.reason, `${path}.expectError.reason`); + } + } if (step.action === "select") { if (!Array.isArray(step.values) || step.values.length === 0) { errors.push(`${path}.values must be a non-empty array`); diff --git a/evals/browser/schemas/case.schema.json b/evals/browser/schemas/case.schema.json index d27c8775..6dfa5a61 100644 --- a/evals/browser/schemas/case.schema.json +++ b/evals/browser/schemas/case.schema.json @@ -62,7 +62,23 @@ "items": { "type": "object", "required": ["action"], - "properties": { "action": { "type": "string" } }, + "properties": { + "action": { "type": "string" }, + "noClear": { + "type": "boolean", + "description": "Fill only: append instead of replacing." + }, + "expectError": { + "type": "object", + "description": "Fill only: require this structured CLI error.", + "required": ["code", "reason"], + "properties": { + "code": { "type": "string", "minLength": 1 }, + "reason": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + } + }, "additionalProperties": true } } diff --git a/evals/browser/tests/eval.test.mjs b/evals/browser/tests/eval.test.mjs index bb8c4d72..6f75054c 100644 --- a/evals/browser/tests/eval.test.mjs +++ b/evals/browser/tests/eval.test.mjs @@ -41,6 +41,7 @@ test("case manifests are discovered, ordered, and grouped into suites", () => { "diagnostics", "mobile-emulation", "generated-form", + "fill-editor-roots", "oopif-scrollbars", "snapshot-coordinates", ], @@ -48,16 +49,16 @@ test("case manifests are discovered, ordered, and grouped into suites", () => { assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1, - regression: 2, + regression: 3, }); }); test("repository validation links every case to a fixture and valid workflow evidence", () => { assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []); const summary = repositorySummary(cases, fixtureRegistry); - assert.equal(summary.cases, 9); - assert.equal(summary.fixtureModules, 10); - assert.equal(summary.fixtureRoutes, 18); + assert.equal(summary.cases, 10); + assert.equal(summary.fixtureModules, 11); + assert.equal(summary.fixtureRoutes, 19); }); test("manifest validation rejects unknown operations and incomplete workflow steps", () => { @@ -240,6 +241,7 @@ test("coverage inventory contains all 28 operations and three manual lanes", () assert.deepEqual(coverage.find(({ operation }) => operation === "interact.fill").smokeCases, [ "form-controls", "generated-form", + "fill-editor-roots", ]); }); diff --git a/evals/browser/tests/fill-workflow.test.mjs b/evals/browser/tests/fill-workflow.test.mjs new file mode 100644 index 00000000..bcc11ad5 --- /dev/null +++ b/evals/browser/tests/fill-workflow.test.mjs @@ -0,0 +1,110 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { runBskSmokeTask } from "../lib/bsk-runner.mjs"; +import { validateCaseManifest } from "../lib/case-loader.mjs"; + +const manifest = JSON.parse( + await readFile( + new URL("../cases/regression/fill-editor-roots/fill-editor-roots.case.json", import.meta.url), + "utf8", + ), +); + +test("fill manifests permit empty/whitespace values and validate rejection options", () => { + for (const value of ["", " ", "\n"]) { + const copy = structuredClone(manifest); + copy.smoke.steps = [{ action: "fill", selector: "#editor", value, noClear: true }]; + assert.deepEqual(validateCaseManifest(copy), []); + } + for (const step of [ + { action: "fill", selector: "#editor", value: 1 }, + { action: "fill", selector: "#editor", value: "", noClear: "true" }, + { action: "fill", selector: "#editor", value: "", expectError: { code: "invalid_params" } }, + { + action: "click", + selector: "#editor", + expectError: { code: "invalid_params", reason: "target_not_fillable" }, + }, + ]) { + const copy = structuredClone(manifest); + copy.smoke.steps = [step]; + assert.notDeepEqual(validateCaseManifest(copy), []); + } +}); + +for (const mode of [ + "expected", + "wrong-code", + "wrong-reason", + "success", + "timeout", + "spawn-error", + "malformed", +]) { + test(`fill rejection workflow: ${mode}`, async () => { + const outputDirectory = await mkdtemp(join(tmpdir(), "fill-workflow-")); + const calls = []; + try { + const report = await runBskSmokeTask({ + bskCommand: "bsk", + outputDirectory, + server: { reset() {}, snapshot: () => ({ events: [] }) }, + serverInfo: { baseUrl: "http://localhost:4173" }, + runId: "fill-test", + task: { + id: "fill-test", + suite: "regression", + tags: [], + startPath: "/fill-editor-roots", + siteAssertions: [], + responseAssertions: [], + adapterAssertions: [], + smokeSteps: [ + { + action: "fill", + selector: "#editor", + value: "", + noClear: true, + expectError: { code: "invalid_params", reason: "target_not_fillable" }, + evidence: "rejected", + }, + ], + }, + executeProcess: async (_command, args) => { + calls.push(args); + let response = {}; + if (args[0] === "session" && args[1] === "start") response = { session_id: "test" }; + if (args[0] === "session" && args[1] === "stop") response = { stopped: ["test"] }; + if (args[0] !== "fill") + return { exitCode: 0, stdout: JSON.stringify(response), stderr: "" }; + assert.equal(args[args.indexOf("--value") + 1], ""); + assert.ok(args.includes("--no-clear")); + return { + exitCode: mode === "success" ? 0 : 1, + stdout: "{}", + timedOut: mode === "timeout", + error: mode === "spawn-error" ? "spawn failed" : undefined, + stderr: + mode === "malformed" + ? "not JSON" + : JSON.stringify({ + code: mode === "wrong-code" ? "cdp_failed" : "invalid_params", + data: { + reason: mode === "wrong-reason" ? "fill_failed" : "target_not_fillable", + }, + }), + }; + }, + }); + assert.equal(report.executionError === undefined, mode === "expected"); + assert.equal(report.evidence.rejected === true, mode === "expected"); + assert.equal(report.evidence.sessionStopped, true); + assert.ok(calls.some((args) => args[0] === "session" && args[1] === "stop")); + } finally { + await rm(outputDirectory, { recursive: true, force: true }); + } + }); +} diff --git a/packages/dsh-plugin-browserskill/src/tools.ts b/packages/dsh-plugin-browserskill/src/tools.ts index 5f9835eb..dd0801f2 100644 --- a/packages/dsh-plugin-browserskill/src/tools.ts +++ b/packages/dsh-plugin-browserskill/src/tools.ts @@ -692,7 +692,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar): defineTool({ name: "interact.fill", description: - "Fill an input / textarea / contenteditable element, clearing it first by default. " + + "Fill an editable text input, textarea, or contenteditable editor root, clearing it first by default. Internal paragraphs and spans are not supported. " + "Target is a snapshot ref (@e3) or a CSS selector.", parameters: { target: { diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index bf7ff797..ab1461a7 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -29,6 +29,26 @@ function coreRefs(refs: ReturnType["refs"]) { } describe("renderVom single-layer page", () => { + it.each([ + "", + "true", + "plaintext-only", + ])("exposes a contenteditable=%s root as a textbox ref", (contenteditable) => { + const out = renderVom( + scene([ + node({ + id: 1, + backendNodeId: 42, + tag: "div", + name: "Message editor", + attrs: { contenteditable }, + rect: { x: 0, y: 0, w: 200, h: 80 }, + }), + ]), + ); + expect(out.text).toContain('@e1 textbox "Message editor"'); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 42 }]); + }); it.each([ { checked: true, marker: " [checked]" }, { checked: false, marker: " [unchecked]" }, diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 6aa16a58..ec9fa946 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -223,7 +223,9 @@ function isFocusable(node: VomNode): boolean { function isContentEditable(node: VomNode): boolean { const contentEditable = node.attrs?.contenteditable?.toLowerCase(); - return contentEditable === "true" || contentEditable === "plaintext-only"; + return ( + contentEditable === "" || contentEditable === "true" || contentEditable === "plaintext-only" + ); } const ICON_REFERENCE_MARKERS = new Set(["glyph", "icon", "icons", "symbol"]);