diff --git a/.changeset/fair-vans-visit.md b/.changeset/fair-vans-visit.md new file mode 100644 index 0000000..24f187b --- /dev/null +++ b/.changeset/fair-vans-visit.md @@ -0,0 +1,5 @@ +--- +"@upstash/box": patch +--- + +Support Jev browser actions with `model: "jev"` (also `typesafe-ai/jev` and `vercel/typesafe-ai/jev`), on the Upstash-provided key by default, named input variables, scoped target discovery, action timeouts, a configurable Jev confidence threshold (default 0.8), and variable-backed replay. diff --git a/packages/python-sdk/CHANGELOG.md b/packages/python-sdk/CHANGELOG.md index bef6cd0..60c8e8b 100644 --- a/packages/python-sdk/CHANGELOG.md +++ b/packages/python-sdk/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to `upstash-box` (Python) are documented here. ## Unreleased +- Add Jev-only `confidence_threshold` to `tab.act()`, with default 0.8 and an inclusive 0–1 range. + +- Support Jev browser actions through `model="jev"` (also `"typesafe-ai/jev"` and + `"vercel/typesafe-ai/jev"`), on the Upstash-provided key by default. Add + `variables`, `scope`, and `timeout` to `tab.act()`, including variable-backed + deterministic action replay. + - Fix `delete_boxes(box_ids=[])` deleting every box on the account. The API read an empty id list as "no filter". `delete_boxes` and `delete_snapshots` now raise `BoxError` before any request is made when the list is empty or contains a diff --git a/packages/python-sdk/PARITY.md b/packages/python-sdk/PARITY.md index f04fee6..ec0ff01 100644 --- a/packages/python-sdk/PARITY.md +++ b/packages/python-sdk/PARITY.md @@ -10,8 +10,13 @@ JS `Run`/`StreamRun` → Python `Run`/`StreamRun` (+ `AsyncRun`/`AsyncStreamRun` ## Module exports +Browser act options: JS `BrowserActOptions` / `BrowserActReplayOptions` map to +Python `Tab.act(..., model=, variables=, scope=, timeout=, confidence_threshold=)` keyword arguments. +Both SDKs accept `jev`, `typesafe-ai/jev` and `vercel/typesafe-ai/jev` for Jev and preserve `%name%` action arguments. + | JS | Python | | ---------------------- | ---------------------------- | +| `BrowserActOptions` / `BrowserActReplayOptions` | `Tab.act()` keyword arguments: `model`, `variables`, `scope`, `timeout`, `confidence_threshold` | | `Box` / `EphemeralBox` | `Box` / `EphemeralBox` (+ `Async*`) | | `Run` / `StreamRun` | `Run` / `StreamRun` (+ `Async*`) | | `BoxError` | `BoxError` | diff --git a/packages/python-sdk/README.md b/packages/python-sdk/README.md index e3b0ff3..e1977c7 100644 --- a/packages/python-sdk/README.md +++ b/packages/python-sdk/README.md @@ -345,3 +345,22 @@ value. ## License MIT + + +### Jev decision threshold + +```python +result = await tab.act( + "Open Edit profile, fill Display name with %name%, and Save profile", + model="jev", # Also "typesafe-ai/jev" or "vercel/typesafe-ai/jev". + variables={"name": "Ada"}, + confidence_threshold=0.7, # Optional; default 0.8. Also available on the sync client. +) +``` + +Jev runs on the Upstash-provided key by default, billed at $0.042 per million +input tokens with free output; a non-managed Box with your own Vercel AI Gateway +key uses that key instead. The threshold accepts finite numbers from 0 to 1 inclusive. Lower values accept +more uncertain action and completion decisions. Target validation and execution +limits remain enforced. This option is only supported for Jev instructions, +not action replay or other models. Check `result.success` and `result.message`. diff --git a/packages/python-sdk/tests/_async/test_box_browser.py b/packages/python-sdk/tests/_async/test_box_browser.py index ee6bd10..1014804 100644 --- a/packages/python-sdk/tests/_async/test_box_browser.py +++ b/packages/python-sdk/tests/_async/test_box_browser.py @@ -17,6 +17,61 @@ BASE = f"{TEST_BASE_URL}/v2/box/box-123" +@respx.mock +async def test_jev_act_options_and_variable_replay(): + box = await make_async_box(respx.mock) + route = respx.post(f"{BASE}/browser/act").mock( + return_value=httpx.Response( + 200, + json={ + "success": True, + "input_tokens": 100, + "output_tokens": 5, + "actions": [ + { + "selector": "#email", + "description": "Email", + "method": "fill", + "arguments": ["%email%"], + } + ], + }, + ) + ) + tab = box.browser.get_tab("tab-1") + result = await tab.act( + "Fill Email with %email%", + model="vercel/typesafe-ai/jev", + variables={"email": "hello@example.com"}, + scope="#login", + timeout=15000, + confidence_threshold=0.7, + ) + assert last_json_body(route) == { + "instruction": "Fill Email with %email%", + "model": "vercel/typesafe-ai/jev", + "tab": "tab-1", + "variables": {"email": "hello@example.com"}, + "scope": "#login", + "timeout": 15000, + "confidence_threshold": 0.7, + } + assert result.actions[0].arguments == ["%email%"] + await tab.act(result.actions[0], variables={"email": "other@example.com"}) + assert "model" not in last_json_body(route) + assert last_json_body(route)["variables"] == {"email": "other@example.com"} + await box.aclose() + + +@pytest.mark.parametrize("timeout", [0, -1, 1.5, 180001, True]) +@respx.mock +async def test_invalid_act_timeout(timeout): + box = await make_async_box(respx.mock) + with pytest.raises(BoxError, match="act timeout"): + await box.browser.get_tab("tab-1").act("Click Submit", timeout=timeout) + await box.aclose() + + # ---------- tabs / page operations ---------- @@ -581,3 +636,50 @@ async def test_recordings_list_paginates_and_get(): assert "limit=100" in str(first) assert "cursor=cursor-2" in str(second) await box.aclose() + + +@pytest.mark.parametrize("threshold", [-0.1, 1.1, float("nan"), float("inf"), True, "0.7"]) +@respx.mock +async def test_invalid_jev_confidence_threshold(threshold): + box = await make_async_box(respx.mock) + with pytest.raises(BoxError, match="confidence threshold"): + await box.browser.get_tab("tab-1").act( + "Click Submit", model="vercel/typesafe-ai/jev", confidence_threshold=threshold + ) + await box.aclose() + + +@pytest.mark.parametrize("threshold", [0, 1]) +@respx.mock +async def test_jev_threshold_boundaries(threshold): + box = await make_async_box(respx.mock) + route = respx.post(f"{BASE}/browser/act").mock( + return_value=httpx.Response(200, json={"success": True}) + ) + await box.browser.get_tab("tab-1").act( + "Click Submit", model="vercel/typesafe-ai/jev", confidence_threshold=threshold + ) + assert last_json_body(route)["confidence_threshold"] == threshold + await box.aclose() + + +@pytest.mark.parametrize("model", ["jev", "typesafe-ai/jev"]) +@respx.mock +async def test_jev_threshold_accepts_aliases(model): + box = await make_async_box(respx.mock) + route = respx.post(f"{BASE}/browser/act").mock( + return_value=httpx.Response(200, json={"success": True}) + ) + await box.browser.get_tab("tab-1").act("Click Submit", model=model, confidence_threshold=0.7) + body = last_json_body(route) + assert body["model"] == model + assert body["confidence_threshold"] == 0.7 + await box.aclose() + + +@respx.mock +async def test_threshold_rejects_other_models(): + box = await make_async_box(respx.mock) + with pytest.raises(BoxError, match="only for Jev instructions"): + await box.browser.get_tab("tab-1").act("Click Submit", confidence_threshold=0.7) + await box.aclose() diff --git a/packages/python-sdk/tests/_sync/test_sync_client.py b/packages/python-sdk/tests/_sync/test_sync_client.py index a2bc040..64634a8 100644 --- a/packages/python-sdk/tests/_sync/test_sync_client.py +++ b/packages/python-sdk/tests/_sync/test_sync_client.py @@ -24,6 +24,44 @@ RUN_URL = f"{BASE}/run/stream" +@respx.mock +def test_jev_act_options_and_replay(): + box = make_sync_box(respx.mock) + route = respx.post(f"{BASE}/browser/act").mock( + return_value=httpx.Response( + 200, + json={ + "success": True, + "actions": [ + { + "selector": "#email", + "description": "Email", + "method": "fill", + "arguments": ["%email%"], + } + ], + }, + ) + ) + tab = box.browser.get_tab("tab-1") + result = tab.act( + "Fill Email with %email%", + model="vercel/typesafe-ai/jev", + variables={"email": "hello@example.com"}, + confidence_threshold=0.7, + scope="#login", + timeout=15000, + ) + assert last_json_body(route)["confidence_threshold"] == 0.7 + assert result.success + assert last_json_body(route)["timeout"] == 15000 + assert last_json_body(route)["scope"] == "#login" + tab.act(result.actions[0], variables={"email": "other@example.com"}) + assert "model" not in last_json_body(route) + assert last_json_body(route)["variables"] == {"email": "other@example.com"} + box.close() + + def _opts(): return {"api_key": TEST_API_KEY, "base_url": TEST_BASE_URL} diff --git a/packages/python-sdk/upstash_box/_async/client.py b/packages/python-sdk/upstash_box/_async/client.py index 18388d0..f5d34b5 100644 --- a/packages/python-sdk/upstash_box/_async/client.py +++ b/packages/python-sdk/upstash_box/_async/client.py @@ -106,6 +106,8 @@ WORKSPACE = common.WORKSPACE _DEFAULT_TIMEOUT_MS = 600000 +# Model names that select Jev for browser act. +_JEV_MODELS = frozenset({"jev", "typesafe-ai/jev", "vercel/typesafe-ai/jev"}) def _resolve_tool_call_id(parsed: Dict[str, Any]) -> Optional[str]: @@ -690,25 +692,54 @@ async def act( instruction: Union[str, BrowserObserveElement, BrowserActAction], *, model: Optional[str] = None, + variables: Optional[Dict[str, str]] = None, + scope: Optional[str] = None, + timeout: Optional[int] = None, + confidence_threshold: Optional[float] = None, ) -> BrowserActResult: - """Resolve and execute one action on this tab. + """Execute a focused instruction on this tab, possibly using several interactions. Pass a string (LLM-resolved, metered) or a pre-resolved ``observe()`` action to replay it with no LLM call and no key (``model`` ignored). + Jev's ``confidence_threshold`` is from 0 to 1 inclusive, default 0.8. + Lower values accept more uncertain model decisions; target checks still apply. """ + if confidence_threshold is not None: + if ( + isinstance(confidence_threshold, bool) + or not isinstance(confidence_threshold, (int, float)) + or not 0 <= confidence_threshold <= 1 + ): + raise BoxError("act confidence threshold must be a finite number from 0 to 1") + if not isinstance(instruction, str) or model not in _JEV_MODELS: + raise BoxError("act confidence threshold is supported only for Jev instructions") + if timeout is not None and ( + isinstance(timeout, bool) or not isinstance(timeout, int) or not 1 <= timeout <= 180000 + ): + raise BoxError("act timeout must be an integer from 1 to 180000 milliseconds") + if scope is not None and not scope.strip(): + raise BoxError("act scope must be a non-empty CSS selector") if isinstance(instruction, str): body: Dict[str, Any] = {"instruction": instruction, "tab": self.id} if model: body["model"] = model + if scope: + body["scope"] = scope else: if not instruction.selector: raise BoxError("act(action) requires a selector; observe() did not resolve one") body = {"action": instruction.model_dump(exclude_none=True), "tab": self.id} + if confidence_threshold is not None: + body["confidence_threshold"] = confidence_threshold + if variables is not None: + body["variables"] = variables + if timeout is not None: + body["timeout"] = timeout resp = await self._box._request( "POST", f"/v2/box/{self._box.id}/browser/act", body=body, - timeout=180000, + timeout=timeout + 5000 if timeout is not None else 185000, ) return BrowserActResult.model_validate(resp) diff --git a/packages/python-sdk/upstash_box/_sync/client.py b/packages/python-sdk/upstash_box/_sync/client.py index 998fb6a..e06c983 100644 --- a/packages/python-sdk/upstash_box/_sync/client.py +++ b/packages/python-sdk/upstash_box/_sync/client.py @@ -105,6 +105,8 @@ WORKSPACE = common.WORKSPACE _DEFAULT_TIMEOUT_MS = 600000 +# Model names that select Jev for browser act. +_JEV_MODELS = frozenset({"jev", "typesafe-ai/jev", "vercel/typesafe-ai/jev"}) def _resolve_tool_call_id(parsed: Dict[str, Any]) -> Optional[str]: @@ -683,25 +685,54 @@ def act( instruction: Union[str, BrowserObserveElement, BrowserActAction], *, model: Optional[str] = None, + variables: Optional[Dict[str, str]] = None, + scope: Optional[str] = None, + timeout: Optional[int] = None, + confidence_threshold: Optional[float] = None, ) -> BrowserActResult: - """Resolve and execute one action on this tab. + """Execute a focused instruction on this tab, possibly using several interactions. Pass a string (LLM-resolved, metered) or a pre-resolved ``observe()`` action to replay it with no LLM call and no key (``model`` ignored). + Jev's ``confidence_threshold`` is from 0 to 1 inclusive, default 0.8. + Lower values accept more uncertain model decisions; target checks still apply. """ + if confidence_threshold is not None: + if ( + isinstance(confidence_threshold, bool) + or not isinstance(confidence_threshold, (int, float)) + or not 0 <= confidence_threshold <= 1 + ): + raise BoxError("act confidence threshold must be a finite number from 0 to 1") + if not isinstance(instruction, str) or model not in _JEV_MODELS: + raise BoxError("act confidence threshold is supported only for Jev instructions") + if timeout is not None and ( + isinstance(timeout, bool) or not isinstance(timeout, int) or not 1 <= timeout <= 180000 + ): + raise BoxError("act timeout must be an integer from 1 to 180000 milliseconds") + if scope is not None and not scope.strip(): + raise BoxError("act scope must be a non-empty CSS selector") if isinstance(instruction, str): body: Dict[str, Any] = {"instruction": instruction, "tab": self.id} if model: body["model"] = model + if scope: + body["scope"] = scope else: if not instruction.selector: raise BoxError("act(action) requires a selector; observe() did not resolve one") body = {"action": instruction.model_dump(exclude_none=True), "tab": self.id} + if confidence_threshold is not None: + body["confidence_threshold"] = confidence_threshold + if variables is not None: + body["variables"] = variables + if timeout is not None: + body["timeout"] = timeout resp = self._box._request( "POST", f"/v2/box/{self._box.id}/browser/act", body=body, - timeout=180000, + timeout=timeout + 5000 if timeout is not None else 185000, ) return BrowserActResult.model_validate(resp) diff --git a/packages/sdk/README.md b/packages/sdk/README.md index f35a7e4..f81fa00 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -114,6 +114,58 @@ ssh @us-east-1.box.upstash.com Use your **Box API key** as the SSH password. +### Browser actions + +Select Jev, TypeSafe AI's fast evaluation model, with +`model: "jev"` on `tab.act()` (`typesafe-ai/jev` and `vercel/typesafe-ai/jev` also +work). No extra setup is needed: Jev runs on the Upstash-provided key by default, +billed at $0.042 per million input tokens with free output. A non-managed Box with +your own Vercel AI Gateway key uses that key instead. + +```ts +const box = await Box.create({ browser: true }); +const tab = await box.browser.tab.create("https://your-app.example/settings"); +const model = "jev"; + +const result = await tab.act("Choose Germany from the country dropdown and confirm", { + model, +}); +if (!result.success) throw new Error(result.message); + +const filled = await tab.act("Fill the Email field with %email%", { + model, + variables: { email: "person@example.com" }, + scope: "form#profile", // Optional CSS selector matching one element. + timeout: 60_000, // Optional, defaults to 180_000 ms. + confidenceThreshold: 0.7, // Optional Jev-only override; default is 0.8. +}); + +// Replay a resolved action without inference, using a different exact value. +if (filled.success && filled.actions.length) { + await tab.act(filled.actions[0], { variables: { email: "other@example.com" } }); +} +``` + +One focused instruction can perform several interactions, such as opening a menu, +choosing an option, and confirming. Jev stops when it determines the instruction +is complete, becomes uncertain or blocked, reaches eight interactions, or runs out +of time. Check `success` and `message`; a failed call may have performed some actions. + +`confidenceThreshold` accepts a finite number from `0` to `1`, inclusive, for +Jev instructions only. It applies to action selection, focused action checks, +and completion. Lower values accept more uncertain model decisions; these scores +are not calibrated guarantees of correctness. Target validation, ambiguity checks, +variable requirements, deadlines and interaction limits still apply. Replay and +other models reject this option. + +For text entry, supply the exact string in `variables` and reference it as `%name%`. +`%name%` is treated as a variable only when `variables` is supplied, so literal +text such as `caf%C3%A9` works without it. Returned messages and actions show +supplied values as `%name%`, never the values themselves. +Jev chooses from available controls and supplied values; it does not generate text. +Its initial support covers DOM controls in the main document and open shadow roots. +It does not support iframe contents, canvas interactions, `extract()`, or `observe()`. + ### Agent #### `box.agent.run(options: RunOptions): Promise` diff --git a/packages/sdk/src/__tests__/box-browser.test.ts b/packages/sdk/src/__tests__/box-browser.test.ts index 827fc87..6d92537 100644 --- a/packages/sdk/src/__tests__/box-browser.test.ts +++ b/packages/sdk/src/__tests__/box-browser.test.ts @@ -202,6 +202,101 @@ describe("Box browser operations", () => { }); }); + it("forwards Jev act options and preserves placeholder arguments", async () => { + const { box, fetchMock } = await createTestBox(); + fetchMock.mockResolvedValueOnce( + mockResponse({ + success: true, + actions: [ + { selector: "#email", description: "Email", method: "fill", arguments: ["%email%"] }, + ], + input_tokens: 120, + output_tokens: 4, + }), + ); + const result = await box.browser.getTab("tab-1").act("Fill Email with %email%", { + model: "vercel/typesafe-ai/jev", + variables: { email: "hello@example.com" }, + scope: "#login", + timeout: 15000, + confidenceThreshold: 0.7, + }); + expect(JSON.parse(fetchMock.mock.calls.at(-1)![1]!.body as string)).toEqual({ + instruction: "Fill Email with %email%", + model: "vercel/typesafe-ai/jev", + tab: "tab-1", + variables: { email: "hello@example.com" }, + scope: "#login", + timeout: 15000, + confidence_threshold: 0.7, + }); + expect(result.actions[0].arguments).toEqual(["%email%"]); + expect(result.inputTokens).toBe(120); + fetchMock.mockResolvedValueOnce(mockResponse({ success: true })); + await box.browser + .getTab("tab-1") + .act(result.actions[0], { variables: { email: "second@example.com" }, timeout: 5000 }); + const replay = JSON.parse(fetchMock.mock.calls.at(-1)![1]!.body as string); + expect(replay.variables).toEqual({ email: "second@example.com" }); + expect(replay).not.toHaveProperty("model"); + expect(replay).not.toHaveProperty("instruction"); + }); + + it.each([-0.1, 1.1, NaN, Infinity, -Infinity])( + "rejects invalid Jev threshold %s", + async (confidenceThreshold) => { + const { box, fetchMock } = await createTestBox(); + const calls = fetchMock.mock.calls.length; + await expect( + box.browser + .getTab("tab-1") + .act("Click Submit", { model: "vercel/typesafe-ai/jev", confidenceThreshold }), + ).rejects.toThrow("confidence threshold"); + expect(fetchMock.mock.calls).toHaveLength(calls); + }, + ); + + it.each([0, 1])("preserves the threshold boundary %s", async (confidenceThreshold) => { + const { box, fetchMock } = await createTestBox(); + fetchMock.mockResolvedValueOnce(mockResponse({ success: true })); + await box.browser + .getTab("tab-1") + .act("Click Submit", { model: "vercel/typesafe-ai/jev", confidenceThreshold }); + expect(JSON.parse(fetchMock.mock.calls.at(-1)![1]!.body as string).confidence_threshold).toBe( + confidenceThreshold, + ); + }); + + it.each(["jev", "typesafe-ai/jev"])("accepts a threshold for the %s alias", async (model) => { + const { box, fetchMock } = await createTestBox(); + fetchMock.mockResolvedValueOnce(mockResponse({ success: true })); + await box.browser.getTab("tab-1").act("Click Submit", { model, confidenceThreshold: 0.7 }); + const body = JSON.parse(fetchMock.mock.calls.at(-1)![1]!.body as string); + expect(body.model).toBe(model); + expect(body.confidence_threshold).toBe(0.7); + }); + + it("rejects a threshold for a different model", async () => { + const { box } = await createTestBox(); + await expect( + box.browser + .getTab("tab-1") + .act("Click Submit", { model: "anthropic/claude-sonnet-4-5", confidenceThreshold: 0.7 }), + ).rejects.toThrow("only for Jev instructions"); + }); + + it.each([0, -1, 1.5, 180001, NaN])( + "rejects invalid act timeout %s before sending", + async (timeout) => { + const { box, fetchMock } = await createTestBox(); + const calls = fetchMock.mock.calls.length; + await expect(box.browser.getTab("tab-1").act("Click Submit", { timeout })).rejects.toThrow( + "act timeout", + ); + expect(fetchMock.mock.calls).toHaveLength(calls); + }, + ); + it("replays a pre-resolved action deterministically (posts action, not instruction)", async () => { const { box, fetchMock } = await createTestBox(); fetchMock diff --git a/packages/sdk/src/__tests__/integration/browser-act-local.integration.test.ts b/packages/sdk/src/__tests__/integration/browser-act-local.integration.test.ts new file mode 100644 index 0000000..ccd960e --- /dev/null +++ b/packages/sdk/src/__tests__/integration/browser-act-local.integration.test.ts @@ -0,0 +1,80 @@ +import { createServer } from "node:http"; +import { afterAll, beforeAll, expect, it } from "vitest"; +import { Box } from "../../index.js"; + +// Exercises the actual SDK HTTP transport against a local API fixture. +// Does not load .env or create remote boxes. +const requests: Record[] = []; +const server = createServer(async (req, res) => { + res.setHeader("Content-Type", "application/json"); + if (req.method === "GET") { + res.end(JSON.stringify({ id: "local-box", status: "idle", browser: true })); + return; + } + let raw = ""; + for await (const chunk of req) raw += chunk; + const body = JSON.parse(raw); + requests.push(body); + if (body.instruction?.includes("unsupported")) { + res.writeHead(400); + res.end(JSON.stringify({ error: "Unsupported Jev browser operation" })); + return; + } + res.end( + JSON.stringify({ + success: true, + message: "done", + actions: [ + { selector: "#email", method: "fill", description: "Email", arguments: ["%email%"] }, + ], + input_tokens: body.action ? 0 : 100, + output_tokens: body.action ? 0 : 5, + }), + ); +}); +let box: Box; +beforeAll(async () => { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as { port: number }; + box = await Box.get("local-box", { + apiKey: "local-test-key", + baseUrl: `http://127.0.0.1:${port}`, + }); +}); +afterAll(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); +}); + +it("sends model and variables over HTTP, then replays without a model", async () => { + const tab = box.browser.getTab("tab-1"); + const result = await tab.act("Fill Email with %email%", { + model: "vercel/typesafe-ai/jev", + variables: { email: "test@example.com" }, + scope: "form", + timeout: 10000, + confidenceThreshold: 0.7, + }); + expect(result.success).toBe(true); + expect(result.inputTokens).toBe(100); + await tab.act(result.actions[0], { variables: { email: "other@example.com" } }); + expect(requests[0]).toMatchObject({ + model: "vercel/typesafe-ai/jev", + scope: "form", + timeout: 10000, + confidence_threshold: 0.7, + }); + expect(requests[1]).toMatchObject({ + action: { arguments: ["%email%"] }, + variables: { email: "other@example.com" }, + }); + expect(requests[1]).not.toHaveProperty("model"); +}); + +it("surfaces an unsupported-operation response without retrying", async () => { + const before = requests.length; + await expect( + box.browser.getTab("tab-1").act("unsupported operation", { model: "vercel/typesafe-ai/jev" }), + ).rejects.toThrow("Unsupported Jev"); + expect(requests).toHaveLength(before + 1); +}); diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index a9bd9ab..bc676ff 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -55,6 +55,8 @@ import { type AgentConfig, type CustomHarnessConfig, type BrowserExtractOptions, + type BrowserActOptions, + type BrowserActReplayOptions, type BrowserContent, type BrowserScreenshotOptions, type BrowserTabCreateOptions, @@ -69,6 +71,9 @@ import { } from "./types.js"; import { telemetryHeaders } from "./telemetry.js"; +// Model names that select Jev for browser act. +const JEV_MODELS = new Set(["jev", "typesafe-ai/jev", "vercel/typesafe-ai/jev"]); + type BrowserExtractSchema = { parse(data: unknown): T; }; @@ -576,25 +581,55 @@ export class Tab { return { elements: resp.elements ?? [] }; } - /** Resolve and execute one natural-language action on this tab (metered). */ - async act(instruction: string, options?: BrowserExtractOptions): Promise; + /** Execute a focused instruction. Jev is available as `jev` (also `typesafe-ai/jev`, `vercel/typesafe-ai/jev`). */ + async act(instruction: string, options?: BrowserActOptions): Promise; /** Replay a pre-resolved `observe()` action with no LLM call and no key (`model` ignored). */ - async act(action: BrowserAction): Promise; + async act(action: BrowserAction, options?: BrowserActReplayOptions): Promise; async act( instructionOrAction: string | BrowserAction, - options?: BrowserExtractOptions, + options?: BrowserActOptions, ): Promise { if (typeof instructionOrAction !== "string" && !instructionOrAction.selector) { throw new BoxError("act(action) requires a selector; observe() did not resolve one"); } + if ( + options?.timeout !== undefined && + (!Number.isInteger(options.timeout) || options.timeout < 1 || options.timeout > 180000) + ) { + throw new BoxError("act timeout must be an integer from 1 to 180000 milliseconds"); + } + if (options?.confidenceThreshold !== undefined) { + if ( + !Number.isFinite(options.confidenceThreshold) || + options.confidenceThreshold < 0 || + options.confidenceThreshold > 1 + ) { + throw new BoxError("act confidence threshold must be a finite number from 0 to 1"); + } + if (typeof instructionOrAction !== "string" || !JEV_MODELS.has(options.model ?? "")) { + throw new BoxError("act confidence threshold is supported only for Jev instructions"); + } + } + if (options?.scope !== undefined && !options.scope.trim()) { + throw new BoxError("act scope must be a non-empty CSS selector"); + } + const executionOptions = { + ...(options?.variables !== undefined ? { variables: options.variables } : {}), + ...(options?.timeout !== undefined ? { timeout: options.timeout } : {}), + }; const body = typeof instructionOrAction === "string" ? { instruction: instructionOrAction, tab: this.id, ...(options?.model ? { model: options.model } : {}), + ...(options?.scope ? { scope: options.scope } : {}), + ...(options?.confidenceThreshold !== undefined + ? { confidence_threshold: options.confidenceThreshold } + : {}), + ...executionOptions, } - : { action: instructionOrAction, tab: this.id }; + : { action: instructionOrAction, tab: this.id, ...executionOptions }; const resp = await this.box._request<{ success?: boolean; message?: string; @@ -605,7 +640,8 @@ export class Tab { output_tokens?: number; }>("POST", `/v2/box/${this.box.id}/browser/act`, { body, - timeout: 180000, + // Allow the server a short grace period to return the operation outcome. + timeout: options?.timeout !== undefined ? options.timeout + 5000 : 185000, }); return { success: Boolean(resp.success), diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index b1089cb..08e781e 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -84,6 +84,8 @@ export type { Schedule, ScheduleStatus, BrowserExtractOptions, + BrowserActOptions, + BrowserActReplayOptions, BrowserContent, BrowserLink, BrowserScreenshotOptions, diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index 7ee14c1..a6b71f0 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -1255,6 +1255,23 @@ export interface BrowserExtractOptions { model?: string; } +/** Options for instruction-based `tab.act()`. */ +export interface BrowserActOptions extends BrowserExtractOptions { + /** Jev-only decision threshold, from 0 to 1 inclusive. Defaults to 0.8. + * Lower values accept more uncertain model decisions. Does not bypass target checks. + */ + confidenceThreshold?: number; + /** Exact values referenced as `%name%` in the instruction. */ + variables?: Record; + /** CSS selector limiting target discovery. Must match exactly one element. */ + scope?: string; + /** Action timeout in milliseconds, from 1 to 180000. Defaults to 180000. */ + timeout?: number; +} + +/** Replay a resolved action without inference, substituting any named values locally. */ +export type BrowserActReplayOptions = Pick; + /** A link on the page. */ export interface BrowserLink { /** Link text. */