diff --git a/.github/actions/detect-codemode-live-credentials/action.yml b/.github/actions/detect-codemode-live-credentials/action.yml new file mode 100644 index 0000000000..6010aa4f3f --- /dev/null +++ b/.github/actions/detect-codemode-live-credentials/action.yml @@ -0,0 +1,44 @@ +name: Detect code-mode live credentials +description: Decide whether Browserbase, Vercel, and optional OpenAI live proofs can run. + +inputs: + require-openai: + description: Require OPENAI_API_KEY in addition to Browserbase and Vercel credentials. + default: "false" + +outputs: + available: + description: Whether every credential required by the live proof is available. + value: ${{ steps.detect.outputs.available }} + +runs: + using: composite + steps: + - id: detect + shell: bash + env: + REQUIRE_OPENAI: ${{ inputs.require-openai }} + run: | + browserbase_ready=false + vercel_ready=false + openai_ready=false + + if [[ -n "$BROWSERBASE_API_KEY" && -n "$BROWSERBASE_PROJECT_ID" ]]; then + browserbase_ready=true + fi + if [[ -n "$VERCEL_TOKEN" ]]; then + if [[ -n "$VERCEL_TEAM_ID" && -n "$VERCEL_PROJECT_ID" ]]; then + vercel_ready=true + fi + elif [[ -n "$VERCEL_OIDC_TOKEN" ]]; then + vercel_ready=true + fi + if [[ "$REQUIRE_OPENAI" != "true" || -n "$OPENAI_API_KEY" ]]; then + openai_ready=true + fi + + if [[ "$browserbase_ready" == "true" && "$vercel_ready" == "true" && "$openai_ready" == "true" ]]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi diff --git a/.github/workflows/codemode-framework-examples.yml b/.github/workflows/codemode-framework-examples.yml index 7d361420d4..b512da03b6 100644 --- a/.github/workflows/codemode-framework-examples.yml +++ b/.github/workflows/codemode-framework-examples.yml @@ -60,7 +60,6 @@ jobs: - uses: ./.github/actions/setup-chrome-verified id: setup-chrome - - run: pnpm exec turbo run build --filter @browserbasehq/stagehand-codemode - run: pnpm --filter ${{ matrix.package }} typecheck - run: pnpm --filter ${{ matrix.package }} test:contract - run: pnpm --filter ${{ matrix.package }} pack:artifacts @@ -70,6 +69,7 @@ jobs: STAGEHAND_BROWSER: local - name: Detect live test credentials id: live-credentials + uses: ./.github/actions/detect-codemode-live-credentials env: BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} @@ -77,13 +77,6 @@ jobs: VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - run: | - if [[ -n "$BROWSERBASE_API_KEY" && -n "$BROWSERBASE_PROJECT_ID" ]] && \ - [[ -n "$VERCEL_OIDC_TOKEN" || ( -n "$VERCEL_TEAM_ID" && -n "$VERCEL_PROJECT_ID" && -n "$VERCEL_TOKEN" ) ]]; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - fi - name: Run live package-installed sandbox proof if: steps.live-credentials.outputs.available == 'true' run: pnpm --filter ${{ matrix.package }} e2e @@ -111,12 +104,14 @@ jobs: with: use-prebuilt-artifacts: "false" - - run: pnpm exec turbo run build --filter @browserbasehq/stagehand-codemode - run: pnpm --filter @browserbasehq/stagehand-integrations-example-mastra typecheck - run: pnpm --filter @browserbasehq/stagehand-integrations-example-mastra test:contract - run: pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts - name: Detect Mastra live test credentials id: mastra-live-credentials + uses: ./.github/actions/detect-codemode-live-credentials + with: + require-openai: "true" env: BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} @@ -125,13 +120,6 @@ jobs: VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - run: | - if [[ -n "$BROWSERBASE_API_KEY" && -n "$BROWSERBASE_PROJECT_ID" && -n "$OPENAI_API_KEY" ]] && \ - [[ -n "$VERCEL_OIDC_TOKEN" || ( -n "$VERCEL_TEAM_ID" && -n "$VERCEL_PROJECT_ID" && -n "$VERCEL_TOKEN" ) ]]; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - fi - name: Run Mastra live sandbox proof if: steps.mastra-live-credentials.outputs.available == 'true' run: pnpm --filter @browserbasehq/stagehand-integrations-example-mastra e2e @@ -144,3 +132,65 @@ jobs: VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + crewai: + name: CrewAI + if: >- + github.event_name == 'push' || + github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'safe-to-test') + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - uses: ./.github/actions/setup-node-pnpm + with: + use-prebuilt-artifacts: "false" + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: packages/integrations/examples/crewai/requirements.txt + + - run: python -m pip install -r packages/integrations/examples/crewai/requirements.txt + - run: >- + python -m py_compile + packages/integrations/examples/shared/vercel_sandbox_lease.py + packages/integrations/examples/shared/test_vercel_sandbox_lease.py + packages/integrations/examples/crewai/agent.py + packages/integrations/examples/crewai/sandbox.py + packages/integrations/examples/crewai/test_agent.py + packages/integrations/examples/crewai/test_e2e.py + packages/integrations/examples/crewai/e2e.py + - run: python packages/integrations/examples/shared/test_vercel_sandbox_lease.py + - run: python packages/integrations/examples/crewai/test_agent.py + - run: python packages/integrations/examples/crewai/test_e2e.py + - run: pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts + - name: Detect CrewAI live test credentials + id: crewai-live-credentials + uses: ./.github/actions/detect-codemode-live-credentials + with: + require-openai: "true" + env: + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + - name: Run CrewAI live sandbox proof + if: steps.crewai-live-credentials.outputs.available == 'true' + run: python packages/integrations/examples/crewai/e2e.py + env: + STAGEHAND_SANDBOX_ARTIFACTS: ${{ github.workspace }}/packages/integrations/examples/vercel-sandbox/.artifacts + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/packages/integrations/README.md b/packages/integrations/README.md index ad29cbec93..e0f5708e79 100644 --- a/packages/integrations/README.md +++ b/packages/integrations/README.md @@ -81,3 +81,5 @@ implementation modules and an in-process arbitrary-code executor are not public - [Vercel Sandbox](./examples/vercel-sandbox) installs the exact packed artifact inside a Firecracker microVM and returns a framework-neutral, bearer-authenticated MCP connection. - [Mastra](./examples/mastra) consumes that connection with one persistent remote MCP client. +- [CrewAI](./examples/crewai) keeps its context-managed MCP adapter open across every tool call in + one crew execution. diff --git a/packages/integrations/examples/crewai/README.md b/packages/integrations/examples/crewai/README.md new file mode 100644 index 0000000000..e96d0ad61a --- /dev/null +++ b/packages/integrations/examples/crewai/README.md @@ -0,0 +1,79 @@ +# CrewAI with Stagehand code mode + +This example gives a CrewAI agent one browser tool, `code_execute`, without running generated +JavaScript in the Python agent process. A small trusted Python lease owner starts the +package-installed [Vercel Sandbox example](../vercel-sandbox) and receives its `{ url, token }` +connection. CrewAI then connects directly through authenticated Streamable HTTP: + +```text +CrewAI MCPServerAdapter -> authenticated HTTPS -> Vercel Sandbox -> Stagehand code-mode MCP + `-> generated JavaScript +``` + +The adapter discovers the canonical tool description and uses it as the agent's backstory. It does +not copy the executor, schema, or code-mode skill. + +## Setup + +Create a Python 3.12 environment and install the pinned CrewAI example dependencies: + +```bash +python3.12 -m venv .venv +. .venv/bin/activate +python -m pip install -r packages/integrations/examples/crewai/requirements.txt +``` + +Build and pack the exact Stagehand packages under review: + +```bash +pnpm install +pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts +``` + +## Run the end-to-end proof + +```bash +STAGEHAND_SANDBOX_ARTIFACTS="$PWD/packages/integrations/examples/vercel-sandbox/.artifacts" \ +BROWSERBASE_API_KEY= \ +BROWSERBASE_PROJECT_ID= \ +VERCEL_OIDC_TOKEN= \ +OPENAI_API_KEY= \ +python packages/integrations/examples/crewai/e2e.py +``` + +For external CI, replace `VERCEL_OIDC_TOKEN` with `VERCEL_TEAM_ID`, `VERCEL_PROJECT_ID`, and +`VERCEL_TOKEN`. Set `CREWAI_MODEL` in your own wrapper if you want to pass a model other than the +example's `openai/gpt-5-mini` default. + +The proof uses one live package-installed sandbox and one context-managed CrewAI MCP adapter. It: + +1. invokes `code_execute` twice directly and requires the same page ID and DOM marker; +2. records CrewAI's tool-usage event from a real model-selected `code_execute` call; +3. invokes the tool again to independently verify the model's browser-side change; +4. proves the model key and a host-only marker are absent inside generated code; and +5. closes the CrewAI MCP adapter before ending the sandbox lease, emitting `PASS` only afterward. + +## Use the agent + +Run the snippet with the example directory as the working directory so its local `agent.py` and +`sandbox.py` modules resolve: + +```bash +cd packages/integrations/examples/crewai +``` + +```python +from agent import run_stagehand_agent +from sandbox import StagehandSandboxLease + +with StagehandSandboxLease() as connection: + result = run_stagehand_agent( + connection, + "Open https://example.com and return its title and URL.", + ) + print(result) +``` + +The lease subprocess receives only Browserbase, Vercel, artifact, and runtime variables from its +allowlist. Outer model-provider credentials are intentionally excluded, and the sandbox foundation +brokers the Browserbase credential at its egress boundary. diff --git a/packages/integrations/examples/crewai/agent.py b/packages/integrations/examples/crewai/agent.py new file mode 100644 index 0000000000..a00e1b7498 --- /dev/null +++ b/packages/integrations/examples/crewai/agent.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import os +import queue +import threading +from builtins import BaseExceptionGroup +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from typing import Any, Protocol + +from crewai import Agent +from crewai.tools import BaseTool +from crewai_tools import MCPServerAdapter + +DEFAULT_STAGEHAND_LLM = os.environ.get("CREWAI_MODEL", "openai/gpt-5-mini") +ADAPTER_STOP_TIMEOUT_SECONDS = 30 + + +class StagehandCrewAIConnectionError(RuntimeError): + """CrewAI could not establish the authenticated Stagehand MCP connection.""" + + +class StagehandCrewAIToolContractError(RuntimeError): + """The remote Stagehand MCP tool contract was not the expected code-mode API.""" + + +class StagehandCrewAICleanupError(RuntimeError): + """CrewAI could not close its Stagehand MCP adapter.""" + + +class StagehandCrewAISetupError(RuntimeError): + """CrewAI could not construct the Stagehand browser agent.""" + + +class StagehandCrewAIRunError(RuntimeError): + """The CrewAI Stagehand browser run failed.""" + + +class StagehandSandboxConnection(Protocol): + url: str + token: str + + +@contextmanager +def stagehand_code_tools( + connection: StagehandSandboxConnection, +) -> Iterator[list[BaseTool]]: + """Keep one authenticated remote MCP client open for a complete CrewAI run.""" + adapter: MCPServerAdapter | None = None + try: + adapter = MCPServerAdapter( + { + "url": connection.url, + "transport": "streamable-http", + "headers": {"Authorization": f"Bearer {connection.token}"}, + }, + connect_timeout=60, + ) + tools = list(adapter.tools) + names = [tool.name for tool in tools] + if names != ["code_execute"]: + raise StagehandCrewAIToolContractError( + "The Stagehand MCP server returned an invalid tool contract." + ) + if "# Stagehand V4 code-mode syntax" not in tools[0].description: + raise StagehandCrewAIToolContractError( + "The Stagehand MCP server returned an invalid tool contract." + ) + except Exception as error: # noqa: BLE001 -- third-party discovery is untyped. + primary_error = ( + error + if isinstance(error, StagehandCrewAIToolContractError) + else StagehandCrewAIConnectionError( + "Could not connect CrewAI to the Stagehand MCP server." + ) + ) + if adapter is None: + raise primary_error from None + try: + stop_adapter(adapter) + except StagehandCrewAICleanupError as cleanup_error: + raise BaseExceptionGroup( + "CrewAI MCP setup and cleanup both failed", + [primary_error, cleanup_error], + ) + raise primary_error from None + + try: + yield tools + except BaseException as primary_error: + try: + stop_adapter(adapter) + except StagehandCrewAICleanupError as cleanup_error: + raise BaseExceptionGroup( + "CrewAI run and MCP cleanup both failed", + [primary_error, cleanup_error], + ) + raise + else: + stop_adapter(adapter) + + +def build_stagehand_agent( + tools: Sequence[BaseTool], + llm: str | Any = DEFAULT_STAGEHAND_LLM, +) -> Agent: + if len(tools) != 1 or tools[0].name != "code_execute": + raise StagehandCrewAIToolContractError( + "The Stagehand MCP server returned an invalid tool contract." + ) + try: + return Agent( + role="Stagehand browser agent", + goal="Complete browser tasks by writing compact, correct Stagehand V4 JavaScript.", + backstory=tools[0].description, + llm=llm, + tools=list(tools), + max_iter=8, + verbose=False, + ) + except Exception: # noqa: BLE001 -- CrewAI construction is an untyped boundary. + raise StagehandCrewAISetupError( + "Could not configure the CrewAI Stagehand agent." + ) from None + + +def run_stagehand_agent( + connection: StagehandSandboxConnection, + prompt: str, + llm: str | Any = DEFAULT_STAGEHAND_LLM, +) -> str: + with stagehand_code_tools(connection) as tools: + try: + return str(build_stagehand_agent(tools, llm).kickoff(prompt)) + except (StagehandCrewAISetupError, StagehandCrewAIToolContractError): + raise + except Exception: # noqa: BLE001 -- CrewAI kickoff is an untyped boundary. + raise StagehandCrewAIRunError( + "The CrewAI Stagehand agent run failed." + ) from None + + +def stop_adapter(adapter: MCPServerAdapter) -> None: + result: queue.Queue[BaseException | None] = queue.Queue(maxsize=1) + + def stop() -> None: + try: + adapter.stop() + result.put(None) + except BaseException as error: # noqa: BLE001 -- adapter shutdown is an untyped boundary. + result.put(error) + + # MCPServerAdapter exposes only a synchronous stop with no cancellation hook or + # documented thread affinity. A daemon supervisor bounds that call; after a + # timeout the sandbox lease may unwind while the orphaned stop finishes, and + # its result is intentionally ignored because cleanup failure was reported. + try: + threading.Thread( + target=stop, + name="stagehand-crewai-mcp-cleanup", + daemon=True, + ).start() + except BaseException: # noqa: BLE001 -- cleanup startup is an untyped boundary. + raise StagehandCrewAICleanupError( + "Could not close the CrewAI Stagehand MCP adapter." + ) from None + try: + error = result.get(timeout=ADAPTER_STOP_TIMEOUT_SECONDS) + except queue.Empty: + error = TimeoutError() + if error is not None: + raise StagehandCrewAICleanupError( + "Could not close the CrewAI Stagehand MCP adapter." + ) from None diff --git a/packages/integrations/examples/crewai/e2e.py b/packages/integrations/examples/crewai/e2e.py new file mode 100644 index 0000000000..f1ca2a654c --- /dev/null +++ b/packages/integrations/examples/crewai/e2e.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import json +import os +from typing import Any +from uuid import uuid4 + +from agent import build_stagehand_agent, stagehand_code_tools +from crewai.events import ToolUsageFinishedEvent, crewai_event_bus +from sandbox import StagehandSandboxLease + + +class StagehandCrewAIResultError(RuntimeError): + """code_execute returned a malformed or unsuccessful result.""" + + +class StagehandCrewAIIsolationError(RuntimeError): + """A host-only value crossed the Vercel Sandbox boundary.""" + + +def successful_result(raw_result: Any) -> dict[str, Any]: + try: + result = json.loads(str(raw_result)) + except (json.JSONDecodeError, TypeError, ValueError): + raise StagehandCrewAIResultError( + "Stagehand code_execute returned an invalid result." + ) from None + if not isinstance(result, dict) or result.get("ok") is not True: + raise StagehandCrewAIResultError( + "Stagehand code_execute returned an invalid result." + ) + value = result.get("value") + if not isinstance(value, dict): + raise StagehandCrewAIResultError( + "Stagehand code_execute returned an invalid result." + ) + return value + + +def main() -> None: + direct_marker = f"crewai-direct-{uuid4()}" + model_marker = f"crewai-model-{uuid4()}" + os.environ["CREWAI_HOST_ONLY_MARKER"] = f"host-{uuid4()}" + model_tool_calls: list[str] = [] + final_state: dict[str, Any] + + with ( + StagehandSandboxLease() as connection, + stagehand_code_tools(connection) as tools, + ): + code_execute = tools[0] + first = successful_result( + code_execute.run( + code=f""" +await page.goto("https://example.com", {{ waitUntil: "domcontentloaded" }}); +await page.evaluate((marker) => {{ + document.documentElement.dataset.crewaiDirectMarker = marker; +}}, {json.dumps(direct_marker)}); +return {{ + pageId: page.pageId, + title: await page.title(), + directMarker: await page.evaluate( + () => document.documentElement.dataset.crewaiDirectMarker, + ), + modelKeyVisible: process.env.OPENAI_API_KEY ?? null, + hostMarkerVisible: process.env.CREWAI_HOST_ONLY_MARKER ?? null, +}}; +""" + ) + ) + assert first["title"] == "Example Domain" + assert first["directMarker"] == direct_marker + if ( + first["modelKeyVisible"] is not None + or first["hostMarkerVisible"] is not None + ): + raise StagehandCrewAIIsolationError( + "A host-only value crossed the CrewAI sandbox boundary." + ) + + second = successful_result( + code_execute.run( + code=""" +return { + pageId: page.pageId, + title: await page.title(), + directMarker: await page.evaluate( + () => document.documentElement.dataset.crewaiDirectMarker, + ), +}; +""" + ) + ) + assert second["title"] == "Example Domain" + assert second["pageId"] == first["pageId"] + assert second["directMarker"] == direct_marker + + @crewai_event_bus.on(ToolUsageFinishedEvent) + def record_model_tool(_source: Any, event: ToolUsageFinishedEvent) -> None: + if event.tool_name == "code_execute": + model_tool_calls.append(event.tool_name) + + try: + agent = build_stagehand_agent(tools) + agent.kickoff( + " ".join( + ( + "Use code_execute to modify the already-open page.", + "Set document.documentElement.dataset.crewaiModelMarker to", + f"{json.dumps(model_marker)}.", + "Then read that dataset value and the current pageId and report them.", + "You must call code_execute; do not merely describe JavaScript.", + ) + ) + ) + assert crewai_event_bus.flush(), "CrewAI tool events did not finish" + finally: + crewai_event_bus.off(ToolUsageFinishedEvent, record_model_tool) + assert model_tool_calls, "the real CrewAI model must select code_execute" + + final_state = successful_result( + code_execute.run( + code=""" +return { + pageId: page.pageId, + title: await page.title(), + directMarker: await page.evaluate( + () => document.documentElement.dataset.crewaiDirectMarker, + ), + modelMarker: await page.evaluate( + () => document.documentElement.dataset.crewaiModelMarker, + ), +}; +""" + ) + ) + assert final_state["title"] == "Example Domain" + assert final_state["pageId"] == first["pageId"] + assert final_state["directMarker"] == direct_marker + assert final_state["modelMarker"] == model_marker + + print( + json.dumps( + { + "status": "PASS", + "framework": "crewai", + "directToolCalls": 3, + "modelToolCalls": len(model_tool_calls), + "sessionPersisted": True, + "modelCredentialIsolated": True, + "finalState": final_state, + "cleanup": ["crewai-mcp", "vercel-sandbox"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/packages/integrations/examples/crewai/requirements.txt b/packages/integrations/examples/crewai/requirements.txt new file mode 100644 index 0000000000..d0f49d9f56 --- /dev/null +++ b/packages/integrations/examples/crewai/requirements.txt @@ -0,0 +1,2 @@ +crewai==1.15.13 +crewai-tools[mcp]==1.15.13 diff --git a/packages/integrations/examples/crewai/sandbox.py b/packages/integrations/examples/crewai/sandbox.py new file mode 100644 index 0000000000..b70b33511b --- /dev/null +++ b/packages/integrations/examples/crewai/sandbox.py @@ -0,0 +1,21 @@ +"""Expose the shared Vercel Sandbox lease from the CrewAI example directory.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +SHARED_EXAMPLES_DIRECTORY = Path(__file__).resolve().parents[1] / "shared" +sys.path.insert(0, str(SHARED_EXAMPLES_DIRECTORY)) + +from vercel_sandbox_lease import ( + StagehandSandboxConnection, + StagehandSandboxLease, + StagehandSandboxLeaseError, +) + +__all__ = [ + "StagehandSandboxConnection", + "StagehandSandboxLease", + "StagehandSandboxLeaseError", +] diff --git a/packages/integrations/examples/crewai/test_agent.py b/packages/integrations/examples/crewai/test_agent.py new file mode 100644 index 0000000000..5bd3a5f57f --- /dev/null +++ b/packages/integrations/examples/crewai/test_agent.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import unittest +from builtins import BaseExceptionGroup +from threading import Event +from types import SimpleNamespace +from unittest.mock import patch + +import agent + + +class FakeAdapter: + def __init__( + self, tools: object, *, discovery_error: Exception | None = None + ) -> None: + self._tools = tools + self._discovery_error = discovery_error + self.stop_error: Exception | None = None + + @property + def tools(self) -> object: + if self._discovery_error is not None: + raise self._discovery_error + return self._tools + + def stop(self) -> None: + if self.stop_error is not None: + raise self.stop_error + + +class StagehandCodeToolsTest(unittest.TestCase): + connection = SimpleNamespace( + url="https://sandbox.example.test/mcp", token="secret-token" + ) + + def test_sanitizes_adapter_discovery_failure(self) -> None: + secret = "https://sandbox.example.test/mcp?token=do-not-reflect" + adapter = FakeAdapter([], discovery_error=RuntimeError(secret)) + with ( + patch.object(agent, "MCPServerAdapter", return_value=adapter), + self.assertRaises(agent.StagehandCrewAIConnectionError) as raised, + agent.stagehand_code_tools(self.connection), + ): + pass + self.assertNotIn(secret, str(raised.exception)) + + def test_rejects_unexpected_remote_tool_without_reflecting_name(self) -> None: + secret = "unexpected-secret-tool" + adapter = FakeAdapter( + [SimpleNamespace(name=secret, description="untrusted description")] + ) + with ( + patch.object(agent, "MCPServerAdapter", return_value=adapter), + self.assertRaises(agent.StagehandCrewAIToolContractError) as raised, + agent.stagehand_code_tools(self.connection), + ): + pass + self.assertNotIn(secret, str(raised.exception)) + + def test_sanitizes_adapter_cleanup_failure(self) -> None: + adapter = FakeAdapter( + [ + SimpleNamespace( + name="code_execute", + description="# Stagehand V4 code-mode syntax", + ) + ] + ) + adapter.stop_error = RuntimeError("cleanup-secret-do-not-reflect") + with ( + patch.object(agent, "MCPServerAdapter", return_value=adapter), + self.assertRaises(agent.StagehandCrewAICleanupError) as raised, + agent.stagehand_code_tools(self.connection), + ): + pass + self.assertEqual( + str(raised.exception), "Could not close the CrewAI Stagehand MCP adapter." + ) + + def test_adapter_cleanup_has_a_deadline(self) -> None: + release = Event() + adapter = FakeAdapter([]) + adapter.stop = release.wait + try: + with ( + patch.object(agent, "ADAPTER_STOP_TIMEOUT_SECONDS", 0.001), + self.assertRaises(agent.StagehandCrewAICleanupError), + ): + agent.stop_adapter(adapter) + finally: + release.set() + + def test_setup_and_cleanup_failures_preserve_order_and_types(self) -> None: + adapter = FakeAdapter([], discovery_error=RuntimeError("discovery-secret")) + adapter.stop_error = RuntimeError("cleanup-secret") + with ( + patch.object(agent, "MCPServerAdapter", return_value=adapter), + self.assertRaises(BaseExceptionGroup) as raised, + agent.stagehand_code_tools(self.connection), + ): + pass + + self.assertEqual(len(raised.exception.exceptions), 2) + self.assertIsInstance( + raised.exception.exceptions[0], agent.StagehandCrewAIConnectionError + ) + self.assertIsInstance( + raised.exception.exceptions[1], agent.StagehandCrewAICleanupError + ) + + def test_setup_and_cleanup_thread_start_failures_preserve_types(self) -> None: + adapter = FakeAdapter([], discovery_error=RuntimeError("discovery-secret")) + with ( + patch.object(agent, "MCPServerAdapter", return_value=adapter), + patch.object( + agent.threading.Thread, + "start", + side_effect=RuntimeError("thread-start-secret"), + ), + self.assertRaises(BaseExceptionGroup) as raised, + agent.stagehand_code_tools(self.connection), + ): + pass + + self.assertIsInstance( + raised.exception.exceptions[0], agent.StagehandCrewAIConnectionError + ) + self.assertIsInstance( + raised.exception.exceptions[1], agent.StagehandCrewAICleanupError + ) + + def test_run_and_cleanup_failures_preserve_order_and_types(self) -> None: + adapter = FakeAdapter( + [ + SimpleNamespace( + name="code_execute", + description="# Stagehand V4 code-mode syntax", + ) + ] + ) + adapter.stop_error = RuntimeError("cleanup-secret") + primary = ValueError("run failed") + with ( + patch.object(agent, "MCPServerAdapter", return_value=adapter), + self.assertRaises(BaseExceptionGroup) as raised, + agent.stagehand_code_tools(self.connection), + ): + raise primary + + self.assertEqual(len(raised.exception.exceptions), 2) + self.assertIs(raised.exception.exceptions[0], primary) + self.assertIsInstance( + raised.exception.exceptions[1], agent.StagehandCrewAICleanupError + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/integrations/examples/crewai/test_e2e.py b/packages/integrations/examples/crewai/test_e2e.py new file mode 100644 index 0000000000..644c5d0c0b --- /dev/null +++ b/packages/integrations/examples/crewai/test_e2e.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import json +import unittest + +from e2e import StagehandCrewAIResultError, successful_result + + +class SuccessfulResultTest(unittest.TestCase): + def test_accepts_successful_object_value(self) -> None: + value = {"title": "Example Domain"} + self.assertEqual( + successful_result(json.dumps({"ok": True, "value": value})), value + ) + + def test_rejects_invalid_json_without_reflecting_it(self) -> None: + secret = "not-json-do-not-reflect" + with self.assertRaises(StagehandCrewAIResultError) as raised: + successful_result(secret) + self.assertNotIn(secret, str(raised.exception)) + + def test_rejects_failed_result_without_reflecting_it(self) -> None: + secret = "remote-error-do-not-reflect" + with self.assertRaises(StagehandCrewAIResultError) as raised: + successful_result(json.dumps({"ok": False, "error": secret})) + self.assertNotIn(secret, str(raised.exception)) + + def test_rejects_non_object_value(self) -> None: + with self.assertRaises(StagehandCrewAIResultError): + successful_result(json.dumps({"ok": True, "value": ["unexpected"]})) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/integrations/examples/shared/test_vercel_sandbox_lease.py b/packages/integrations/examples/shared/test_vercel_sandbox_lease.py new file mode 100644 index 0000000000..a9ff0b465f --- /dev/null +++ b/packages/integrations/examples/shared/test_vercel_sandbox_lease.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import io +import os +import subprocess +import unittest +from builtins import BaseExceptionGroup +from pathlib import Path +from unittest.mock import patch + +import vercel_sandbox_lease as lease + +VALID_TOKEN = "A" * 43 +SHORT_TOKEN = "A" * 42 +LONG_TOKEN = "A" * 44 + + +class FakeProcess: + def __init__(self, *, exit_code: int = 0, stderr: str = "") -> None: + self.stdin = io.StringIO() + self.stdout = io.StringIO( + '{"url":"https://sandbox.example.vercel.run/mcp",' + f'"token":"{VALID_TOKEN}"}}\n' + ) + self.stderr = io.StringIO(stderr) + self.returncode: int | None = None + self._exit_code = exit_code + self.waited_after_stdin_closed = False + + def poll(self) -> int | None: + return self.returncode + + def wait(self, timeout: float) -> int: + del timeout + self.waited_after_stdin_closed = self.stdin.closed + self.returncode = self._exit_code + return self._exit_code + + def terminate(self) -> None: + self.returncode = -15 + + def kill(self) -> None: + self.returncode = -9 + + +class EscalatingFakeProcess(FakeProcess): + def __init__(self) -> None: + super().__init__() + self.wait_calls = 0 + self.terminated = False + self.killed = False + + def wait(self, timeout: float) -> int: + self.wait_calls += 1 + self.waited_after_stdin_closed = self.stdin.closed + if self.wait_calls < 3: + raise subprocess.TimeoutExpired("stagehand-lease", timeout) + self.returncode = -9 + return -9 + + def terminate(self) -> None: + self.terminated = True + + def kill(self) -> None: + self.killed = True + + +class UnstoppableFakeProcess(EscalatingFakeProcess): + def wait(self, timeout: float) -> int: + self.wait_calls += 1 + self.waited_after_stdin_closed = self.stdin.closed + raise subprocess.TimeoutExpired("stagehand-lease", timeout) + + +class StagehandSandboxLeaseTest(unittest.TestCase): + def test_model_credentials_are_excluded_from_lease_environment(self) -> None: + environment = { + "PATH": "/usr/bin", + "BROWSERBASE_API_KEY": "browserbase-secret", + "VERCEL_OIDC_TOKEN": "vercel-token", + "OPENAI_API_KEY": "model-secret", + "ANTHROPIC_API_KEY": "other-model-secret", + } + with patch.dict(os.environ, environment, clear=True): + child_environment = lease.lease_environment() + + self.assertEqual(child_environment["BROWSERBASE_API_KEY"], "browserbase-secret") + self.assertEqual(child_environment["VERCEL_OIDC_TOKEN"], "vercel-token") + self.assertNotIn("OPENAI_API_KEY", child_environment) + self.assertNotIn("ANTHROPIC_API_KEY", child_environment) + + def test_context_returns_connection_then_closes_stdin_before_waiting(self) -> None: + process = FakeProcess() + with ( + patch.object(lease, "TSX_PATH", Path(__file__)), + patch.object(lease, "LEASE_PATH", Path(__file__)), + patch.object(subprocess, "Popen", return_value=process), + lease.StagehandSandboxLease() as connection, + ): + self.assertEqual(connection.url, "https://sandbox.example.vercel.run/mcp") + self.assertEqual(connection.token, VALID_TOKEN) + + self.assertTrue(process.waited_after_stdin_closed) + + def test_cleanup_failure_preserves_primary_failure(self) -> None: + secret = "cleanup-secret-do-not-reflect" + process = FakeProcess(exit_code=1, stderr=f"cleanup failed: {secret}\n") + with ( + self.assertRaises(BaseExceptionGroup) as raised, + patch.object(lease, "TSX_PATH", Path(__file__)), + patch.object(lease, "LEASE_PATH", Path(__file__)), + patch.object(subprocess, "Popen", return_value=process), + lease.StagehandSandboxLease(), + ): + raise ValueError("primary failed") + + self.assertEqual(len(raised.exception.exceptions), 2) + self.assertIsInstance(raised.exception.exceptions[0], ValueError) + self.assertIsInstance( + raised.exception.exceptions[1], lease.StagehandSandboxLeaseError + ) + self.assertNotIn(secret, str(raised.exception.exceptions[1])) + + def test_setup_timeout_is_typed_and_closes_the_owner(self) -> None: + process = FakeProcess() + with ( + patch.object(lease, "TSX_PATH", Path(__file__)), + patch.object(lease, "LEASE_PATH", Path(__file__)), + patch.object(subprocess, "Popen", return_value=process), + patch.object( + lease.queue.Queue, + "get", + side_effect=lease.queue.Empty, + ), + self.assertRaises(lease.StagehandSandboxLeaseError) as raised, + ): + lease.StagehandSandboxLease().start() + + self.assertEqual( + str(raised.exception), "Stagehand sandbox lease timed out while starting" + ) + self.assertTrue(process.waited_after_stdin_closed) + + def test_keyboard_interrupt_during_setup_is_preserved(self) -> None: + process = FakeProcess() + with ( + patch.object(lease, "TSX_PATH", Path(__file__)), + patch.object(lease, "LEASE_PATH", Path(__file__)), + patch.object(subprocess, "Popen", return_value=process), + patch.object( + lease.queue.Queue, + "get", + side_effect=KeyboardInterrupt, + ), + self.assertRaises(KeyboardInterrupt), + ): + lease.StagehandSandboxLease().start() + + self.assertTrue(process.waited_after_stdin_closed) + + def test_unexpected_setup_exception_is_sanitized(self) -> None: + process = FakeProcess() + with ( + patch.object(lease, "TSX_PATH", Path(__file__)), + patch.object(lease, "LEASE_PATH", Path(__file__)), + patch.object(subprocess, "Popen", return_value=process), + patch.object( + lease.queue.Queue, + "get", + side_effect=ValueError("setup-secret-do-not-reflect"), + ), + self.assertRaises(lease.StagehandSandboxLeaseError) as raised, + ): + lease.StagehandSandboxLease().start() + + self.assertEqual( + str(raised.exception), "Stagehand sandbox lease failed during setup" + ) + self.assertTrue(process.waited_after_stdin_closed) + + def test_connection_parser_rejects_untrusted_shapes(self) -> None: + invalid_connections = ( + "not-json", + f'{{"url":"http://sandbox.example.test/mcp","token":"{VALID_TOKEN}"}}', + f'{{"url":"https://sandbox.example.test/not-mcp","token":"{VALID_TOKEN}"}}', + f'{{"url":"https://user:pass@sandbox.example.test/mcp","token":"{VALID_TOKEN}"}}', + f'{{"url":"https://sandbox.example.test/mcp?x=1","token":"{VALID_TOKEN}"}}', + f'{{"url":"https://sandbox.example.test/mcp#fragment","token":"{VALID_TOKEN}"}}', + f'{{"url":"https://sandbox.example.test/mcp","token":"{SHORT_TOKEN}"}}', + f'{{"url":"https://sandbox.example.test/mcp","token":"{LONG_TOKEN}"}}', + '{"url":"https://sandbox.example.test/mcp","token":""}', + ) + for connection in invalid_connections: + with ( + self.subTest(connection=connection), + self.assertRaises(lease.StagehandSandboxLeaseError), + ): + lease.parse_connection(connection) + + def test_close_escalates_from_terminate_to_kill(self) -> None: + process = EscalatingFakeProcess() + owner = lease.StagehandSandboxLease() + owner._process = process + + with self.assertRaises(lease.StagehandSandboxLeaseError): + owner.close() + + self.assertTrue(process.waited_after_stdin_closed) + self.assertTrue(process.terminated) + self.assertTrue(process.killed) + self.assertEqual(process.wait_calls, 3) + + def test_close_reports_owner_that_survives_kill(self) -> None: + process = UnstoppableFakeProcess() + owner = lease.StagehandSandboxLease() + owner._process = process + + with self.assertRaises(lease.StagehandSandboxLeaseError) as raised: + owner.close() + + self.assertEqual( + str(raised.exception), + "Stagehand sandbox lease could not stop the trusted owner", + ) + self.assertTrue(process.waited_after_stdin_closed) + self.assertTrue(process.terminated) + self.assertTrue(process.killed) + self.assertEqual(process.wait_calls, 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/integrations/examples/shared/vercel_sandbox_lease.py b/packages/integrations/examples/shared/vercel_sandbox_lease.py new file mode 100644 index 0000000000..1a58a1b59c --- /dev/null +++ b/packages/integrations/examples/shared/vercel_sandbox_lease.py @@ -0,0 +1,244 @@ +"""Own the TypeScript Vercel Sandbox lease from a Python framework example.""" + +from __future__ import annotations + +import json +import os +import queue +import re +import subprocess +import threading +from builtins import BaseExceptionGroup +from dataclasses import dataclass +from pathlib import Path +from types import TracebackType +from typing import TextIO +from urllib.parse import urlsplit + +REPOSITORY_ROOT = Path(__file__).resolve().parents[4] +LEASE_PATH = ( + REPOSITORY_ROOT / "packages/integrations/examples/vercel-sandbox/src/lease.ts" +) +TSX_PATH = REPOSITORY_ROOT / "node_modules/.bin/tsx" +LEASE_SETUP_TIMEOUT_SECONDS = 3 * 60 +LEASE_CLEANUP_TIMEOUT_SECONDS = 60 +LEASE_TERMINATION_TIMEOUT_SECONDS = 60 +LEASE_KILL_TIMEOUT_SECONDS = 5 +LEASE_ENVIRONMENT_KEYS = ( + "PATH", + "HOME", + "TMPDIR", + "LANG", + "LC_ALL", + "SSL_CERT_FILE", + "NODE_EXTRA_CA_CERTS", + "NODE_OPTIONS", + "CI", + "STAGEHAND_SANDBOX_ARTIFACTS", + "BROWSERBASE_API_KEY", + "BROWSERBASE_PROJECT_ID", + "VERCEL_OIDC_TOKEN", + "VERCEL_TEAM_ID", + "VERCEL_PROJECT_ID", + "VERCEL_TOKEN", +) + + +class StagehandSandboxLeaseError(RuntimeError): + """The trusted sandbox owner could not start or clean up its lease.""" + + +@dataclass(frozen=True) +class StagehandSandboxConnection: + url: str + token: str + + +class StagehandSandboxLease: + """Start the reviewed Node lease and close it after the Python MCP client.""" + + def __init__(self) -> None: + self._process: subprocess.Popen[str] | None = None + self._stderr_thread: threading.Thread | None = None + self._closed = False + + def __enter__(self) -> StagehandSandboxConnection: + return self.start() + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool: + del exc_type, traceback + try: + self.close() + except BaseException as cleanup_error: + if exc_value is not None: + raise BaseExceptionGroup( + "Python framework run and Vercel Sandbox cleanup both failed", + [exc_value, cleanup_error], + ) + raise + return False + + def start(self) -> StagehandSandboxConnection: + if self._process is not None: + raise StagehandSandboxLeaseError( + "Stagehand sandbox lease was already started" + ) + if not TSX_PATH.is_file() or not LEASE_PATH.is_file(): + raise StagehandSandboxLeaseError( + "Install the workspace before starting the Stagehand sandbox lease" + ) + + try: + process = subprocess.Popen( + [str(TSX_PATH), str(LEASE_PATH)], + cwd=REPOSITORY_ROOT, + env=lease_environment(), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + bufsize=1, + ) + except (OSError, subprocess.SubprocessError): + raise self._lease_error("could not start the trusted owner") from None + self._process = process + assert process.stdout is not None + assert process.stderr is not None + self._stderr_thread = threading.Thread( + target=self._drain_stderr, + args=(process.stderr,), + name="stagehand-sandbox-lease-stderr", + daemon=True, + ) + self._stderr_thread.start() + + line_queue: queue.Queue[str] = queue.Queue(maxsize=1) + line_thread = threading.Thread( + target=lambda: line_queue.put(process.stdout.readline()), + name="stagehand-sandbox-lease-connection", + daemon=True, + ) + line_thread.start() + + try: + try: + line = line_queue.get(timeout=LEASE_SETUP_TIMEOUT_SECONDS) + except queue.Empty: + raise self._lease_error("timed out while starting") from None + if not line: + raise self._lease_error("exited before returning a connection") + connection = parse_connection(line) + if process.poll() is not None: + raise self._lease_error( + "exited immediately after returning a connection" + ) + return connection + except BaseException as error: # noqa: BLE001 -- cleanup must preserve any primary failure. + primary_error = ( + error + if not isinstance(error, Exception) + or isinstance(error, StagehandSandboxLeaseError) + else self._lease_error("failed during setup") + ) + try: + self.close() + except BaseException as cleanup_error: # noqa: BLE001 -- preserve both failures. + raise BaseExceptionGroup( + "Vercel Sandbox lease setup and cleanup both failed", + [primary_error, cleanup_error], + ) + raise primary_error from None + + def close(self) -> None: + if self._closed: + return + self._closed = True + process = self._process + if process is None: + return + + try: + assert process.stdin is not None + try: + process.stdin.close() + except BrokenPipeError: + pass + + try: + return_code = process.wait(timeout=LEASE_CLEANUP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + process.terminate() + try: + return_code = process.wait( + timeout=LEASE_TERMINATION_TIMEOUT_SECONDS + ) + except subprocess.TimeoutExpired: + process.kill() + try: + return_code = process.wait(timeout=LEASE_KILL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + raise self._lease_error( + "could not stop the trusted owner" + ) from None + except (OSError, subprocess.SubprocessError): + raise self._lease_error("could not stop the trusted owner") from None + + if self._stderr_thread is not None: + self._stderr_thread.join(timeout=1) + if return_code != 0: + raise self._lease_error(f"exited with status {return_code}") + + def _drain_stderr(self, stderr: TextIO) -> None: + for _line in stderr: + pass + + def _lease_error(self, message: str) -> StagehandSandboxLeaseError: + return StagehandSandboxLeaseError(f"Stagehand sandbox lease {message}") + + +def lease_environment() -> dict[str, str]: + """Return only the trusted lease inputs; outer model credentials stay outside.""" + return { + key: value + for key in LEASE_ENVIRONMENT_KEYS + if (value := os.environ.get(key)) is not None + } + + +def parse_connection(line: str) -> StagehandSandboxConnection: + try: + value = json.loads(line) + url = value["url"] + token = value["token"] + except (json.JSONDecodeError, KeyError, TypeError): + raise StagehandSandboxLeaseError( + "Stagehand sandbox lease returned an invalid connection" + ) from None + if ( + not isinstance(url, str) + or not isinstance(token, str) + or re.fullmatch(r"[A-Za-z0-9_-]{43}", token) is None + ): + raise StagehandSandboxLeaseError( + "Stagehand sandbox lease returned an invalid connection" + ) + parsed = urlsplit(url) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.path != "/mcp" + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise StagehandSandboxLeaseError( + "Stagehand sandbox lease returned an invalid connection" + ) + return StagehandSandboxConnection(url=url, token=token) diff --git a/packages/integrations/examples/vercel-sandbox/src/lease.test.mjs b/packages/integrations/examples/vercel-sandbox/src/lease.test.mjs index 5780eb200b..bea55cecd2 100644 --- a/packages/integrations/examples/vercel-sandbox/src/lease.test.mjs +++ b/packages/integrations/examples/vercel-sandbox/src/lease.test.mjs @@ -1,7 +1,10 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; +import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import test from "node:test"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const EXIT_TIMEOUT_MS = 2_000; @@ -45,3 +48,99 @@ test("lease setup failure exits while the parent keeps stdin open", async () => assert.equal(Buffer.concat(stdout).length, 0); assert.match(Buffer.concat(stderr).toString(), /^Stagehand sandbox lease failed: Missing /); }); + +test("lease preserves signal exit semantics during setup", async () => { + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), "stagehand-lease-signal-")); + const artifactRoot = path.join(temporaryRoot, "artifacts"); + const packageRoot = path.join(artifactRoot, "packages"); + const runtimeRoot = path.join(artifactRoot, "runtime"); + const readyPath = path.join(temporaryRoot, "fetch-ready"); + const preloadPath = path.join(temporaryRoot, "block-fetch.mjs"); + const dependencies = { + "@browserbasehq/stagehand": "file:../packages/stagehand.tgz", + "@browserbasehq/stagehand-codemode": "file:../packages/stagehand-codemode.tgz", + supergateway: "3.4.3", + }; + await Promise.all([ + mkdir(packageRoot, { recursive: true }), + mkdir(runtimeRoot, { recursive: true }), + ]); + await Promise.all([ + writeFile(path.join(packageRoot, "stagehand.tgz"), Buffer.from([0x1f, 0x8b])), + writeFile(path.join(packageRoot, "stagehand-codemode.tgz"), Buffer.from([0x1f, 0x8b])), + writeFile(path.join(runtimeRoot, "package.json"), JSON.stringify({ dependencies })), + writeFile( + path.join(runtimeRoot, "package-lock.json"), + JSON.stringify({ + lockfileVersion: 3, + packages: { "": { dependencies } }, + }), + ), + writeFile( + preloadPath, + [ + 'import { writeFileSync } from "node:fs";', + "globalThis.fetch = (_input, init = {}) => {", + ' writeFileSync(process.env.STAGEHAND_SIGNAL_READY_FILE, "ready");', + " return new Promise((_resolve, reject) => {", + ' init.signal?.addEventListener("abort", () => reject(new Error("aborted")), { once: true });', + " });", + "};", + ].join("\n"), + ), + ]); + + const child = spawn( + process.execPath, + [ + fileURLToPath(import.meta.resolve("tsx/cli")), + fileURLToPath(new URL("./lease.ts", import.meta.url)), + ], + { + env: { + PATH: process.env.PATH ?? "", + NODE_OPTIONS: `--import=${pathToFileURL(preloadPath).href}`, + STAGEHAND_SIGNAL_READY_FILE: readyPath, + STAGEHAND_SANDBOX_ARTIFACTS: artifactRoot, + BROWSERBASE_API_KEY: "unused-test-key", + BROWSERBASE_PROJECT_ID: "unused-test-project", + }, + stdio: ["pipe", "pipe", "pipe"], + }, + ); + + try { + await waitForFile(readyPath); + child.kill("SIGTERM"); + const exit = await waitForExit(child, "Lease did not preserve the setup signal"); + // The production lease is launched through the tsx executable, whose signal + // proxy reports the conventional 128 + SIGTERM status rather than signalCode. + assert.deepEqual(exit, { code: 143, signal: null }); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await rm(temporaryRoot, { force: true, recursive: true }); + } +}); + +async function waitForFile(filePath) { + const deadline = Date.now() + EXIT_TIMEOUT_MS; + while (Date.now() < deadline) { + try { + await access(filePath); + return; + } catch { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } + throw new Error("Lease did not reach the blocked setup request"); +} + +async function waitForExit(child, message) { + let timeout; + return Promise.race([ + new Promise((resolve) => child.once("close", (code, signal) => resolve({ code, signal }))), + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(message)), EXIT_TIMEOUT_MS); + }), + ]).finally(() => clearTimeout(timeout)); +} diff --git a/packages/integrations/examples/vercel-sandbox/src/lease.ts b/packages/integrations/examples/vercel-sandbox/src/lease.ts index 96fb9d7956..6259691877 100644 --- a/packages/integrations/examples/vercel-sandbox/src/lease.ts +++ b/packages/integrations/examples/vercel-sandbox/src/lease.ts @@ -6,17 +6,22 @@ const SHUTDOWN_FALLBACK_MS = 35_000; type LeaseEnd = { signal?: NodeJS.Signals }; +let setupSignal: NodeJS.Signals | undefined; try { const packageArtifactsPath = requiredEnvironment("STAGEHAND_SANDBOX_ARTIFACTS"); const browserbaseApiKey = requiredEnvironment("BROWSERBASE_API_KEY"); const browserbaseProjectId = requiredEnvironment("BROWSERBASE_PROJECT_ID"); const vercelCredentials = vercelCredentialsFromEnvironment(); - const leaseEnd = waitForLeaseEnd(); + const setupAbort = new AbortController(); + const leaseEnd = waitForLeaseEnd(setupAbort, (signal) => { + setupSignal = signal; + }); const connection = await createStagehandSandbox({ packageArtifactsPath, browserbaseApiKey, browserbaseProjectId, vercelCredentials, + signal: setupAbort.signal, }); process.stdout.write( @@ -39,21 +44,32 @@ try { if (signal) forwardSignal(signal); } catch (error) { process.stdin.pause(); + if (setupSignal) forwardSignal(setupSignal); process.stderr.write(`Stagehand sandbox lease failed: ${safeMessage(error)}\n`); process.exitCode = 1; } -function waitForLeaseEnd(): Promise { +function waitForLeaseEnd( + setupAbort: AbortController, + recordSignal: (signal: NodeJS.Signals) => void, +): Promise { return new Promise((resolve) => { let finished = false; const finish = (end: LeaseEnd = {}) => { if (finished) return; finished = true; + setupAbort.abort(); resolve(end); }; - process.once("SIGINT", () => finish({ signal: "SIGINT" })); - process.once("SIGTERM", () => finish({ signal: "SIGTERM" })); + process.once("SIGINT", () => { + recordSignal("SIGINT"); + finish({ signal: "SIGINT" }); + }); + process.once("SIGTERM", () => { + recordSignal("SIGTERM"); + finish({ signal: "SIGTERM" }); + }); process.stdin.once("end", () => finish()); process.stdin.once("close", () => finish()); process.stdin.resume(); diff --git a/packages/integrations/examples/vercel-sandbox/src/sandbox.test.ts b/packages/integrations/examples/vercel-sandbox/src/sandbox.test.ts index 0e19789276..15d28fbb0b 100644 --- a/packages/integrations/examples/vercel-sandbox/src/sandbox.test.ts +++ b/packages/integrations/examples/vercel-sandbox/src/sandbox.test.ts @@ -1,4 +1,6 @@ import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { getEventListeners } from "node:events"; import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -67,6 +69,114 @@ void test("runtime lock rejects non-string resolved values", async () => { } }); +void test("an already-aborted setup never creates a sandbox", async () => { + const controller = new AbortController(); + controller.abort(); + const createMock = mock.method(Sandbox, "create", async () => { + throw new Error("Sandbox.create must not run"); + }); + try { + await assert.rejects( + createStagehandSandbox({ + packageArtifactsPath: path.join(os.tmpdir(), "unused-stagehand-artifacts"), + browserbaseApiKey: "unused-test-key", + browserbaseProjectId: "unused-test-project", + signal: controller.signal, + }), + { name: "StagehandSandboxSetupError" }, + ); + assert.equal(createMock.mock.callCount(), 0); + assert.equal(getEventListeners(controller.signal, "abort").length, 0); + } finally { + createMock.mock.restore(); + } +}); + +void test("abort during CDP discovery is reported as setup cancellation", async () => { + const artifactRoot = await writeArtifacts("https://registry.npmjs.org/supergateway.tgz"); + const controller = new AbortController(); + let discoveryStarted!: () => void; + const started = new Promise((resolve) => { + discoveryStarted = resolve; + }); + const fetchMock = mock.method(globalThis, "fetch", async (_input, init) => { + discoveryStarted(); + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(new Error("aborted")), { + once: true, + }); + }); + }); + const createMock = mock.method(Sandbox, "create", async () => { + throw new Error("Sandbox.create must not run"); + }); + + try { + const setup = createStagehandSandbox({ + packageArtifactsPath: artifactRoot, + browserbaseApiKey: "unused-test-key", + browserbaseProjectId: "unused-test-project", + signal: controller.signal, + }); + await started; + controller.abort(); + await assert.rejects(setup, { name: "StagehandSandboxSetupError" }); + assert.equal(createMock.mock.callCount(), 0); + assert.equal(getEventListeners(controller.signal, "abort").length, 0); + } finally { + createMock.mock.restore(); + fetchMock.mock.restore(); + await rm(artifactRoot, { force: true, recursive: true }); + } +}); + +void test("abort during readiness stops polling, clears listeners, and disposes", async () => { + const resolved = "https://registry.npmjs.org/supergateway.tgz"; + const artifactRoot = await writeArtifacts(resolved); + const controller = new AbortController(); + const fake = readySandboxFake(resolved); + let healthRequests = 0; + const fetchMock = mock.method(globalThis, "fetch", async (input) => { + const url = new URL(input.toString()); + if (url.hostname === "api.browserbase.com" && url.pathname === "/v1/sessions") { + return { + ok: true, + json: async () => ({ + id: "discovery-session", + connectUrl: "wss://connect.browserbase.com/devtools/browser/test", + }), + } as Response; + } + if (url.hostname === "api.browserbase.com") return { ok: true } as Response; + healthRequests += 1; + if (healthRequests === 1) setTimeout(() => controller.abort(), 10); + return { ok: false, status: 503 } as Response; + }); + const createMock = mock.method(Sandbox, "create", async () => fake.sandbox); + + try { + await assert.rejects( + createStagehandSandbox({ + packageArtifactsPath: artifactRoot, + browserbaseApiKey: "unused-test-key", + browserbaseProjectId: "unused-test-project", + readinessTimeoutMs: 5_000, + signal: controller.signal, + }), + { name: "StagehandSandboxSetupError" }, + ); + await new Promise((resolve) => setTimeout(resolve, 300)); + assert.equal(healthRequests, 1); + assert.equal(fake.stop.mock.callCount(), 1); + assert.equal(fake.deleteSandbox.mock.callCount(), 1); + assert.equal(getEventListeners(controller.signal, "abort").length, 0); + } finally { + createMock.mock.restore(); + fetchMock.mock.restore(); + await rm(artifactRoot, { force: true, recursive: true }); + } +}); + void test("Sandbox.create receives only allowlisted Vercel credentials", async () => { const createOptions = await captureSandboxCreateOptions({ teamId: "expected-team", @@ -139,13 +249,53 @@ async function writeArtifacts(resolved: unknown): Promise { const artifactRoot = await mkdtemp(path.join(os.tmpdir(), "stagehand-artifacts-")); const packageRoot = path.join(artifactRoot, "packages"); const runtimeRoot = path.join(artifactRoot, "runtime"); + const contents = artifactContents(resolved); await Promise.all([mkdir(packageRoot), mkdir(runtimeRoot)]); await Promise.all([ - writeFile(path.join(packageRoot, "stagehand.tgz"), Buffer.from([0x1f, 0x8b])), - writeFile(path.join(packageRoot, "stagehand-codemode.tgz"), Buffer.from([0x1f, 0x8b])), - writeFile(path.join(runtimeRoot, "package.json"), JSON.stringify({ dependencies })), - writeFile( - path.join(runtimeRoot, "package-lock.json"), + writeFile(path.join(packageRoot, "stagehand.tgz"), contents.gzip), + writeFile(path.join(packageRoot, "stagehand-codemode.tgz"), contents.gzip), + writeFile(path.join(runtimeRoot, "package.json"), contents.manifest), + writeFile(path.join(runtimeRoot, "package-lock.json"), contents.lock), + ]); + return artifactRoot; +} + +function readySandboxFake(resolved: unknown) { + const contents = artifactContents(resolved); + const sha256 = (content: Buffer) => createHash("sha256").update(content).digest("hex"); + const artifactDigests = new Map([ + ["/vercel/sandbox/packages/stagehand.tgz", sha256(contents.gzip)], + ["/vercel/sandbox/packages/stagehand-codemode.tgz", sha256(contents.gzip)], + ["/vercel/sandbox/stagehand-runtime/package.json", sha256(contents.manifest)], + ["/vercel/sandbox/stagehand-runtime/package-lock.json", sha256(contents.lock)], + ]); + const runCommand = mock.fn(async ({ cmd, args }: { cmd: string; args?: string[] }) => ({ + exitCode: cmd === "sudo" ? 1 : 0, + stdout: async () => + cmd === "sha256sum" ? `${artifactDigests.get(args?.[0] ?? "")} ${args?.[0]}\n` : "", + stderr: async () => "", + })); + const user = { runCommand }; + const stop = mock.fn(async () => undefined); + const deleteSandbox = mock.fn(async () => undefined); + const sandbox = { + runCommand, + writeFiles: async () => undefined, + createUser: async () => user, + asUser: () => user, + update: async () => undefined, + domain: () => "https://sandbox.example.vercel.run", + stop, + delete: deleteSandbox, + } as unknown as Sandbox; + return { sandbox, stop, deleteSandbox }; +} + +function artifactContents(resolved: unknown) { + return { + gzip: Buffer.from([0x1f, 0x8b]), + manifest: Buffer.from(JSON.stringify({ dependencies })), + lock: Buffer.from( JSON.stringify({ lockfileVersion: 3, packages: { @@ -154,6 +304,5 @@ async function writeArtifacts(resolved: unknown): Promise { }, }), ), - ]); - return artifactRoot; + }; } diff --git a/packages/integrations/examples/vercel-sandbox/src/sandbox.ts b/packages/integrations/examples/vercel-sandbox/src/sandbox.ts index 4a2fb82f5a..24a73c5473 100644 --- a/packages/integrations/examples/vercel-sandbox/src/sandbox.ts +++ b/packages/integrations/examples/vercel-sandbox/src/sandbox.ts @@ -87,6 +87,7 @@ export type StagehandSandboxOptions = { readinessTimeoutMs?: number; sandboxTimeoutMs?: number; cleanupTimeoutMs?: number; + signal?: AbortSignal; }; export type StagehandSandboxConnection = { @@ -105,9 +106,11 @@ export async function createStagehandSandbox( ): Promise { assertNonEmpty(options.browserbaseApiKey, "browserbaseApiKey"); assertNonEmpty(options.browserbaseProjectId, "browserbaseProjectId"); + assertNotAborted(options.signal); const artifacts = await loadPackageArtifacts(options); const cdpHost = await discoverBrowserbaseCdpHost(options); + assertNotAborted(options.signal); let sandbox: Sandbox; try { const vercelCredentials = options.vercelCredentials; @@ -133,51 +136,61 @@ export async function createStagehandSandbox( const close = sandboxCloser(sandbox, options.cleanupTimeoutMs ?? 30_000); try { - await installStagehandPackages(sandbox, artifacts); - await sandbox.writeFiles([ - { - path: AUTH_PROXY_PATH, - content: await readFile(new URL("./guest/auth-proxy.mjs", import.meta.url)), - mode: 0o555, - }, - { - path: STDIO_WRAPPER_PATH, - content: await readFile(new URL("./guest/stdio-wrapper.mjs", import.meta.url)), - mode: 0o555, - }, - ]); + assertNotAborted(options.signal); + await abortable(installStagehandPackages(sandbox, artifacts), options.signal); + await abortable( + sandbox.writeFiles([ + { + path: AUTH_PROXY_PATH, + content: await readFile(new URL("./guest/auth-proxy.mjs", import.meta.url)), + mode: 0o555, + }, + { + path: STDIO_WRAPPER_PATH, + content: await readFile(new URL("./guest/stdio-wrapper.mjs", import.meta.url)), + mode: 0o555, + }, + ]), + options.signal, + ); - const mcpUser = await sandbox.createUser(MCP_USER); - const proxyUser = await sandbox.createUser(PROXY_USER); - await assertUnprivileged(mcpUser, MCP_USER); - await assertUnprivileged(proxyUser, PROXY_USER); - await protectRuntimeFiles(sandbox); + const mcpUser = await abortable(sandbox.createUser(MCP_USER), options.signal); + const proxyUser = await abortable(sandbox.createUser(PROXY_USER), options.signal); + await abortable(assertUnprivileged(mcpUser, MCP_USER), options.signal); + await abortable(assertUnprivileged(proxyUser, PROXY_USER), options.signal); + await abortable(protectRuntimeFiles(sandbox), options.signal); // This update is the trust transition: everything above is trusted setup; // everything below may eventually execute model-generated JavaScript. - await sandbox.update({ - networkPolicy: { - allow: { - [BROWSERBASE_API_HOST]: [ - { - transform: [{ headers: { "X-BB-API-Key": options.browserbaseApiKey } }], - }, - ], - [cdpHost]: [], + await abortable( + sandbox.update({ + networkPolicy: { + allow: { + [BROWSERBASE_API_HOST]: [ + { + transform: [{ headers: { "X-BB-API-Key": options.browserbaseApiKey } }], + }, + ], + [cdpHost]: [], + }, }, - }, - }); + }), + options.signal, + ); const token = randomBytes(32).toString("base64url"); const tokenDigest = createHash("sha256").update(token).digest("hex"); - await startGateway(mcpUser, options.browserbaseProjectId); - await startAuthProxy(proxyUser, tokenDigest); + await abortable(startGateway(mcpUser, options.browserbaseProjectId), options.signal); + await abortable(startAuthProxy(proxyUser, tokenDigest), options.signal); const origin = new URL(sandbox.domain(BRIDGE_PORT)); - await waitForHealth(origin, token, options.readinessTimeoutMs ?? 2 * 60_000); - const unauthorized = await fetch(new URL("/healthz", origin), { - signal: AbortSignal.timeout(HEALTH_REQUEST_TIMEOUT_MS), - }).catch(() => undefined); + await waitForHealth(origin, token, options.readinessTimeoutMs ?? 2 * 60_000, options.signal); + const unauthorized = await abortable( + fetch(new URL("/healthz", origin), { + signal: AbortSignal.timeout(HEALTH_REQUEST_TIMEOUT_MS), + }).catch(() => undefined), + options.signal, + ); if (unauthorized?.status !== 401) { throw new StagehandSandboxHealthError(); } @@ -302,7 +315,13 @@ async function assertUnprivileged(user: SandboxUser, name: string): Promise { +async function waitForHealth( + origin: URL, + token: string, + timeoutMs: number, + signal?: AbortSignal, +): Promise { + assertNotAborted(signal); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const requestTimeoutMs = Math.max( @@ -311,10 +330,13 @@ async function waitForHealth(origin: URL, token: string, timeoutMs: number): Pro ); const response = await fetch(new URL("/healthz", origin), { headers: { Authorization: `Bearer ${token}` }, - signal: AbortSignal.timeout(requestTimeoutMs), + signal: signal + ? AbortSignal.any([signal, AbortSignal.timeout(requestTimeoutMs)]) + : AbortSignal.timeout(requestTimeoutMs), }).catch(() => undefined); + assertNotAborted(signal); if (response?.ok) return; - await delay(Math.min(250, Math.max(0, deadline - Date.now()))); + await delay(Math.min(250, Math.max(0, deadline - Date.now())), signal); } throw new StagehandSandboxHealthError(); } @@ -332,8 +354,11 @@ async function discoverBrowserbaseCdpHost(options: StagehandSandboxOptions): Pro "X-BB-API-Key": options.browserbaseApiKey, }, body: JSON.stringify({ projectId: options.browserbaseProjectId }), - signal: AbortSignal.timeout(30_000), + signal: options.signal + ? AbortSignal.any([options.signal, AbortSignal.timeout(30_000)]) + : AbortSignal.timeout(30_000), }); + assertNotAborted(options.signal); if (!response.ok) { throw new Error(`Browserbase CDP host discovery returned ${response.status}`); } @@ -344,7 +369,7 @@ async function discoverBrowserbaseCdpHost(options: StagehandSandboxOptions): Pro } discoveredHost = assertBrowserbaseCdpHost(new URL(session.connectUrl).hostname); } catch (error) { - primaryError = error; + primaryError = options.signal?.aborted ? new StagehandSandboxSetupError() : error; } let cleanupError: unknown = NO_ERROR; @@ -370,6 +395,7 @@ async function discoverBrowserbaseCdpHost(options: StagehandSandboxOptions): Pro } if (primaryError !== NO_ERROR || cleanupError !== NO_ERROR || !discoveredHost) { + if (primaryError instanceof StagehandSandboxSetupError) throw primaryError; throw new StagehandCdpDiscoveryError(); } return discoveredHost; @@ -438,8 +464,40 @@ function assertBrowserbaseCdpHost(hostname: string): string { return hostname; } -function delay(milliseconds: number): Promise { - return new Promise((resolve) => setTimeout(resolve, milliseconds)); +function delay(milliseconds: number, signal?: AbortSignal): Promise { + if (!signal) return new Promise((resolve) => setTimeout(resolve, milliseconds)); + assertNotAborted(signal); + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, milliseconds); + const onAbort = () => { + clearTimeout(timeout); + signal.removeEventListener("abort", onAbort); + reject(new StagehandSandboxSetupError()); + }; + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +function assertNotAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new StagehandSandboxSetupError(); +} + +async function abortable(promise: Promise, signal: AbortSignal | undefined): Promise { + if (!signal) return promise; + assertNotAborted(signal); + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(new StagehandSandboxSetupError()); + signal.addEventListener("abort", onAbort, { once: true }); + }); + try { + return await Promise.race([promise, aborted]); + } finally { + if (onAbort) signal.removeEventListener("abort", onAbort); + } } type PackageArtifact = { content: Buffer; sha256: string };