diff --git a/.github/workflows/codemode-framework-examples.yml b/.github/workflows/codemode-framework-examples.yml index b512da03b6..c6ca1a1eb4 100644 --- a/.github/workflows/codemode-framework-examples.yml +++ b/.github/workflows/codemode-framework-examples.yml @@ -194,3 +194,65 @@ jobs: VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + langchain: + name: LangChain Deep Agents + if: >- + github.event_name == 'push' || + github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'safe-to-test') + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - uses: ./.github/actions/setup-node-pnpm + with: + use-prebuilt-artifacts: "false" + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: packages/integrations/examples/langchain/requirements.txt + + - run: python -m pip install -r packages/integrations/examples/langchain/requirements.txt + - run: >- + python -m py_compile + packages/integrations/examples/shared/vercel_sandbox_lease.py + packages/integrations/examples/shared/test_vercel_sandbox_lease.py + packages/integrations/examples/langchain/agent.py + packages/integrations/examples/langchain/sandbox.py + packages/integrations/examples/langchain/test_agent.py + packages/integrations/examples/langchain/test_e2e.py + packages/integrations/examples/langchain/e2e.py + - run: python packages/integrations/examples/shared/test_vercel_sandbox_lease.py + - run: python packages/integrations/examples/langchain/test_agent.py + - run: python packages/integrations/examples/langchain/test_e2e.py + - run: pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts + - name: Detect LangChain live test credentials + id: langchain-live-credentials + uses: ./.github/actions/detect-codemode-live-credentials + with: + require-openai: "true" + env: + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + - name: Run LangChain live sandbox proof + if: steps.langchain-live-credentials.outputs.available == 'true' + run: python packages/integrations/examples/langchain/e2e.py + env: + STAGEHAND_SANDBOX_ARTIFACTS: ${{ github.workspace }}/packages/integrations/examples/vercel-sandbox/.artifacts + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/packages/integrations/README.md b/packages/integrations/README.md index e0f5708e79..3507efce28 100644 --- a/packages/integrations/README.md +++ b/packages/integrations/README.md @@ -83,3 +83,5 @@ implementation modules and an in-process arbitrary-code executor are not public - [Mastra](./examples/mastra) consumes that connection with one persistent remote MCP client. - [CrewAI](./examples/crewai) keeps its context-managed MCP adapter open across every tool call in one crew execution. +- [LangChain Deep Agents](./examples/langchain) uses one explicit MCP session so every tool call + reaches the same browser. diff --git a/packages/integrations/examples/langchain/README.md b/packages/integrations/examples/langchain/README.md new file mode 100644 index 0000000000..8ad85508fb --- /dev/null +++ b/packages/integrations/examples/langchain/README.md @@ -0,0 +1,75 @@ +# LangChain Deep Agents with Stagehand code mode + +This example gives a Deep Agent one browser tool, `code_execute`, without running generated +JavaScript in the Python agent process. A small trusted Python lease owner starts the +package-installed [Vercel Sandbox example](../vercel-sandbox) and receives its `{ url, token }` +connection. LangChain then connects directly through authenticated Streamable HTTP: + +```text +Deep Agent -> persistent LangChain MCP session -> authenticated HTTPS -> Vercel Sandbox + `-> Stagehand MCP + `-> generated JavaScript +``` + +The integration discovers the package's canonical tool description and uses it as the agent system +prompt. It does not copy the executor, schema, or code-mode skill. + +## Setup + +Create a Python 3.12 environment and install the exact tested dependencies: + +```bash +python3.12 -m venv .venv +. .venv/bin/activate +python -m pip install -r packages/integrations/examples/langchain/requirements.txt +``` + +Build and pack the exact Stagehand packages under review: + +```bash +pnpm install +pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts +``` + +## Run the end-to-end proof + +```bash +STAGEHAND_SANDBOX_ARTIFACTS="$PWD/packages/integrations/examples/vercel-sandbox/.artifacts" \ +BROWSERBASE_API_KEY= \ +BROWSERBASE_PROJECT_ID= \ +VERCEL_OIDC_TOKEN= \ +OPENAI_API_KEY= \ +python packages/integrations/examples/langchain/e2e.py +``` + +For external CI, replace `VERCEL_OIDC_TOKEN` with `VERCEL_TEAM_ID`, `VERCEL_PROJECT_ID`, and +`VERCEL_TOKEN`. `STAGEHAND_LANGCHAIN_MODEL` selects the host model for the standalone agent and +defaults to `openai:gpt-5-mini`. + +The proof uses one live package-installed sandbox and one explicit LangChain MCP session. It: + +1. invokes `code_execute` twice directly and requires the same page ID and DOM marker; +2. requires a real Deep Agent model to select `code_execute` inside that same session; +3. invokes the tool again to independently verify the model's browser-side change; +4. proves the model key and a host-only marker are absent inside generated code; and +5. closes the LangChain MCP session before ending the sandbox lease, emitting `PASS` only afterward. + +The explicit `client.session("stagehand")` context and `load_mcp_tools(session)` call are important. +LangChain's convenience tool loader is stateless by default and would otherwise create a fresh MCP +session for each invocation, losing Stagehand's browser state. + +## Run a Deep Agent + +```bash +STAGEHAND_SANDBOX_ARTIFACTS="$PWD/packages/integrations/examples/vercel-sandbox/.artifacts" \ +BROWSERBASE_API_KEY= \ +BROWSERBASE_PROJECT_ID= \ +VERCEL_OIDC_TOKEN= \ +OPENAI_API_KEY= \ +python packages/integrations/examples/langchain/agent.py \ + "Open https://example.com and return its title and URL." +``` + +The lease subprocess receives only Browserbase, Vercel, artifact, and runtime variables from its +allowlist. Outer model-provider credentials are intentionally excluded, and the sandbox foundation +brokers the Browserbase credential at its egress boundary. diff --git a/packages/integrations/examples/langchain/agent.py b/packages/integrations/examples/langchain/agent.py new file mode 100644 index 0000000000..cc09d60e10 --- /dev/null +++ b/packages/integrations/examples/langchain/agent.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import asyncio +import os +import sys +from builtins import BaseExceptionGroup +from collections.abc import AsyncIterator, Sequence +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any, Protocol + +from deepagents import create_deep_agent +from langchain_core.tools import BaseTool +from langchain_mcp_adapters.client import MultiServerMCPClient +from langchain_mcp_adapters.tools import load_mcp_tools +from mcp import ClientSession + +DEFAULT_STAGEHAND_MODEL = "openai:gpt-5-mini" + + +class StagehandLangChainConnectionError(RuntimeError): + """LangChain could not establish the authenticated Stagehand MCP connection.""" + + +class StagehandLangChainToolContractError(RuntimeError): + """The remote Stagehand MCP tool contract was not the expected code-mode API.""" + + +class StagehandLangChainCleanupError(RuntimeError): + """LangChain could not close its Stagehand MCP session.""" + + +class StagehandLangChainSetupError(RuntimeError): + """LangChain could not construct the Stagehand browser agent.""" + + +class StagehandLangChainRunError(RuntimeError): + """The LangChain Stagehand browser run failed.""" + + +class StagehandSandboxConnection(Protocol): + url: str + token: str + + +def create_stagehand_mcp_client( + connection: StagehandSandboxConnection, +) -> MultiServerMCPClient: + """Create a native authenticated Streamable HTTP client.""" + return MultiServerMCPClient( + { + "stagehand": { + "transport": "streamable_http", + "url": connection.url, + "headers": {"Authorization": f"Bearer {connection.token}"}, + } + }, + tool_name_prefix=False, + handle_tool_errors=True, + ) + + +async def load_stagehand_code_tool(session: ClientSession) -> BaseTool: + """Discover the one canonical tool without creating another MCP session.""" + try: + tools = await load_mcp_tools(session) + except Exception: # noqa: BLE001 -- MCP discovery is an untyped boundary. + raise StagehandLangChainConnectionError( + "Could not discover the Stagehand MCP tool." + ) from None + tool_names = [tool.name for tool in tools] + if tool_names != ["code_execute"]: + raise StagehandLangChainToolContractError( + "The Stagehand MCP server returned an invalid tool contract." + ) + if "# Stagehand V4 code-mode syntax" not in tools[0].description: + raise StagehandLangChainToolContractError( + "The Stagehand MCP server returned an invalid tool contract." + ) + return tools[0] + + +@asynccontextmanager +async def stagehand_code_session( + connection: StagehandSandboxConnection, +) -> AsyncIterator[BaseTool]: + """Keep one remote MCP session open for every tool call in an agent run.""" + try: + client = create_stagehand_mcp_client(connection) + except Exception: # noqa: BLE001 -- client construction is an untyped boundary. + raise StagehandLangChainConnectionError( + "Could not connect LangChain to the Stagehand MCP server." + ) from None + + primary_error: BaseException | None = None + session_entered = False + try: + async with client.session("stagehand") as session: + session_entered = True + try: + code_tool = await load_stagehand_code_tool(session) + yield code_tool + except BaseException as error: + primary_error = error + raise + except ( + StagehandLangChainConnectionError, + StagehandLangChainToolContractError, + ): + raise + except BaseException as error: + if primary_error is not None: + if error is primary_error: + raise + raise BaseExceptionGroup( + "LangChain run and MCP cleanup both failed", + [ + primary_error, + StagehandLangChainCleanupError( + "Could not close the LangChain Stagehand MCP session." + ), + ], + ) from None + if not isinstance(error, Exception): + raise + if session_entered: + raise StagehandLangChainCleanupError( + "Could not close the LangChain Stagehand MCP session." + ) from None + raise StagehandLangChainConnectionError( + "Could not connect LangChain to the Stagehand MCP server." + ) from None + + +def build_stagehand_agent( + code_tool: BaseTool, + model: str | Any = DEFAULT_STAGEHAND_MODEL, +) -> Any: + if code_tool.name != "code_execute": + raise StagehandLangChainToolContractError( + "The Stagehand MCP server returned an invalid tool contract." + ) + try: + return create_deep_agent( + model=model, + tools=[code_tool], + system_prompt=code_tool.description, + ) + except Exception: # noqa: BLE001 -- Deep Agent construction is untyped. + raise StagehandLangChainSetupError( + "Could not configure the LangChain Stagehand agent." + ) from None + + +def stagehand_tool_call_count(result: dict[str, Any]) -> int: + """Count model-selected code_execute calls in a LangGraph result.""" + count = 0 + for message in result.get("messages", []): + for call in getattr(message, "tool_calls", []): + name = ( + call.get("name") + if isinstance(call, dict) + else getattr(call, "name", None) + ) + if name == "code_execute": + count += 1 + return count + + +async def run_stagehand_agent( + connection: StagehandSandboxConnection, + prompt: str, + model: str | Any = DEFAULT_STAGEHAND_MODEL, +) -> dict[str, Any]: + async with stagehand_code_session(connection) as code_tool: + try: + return await build_stagehand_agent(code_tool, model).ainvoke( + {"messages": [{"role": "user", "content": prompt}]}, + config={"recursion_limit": 20}, + ) + except ( + StagehandLangChainSetupError, + StagehandLangChainToolContractError, + ): + raise + except Exception: # noqa: BLE001 -- Deep Agent execution is untyped. + raise StagehandLangChainRunError( + "The LangChain Stagehand agent run failed." + ) from None + + +def _last_message_text(result: dict[str, Any]) -> str: + messages: Sequence[Any] = result.get("messages", []) + if not messages: + return str(result) + content = getattr(messages[-1], "content", messages[-1]) + return content if isinstance(content, str) else str(content) + + +async def _main() -> None: + if len(sys.argv) < 2: + raise SystemExit(f'Usage: {Path(sys.argv[0]).name} ""') + from sandbox import StagehandSandboxLease + + prompt = " ".join(sys.argv[1:]) + model = os.environ.get("STAGEHAND_LANGCHAIN_MODEL", DEFAULT_STAGEHAND_MODEL) + with StagehandSandboxLease() as connection: + result = await run_stagehand_agent(connection, prompt, model=model) + print(_last_message_text(result)) + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/packages/integrations/examples/langchain/e2e.py b/packages/integrations/examples/langchain/e2e.py new file mode 100644 index 0000000000..3f0d04b2bd --- /dev/null +++ b/packages/integrations/examples/langchain/e2e.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import asyncio +import json +import os +from typing import Any +from uuid import uuid4 + +from agent import ( + build_stagehand_agent, + stagehand_code_session, + stagehand_tool_call_count, +) +from sandbox import StagehandSandboxLease + + +class StagehandLangChainResultError(RuntimeError): + """code_execute returned a malformed or unsuccessful result.""" + + +class StagehandLangChainIsolationError(RuntimeError): + """A host-only value crossed the Vercel Sandbox boundary.""" + + +def successful_result(raw_result: Any) -> dict[str, Any]: + try: + if isinstance(raw_result, str): + result = json.loads(raw_result) + elif isinstance(raw_result, dict): + structured = raw_result.get("structuredContent") + result = structured if isinstance(structured, dict) else raw_result + elif isinstance(raw_result, list) and len(raw_result) == 1: + block = raw_result[0] + text = ( + block.get("text") + if isinstance(block, dict) + else getattr(block, "text", None) + ) + if not isinstance(text, str): + raise StagehandLangChainResultError( + "Stagehand code_execute returned an invalid result." + ) + result = json.loads(text) + else: + raise StagehandLangChainResultError( + "Stagehand code_execute returned an invalid result." + ) + except StagehandLangChainResultError: + raise + except Exception: # noqa: BLE001 -- MCP result objects are untyped. + raise StagehandLangChainResultError( + "Stagehand code_execute returned an invalid result." + ) from None + + if not isinstance(result, dict) or result.get("ok") is not True: + raise StagehandLangChainResultError( + "Stagehand code_execute returned an invalid result." + ) + value = result.get("value") + if not isinstance(value, dict): + raise StagehandLangChainResultError( + "Stagehand code_execute returned an invalid result." + ) + return value + + +async def main() -> None: + direct_marker = f"langchain-direct-{uuid4()}" + model_marker = f"langchain-model-{uuid4()}" + os.environ["LANGCHAIN_HOST_ONLY_MARKER"] = f"host-{uuid4()}" + final_state: dict[str, Any] + model_tool_calls = 0 + + with StagehandSandboxLease() as connection: + async with stagehand_code_session(connection) as code_tool: + first = successful_result( + await code_tool.ainvoke( + { + "code": f""" +await page.goto("https://example.com", {{ waitUntil: "domcontentloaded" }}); +await page.evaluate((marker) => {{ + document.documentElement.dataset.langchainDirectMarker = marker; +}}, {json.dumps(direct_marker)}); +return {{ + pageId: page.pageId, + title: await page.title(), + directMarker: await page.evaluate( + () => document.documentElement.dataset.langchainDirectMarker, + ), + modelKeyVisible: process.env.OPENAI_API_KEY ?? null, + hostMarkerVisible: process.env.LANGCHAIN_HOST_ONLY_MARKER ?? null, +}}; +""" + } + ) + ) + assert first["title"] == "Example Domain" + assert first["directMarker"] == direct_marker + if ( + first["modelKeyVisible"] is not None + or first["hostMarkerVisible"] is not None + ): + raise StagehandLangChainIsolationError( + "A host-only value crossed the LangChain sandbox boundary." + ) + + second = successful_result( + await code_tool.ainvoke( + { + "code": """ +return { + pageId: page.pageId, + title: await page.title(), + directMarker: await page.evaluate( + () => document.documentElement.dataset.langchainDirectMarker, + ), +}; +""" + } + ) + ) + assert second["title"] == "Example Domain" + assert second["pageId"] == first["pageId"] + assert second["directMarker"] == direct_marker + + agent = build_stagehand_agent(code_tool) + agent_result = await agent.ainvoke( + { + "messages": [ + { + "role": "user", + "content": " ".join( + ( + "Use code_execute to modify the already-open page.", + "Set document.documentElement.dataset.langchainModelMarker to", + f"{json.dumps(model_marker)}.", + "Then read that value and the current pageId and report them.", + "You must call code_execute; do not merely describe JavaScript.", + ) + ), + } + ] + }, + config={"recursion_limit": 20}, + ) + model_tool_calls = stagehand_tool_call_count(agent_result) + assert model_tool_calls, ( + "the real Deep Agent model must select code_execute" + ) + + final_state = successful_result( + await code_tool.ainvoke( + { + "code": """ +return { + pageId: page.pageId, + title: await page.title(), + directMarker: await page.evaluate( + () => document.documentElement.dataset.langchainDirectMarker, + ), + modelMarker: await page.evaluate( + () => document.documentElement.dataset.langchainModelMarker, + ), +}; +""" + } + ) + ) + assert final_state["title"] == "Example Domain" + assert final_state["pageId"] == first["pageId"] + assert final_state["directMarker"] == direct_marker + assert final_state["modelMarker"] == model_marker + + print( + json.dumps( + { + "status": "PASS", + "framework": "langchain-deep-agents", + "directToolCalls": 3, + "modelToolCalls": model_tool_calls, + "sessionPersisted": True, + "modelCredentialIsolated": True, + "finalState": final_state, + "cleanup": ["langchain-mcp", "vercel-sandbox"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/packages/integrations/examples/langchain/requirements.txt b/packages/integrations/examples/langchain/requirements.txt new file mode 100644 index 0000000000..62bf74adb0 --- /dev/null +++ b/packages/integrations/examples/langchain/requirements.txt @@ -0,0 +1,4 @@ +deepagents==0.7.5 +langchain-mcp-adapters==0.3.2 +langchain-openai==1.4.2 +mcp==1.29.0 diff --git a/packages/integrations/examples/langchain/sandbox.py b/packages/integrations/examples/langchain/sandbox.py new file mode 100644 index 0000000000..95cfd9d80e --- /dev/null +++ b/packages/integrations/examples/langchain/sandbox.py @@ -0,0 +1,21 @@ +"""Expose the shared Vercel Sandbox lease from the LangChain example directory.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +SHARED_EXAMPLES_DIRECTORY = Path(__file__).resolve().parents[1] / "shared" +sys.path.insert(0, str(SHARED_EXAMPLES_DIRECTORY)) + +from vercel_sandbox_lease import ( + StagehandSandboxConnection, + StagehandSandboxLease, + StagehandSandboxLeaseError, +) + +__all__ = [ + "StagehandSandboxConnection", + "StagehandSandboxLease", + "StagehandSandboxLeaseError", +] diff --git a/packages/integrations/examples/langchain/test_agent.py b/packages/integrations/examples/langchain/test_agent.py new file mode 100644 index 0000000000..442a52cfa2 --- /dev/null +++ b/packages/integrations/examples/langchain/test_agent.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import unittest +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import agent + + +class LangChainStagehandAgentTest(unittest.IsolatedAsyncioTestCase): + def test_client_uses_authenticated_streamable_http(self) -> None: + client = agent.create_stagehand_mcp_client( + SimpleNamespace(url="https://sandbox.example/mcp", token="secret") + ) + + self.assertEqual( + client.connections, + { + "stagehand": { + "transport": "streamable_http", + "url": "https://sandbox.example/mcp", + "headers": {"Authorization": "Bearer secret"}, + } + }, + ) + + async def test_tool_discovery_requires_canonical_code_execute(self) -> None: + tool = SimpleNamespace( + name="code_execute", + description="# Stagehand V4 code-mode syntax\nUse Stagehand.", + ) + with patch.object(agent, "load_mcp_tools", AsyncMock(return_value=[tool])): + self.assertIs(await agent.load_stagehand_code_tool(SimpleNamespace()), tool) + + async def test_tool_discovery_rejects_noncanonical_description(self) -> None: + tool = SimpleNamespace( + name="code_execute", description="Execute arbitrary code." + ) + with ( + patch.object(agent, "load_mcp_tools", AsyncMock(return_value=[tool])), + self.assertRaises(agent.StagehandLangChainToolContractError), + ): + await agent.load_stagehand_code_tool(SimpleNamespace()) + + async def test_tool_discovery_sanitizes_adapter_failure(self) -> None: + secret = "https://sandbox.example.test/mcp?token=do-not-reflect" + with ( + patch.object( + agent, + "load_mcp_tools", + AsyncMock(side_effect=RuntimeError(secret)), + ), + self.assertRaises(agent.StagehandLangChainConnectionError) as raised, + ): + await agent.load_stagehand_code_tool(SimpleNamespace()) + + self.assertNotIn(secret, str(raised.exception)) + + async def test_session_scopes_loaded_tool_to_one_client_session(self) -> None: + connection = SimpleNamespace( + url="https://sandbox.example.test/mcp", token="secret" + ) + session = SimpleNamespace() + code_tool = SimpleNamespace( + name="code_execute", + description="# Stagehand V4 code-mode syntax", + ) + session_names: list[str] = [] + + @asynccontextmanager + async def session_context(name: str): + session_names.append(name) + yield session + + client = SimpleNamespace(session=session_context) + load_tool = AsyncMock(return_value=code_tool) + with ( + patch.object(agent, "create_stagehand_mcp_client", return_value=client), + patch.object(agent, "load_stagehand_code_tool", load_tool), + ): + async with agent.stagehand_code_session(connection) as loaded: + self.assertIs(loaded, code_tool) + + self.assertEqual(session_names, ["stagehand"]) + load_tool.assert_awaited_once_with(session) + + async def test_session_reports_cleanup_only_failure_without_reflection(self) -> None: + secret = "https://sandbox.example.test/mcp?token=do-not-reflect" + connection = SimpleNamespace( + url="https://sandbox.example.test/mcp", token="secret" + ) + session = SimpleNamespace() + code_tool = SimpleNamespace( + name="code_execute", + description="# Stagehand V4 code-mode syntax", + ) + + class CleanupFailingSession: + async def __aenter__(self): + return session + + async def __aexit__(self, *_args): + raise RuntimeError(secret) + + client = SimpleNamespace(session=lambda _name: CleanupFailingSession()) + with ( + patch.object(agent, "create_stagehand_mcp_client", return_value=client), + patch.object( + agent, + "load_stagehand_code_tool", + AsyncMock(return_value=code_tool), + ), + self.assertRaises(agent.StagehandLangChainCleanupError) as raised, + ): + async with agent.stagehand_code_session(connection) as loaded: + self.assertIs(loaded, code_tool) + + self.assertEqual( + str(raised.exception), + "Could not close the LangChain Stagehand MCP session.", + ) + self.assertNotIn(secret, str(raised.exception)) + + def test_build_agent_uses_canonical_tool_description_as_system_prompt(self) -> None: + code_tool = SimpleNamespace( + name="code_execute", + description="# Stagehand V4 code-mode syntax\nCanonical instructions.", + ) + built_agent = SimpleNamespace() + with patch.object( + agent, "create_deep_agent", return_value=built_agent + ) as create_agent: + result = agent.build_stagehand_agent(code_tool, model="test-model") + + self.assertIs(result, built_agent) + create_agent.assert_called_once_with( + model="test-model", + tools=[code_tool], + system_prompt=code_tool.description, + ) + + def test_build_agent_rejects_non_code_execute_tool_without_reflection(self) -> None: + secret = "unexpected-secret-tool" + code_tool = SimpleNamespace(name=secret, description="untrusted") + + with self.assertRaises(agent.StagehandLangChainToolContractError) as raised: + agent.build_stagehand_agent(code_tool) + + self.assertNotIn(secret, str(raised.exception)) + + async def test_run_agent_uses_one_session_tool_and_bounded_recursion(self) -> None: + connection = SimpleNamespace( + url="https://sandbox.example.test/mcp", token="secret" + ) + code_tool = SimpleNamespace(name="code_execute", description="canonical") + expected = {"messages": []} + deep_agent = SimpleNamespace(ainvoke=AsyncMock(return_value=expected)) + + @asynccontextmanager + async def code_session(actual_connection): + self.assertIs(actual_connection, connection) + yield code_tool + + with ( + patch.object(agent, "stagehand_code_session", code_session), + patch.object( + agent, "build_stagehand_agent", return_value=deep_agent + ) as build_agent, + ): + result = await agent.run_stagehand_agent( + connection, "Open example.com", model="test-model" + ) + + self.assertIs(result, expected) + build_agent.assert_called_once_with(code_tool, "test-model") + deep_agent.ainvoke.assert_awaited_once_with( + { + "messages": [ + {"role": "user", "content": "Open example.com"}, + ] + }, + config={"recursion_limit": 20}, + ) + + def test_model_tool_call_counter_ignores_other_tools(self) -> None: + result = { + "messages": [ + SimpleNamespace( + tool_calls=[ + {"name": "write_todos"}, + {"name": "code_execute"}, + ] + ) + ] + } + self.assertEqual(agent.stagehand_tool_call_count(result), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/integrations/examples/langchain/test_e2e.py b/packages/integrations/examples/langchain/test_e2e.py new file mode 100644 index 0000000000..2cfc84b848 --- /dev/null +++ b/packages/integrations/examples/langchain/test_e2e.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import unittest + +from e2e import StagehandLangChainResultError, successful_result + + +class SuccessfulResultTest(unittest.TestCase): + def test_accepts_successful_structured_result(self) -> None: + self.assertEqual( + successful_result( + {"structuredContent": {"ok": True, "value": {"title": "Example"}}} + ), + {"title": "Example"}, + ) + + def test_rejects_invalid_json_without_reflecting_payload(self) -> None: + secret = "invalid-json-secret-do-not-reflect" + with self.assertRaises(StagehandLangChainResultError) as raised: + successful_result(secret) + self.assertNotIn(secret, str(raised.exception)) + + def test_rejects_failed_result_without_reflecting_payload(self) -> None: + secret = "failure-secret-do-not-reflect" + with self.assertRaises(StagehandLangChainResultError) as raised: + successful_result({"ok": False, "error": secret}) + self.assertNotIn(secret, str(raised.exception)) + + def test_rejects_non_object_value(self) -> None: + with self.assertRaises(StagehandLangChainResultError): + successful_result({"ok": True, "value": ["unexpected"]}) + + +if __name__ == "__main__": + unittest.main()