diff --git a/.agents/references/sandbox-runtime-boundary.md b/.agents/references/sandbox-runtime-boundary.md index 9afae44e84..221c7e16ec 100644 --- a/.agents/references/sandbox-runtime-boundary.md +++ b/.agents/references/sandbox-runtime-boundary.md @@ -44,6 +44,14 @@ Resolve the session source in this order: injected live session, resumable sandb - Temporary clones, mounts, sinks, and dependency resources need failure cleanup during partial startup as well as normal shutdown. - Capability tools should report bounded output and preserve provider exit status or structured error data without exposing private runtime metadata to the model. +## Remote Mount Simplicity Boundary + +Remote mounts should default to one narrow lifecycle: declare them during sandbox creation, keep their contents outside workspace persistence, and unmount them during close. When tar persistence or hydration requires detaching a mount, restore it immediately afterward. Mount credentials must remain trusted live configuration and must not be reconstructed from serialized session state. + +Treat dynamic mount mutation, native-snapshot-backed mounts, and resumable mounts as opt-in provider capabilities rather than default requirements. If a privileged mount transition becomes ambiguous, stop the sandbox instead of adding reconciliation or recovery state. Do not add credential resolvers, refresh loops, persisted mount registries, or dynamic mount APIs unless the provider exposes a trusted primitive that makes the lifecycle transition unambiguous and the change is supported by focused provider evidence. + +Provider adapters may deliberately support a narrower lifecycle. Document that boundary next to the adapter state that enforces it so future maintainers do not mistake an intentional exclusion for an unfinished feature. The Vercel S3 adapter follows the create-time-only form of this policy: its trusted mount configuration is live-session-only, sessions containing mounts cannot resume, and mount topology cannot change after creation. + ## Review Checklist 1. Name the owner of every live session, provider client, mount, process, capability, and temporary resource. @@ -63,6 +71,8 @@ Resolve the session source in this order: injected live session, resumable sandb - `src/agents/sandbox/materialization.py` - `src/agents/sandbox/workspace_paths.py` - `src/agents/sandbox/session/archive_extraction.py` +- `src/agents/extensions/sandbox/vercel/mounts.py` +- `src/agents/extensions/sandbox/vercel/sandbox.py` - `tests/sandbox/test_runtime.py` - `tests/sandbox/test_runtime_agent_preparation.py` - `tests/sandbox/test_session_state_roundtrip.py` diff --git a/.agents/skills/integration-tests/SKILL.md b/.agents/skills/integration-tests/SKILL.md new file mode 100644 index 0000000000..1866d674cb --- /dev/null +++ b/.agents/skills/integration-tests/SKILL.md @@ -0,0 +1,64 @@ +--- +name: integration-tests +description: Run the packaged OpenAI Agents Python SDK integration tests from clean wheel and source-distribution environments. Use for release readiness, live OpenAI regression checks, package import compatibility, optional-extra validation, or when asked to run integration tests after examples-auto-run. +--- + +# Integration Tests + +## Overview + +Run the release-oriented integration suite against the exact wheel and source distribution produced by `uv build`. The runner installs both artifacts into isolated environments and validates supported imports, optional extras, OpenAI model adapters, hosted tools, Realtime, and voice workflows. + +## Execution requirements + +- Fresh isolated environments download optional dependencies from PyPI and connect to the configured API providers. +- When the execution environment requires approval for package downloads or configured provider connections, request elevated command execution (`sandbox_permissions=require_escalated`). Retry with the required network permissions before classifying a connectivity failure as an SDK regression. + +## Release workflow + +Run this command from the repository root: + +```bash +env UV_DEFAULT_INDEX=https://pypi.org/simple \ + OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS=1 \ + OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS=0 \ + make integration-tests-release +``` + +- Use the release profile as the default whenever `$integration-tests` is invoked without a narrower request. +- Use OpenRouter as the standard multi-provider gateway. Add provider-specific direct connections only when the user explicitly requests that additional credential matrix. +- Use existing `OPENAI_API_KEY` and `OPENROUTER_API_KEY` values without printing them. Missing optional service configuration may skip capability-specific tests unless strict mode was explicitly requested. +- The command rebuilds the wheel and source distribution, creates isolated virtual environments, checks public imports and optional dependencies, and runs the release-oriented live suites. +- Do not run watch mode, modify source files, create a branch, commit, push, or open a pull request as part of this skill. + +## Paired release validation + +When the user requests both pre-release checks, run `$examples-auto-run` first and follow that skill's required per-example behavioral validation. Then run the command above and report the examples and integration outcomes separately. Invoking `$integration-tests` alone does not implicitly start the examples suite. + +## Focused commands + +Use a focused target only when the user specifically asks to narrow the run: + +```bash +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-packaging +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-core +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-providers +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-hosted +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-realtime +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-voice +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-extras +``` + +For the minimum supported Python package boundary, use: + +```bash +env UV_DEFAULT_INDEX=https://pypi.org/simple \ + OPENAI_AGENTS_INTEGRATION_PYTHON=3.10 \ + make integration-tests-packaging +``` + +Nightly and manual profiles include additional capability-specific or higher-cost checks. Run them only when explicitly requested; use the configured OpenRouter matrix by default and include direct providers only when explicitly selected. + +## Reporting + +Report the final pass, fail, skip, and deselection counts for each isolated environment. If a command fails, identify the exact profile, package environment, failing test, and actionable error. Separate product regressions from missing credentials, unsupported hosted features, dependency installation failures, and execution-environment restrictions. diff --git a/.agents/skills/integration-tests/agents/openai.yaml b/.agents/skills/integration-tests/agents/openai.yaml new file mode 100644 index 0000000000..cd918c14f2 --- /dev/null +++ b/.agents/skills/integration-tests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Integration Tests" + short_description: "Run packaged Python SDK integration tests" + default_prompt: "Use $integration-tests to run the packaged Python SDK integration suite." diff --git a/.agents/skills/sensitive-logging-audit/SKILL.md b/.agents/skills/sensitive-logging-audit/SKILL.md new file mode 100644 index 0000000000..ca150f4240 --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/SKILL.md @@ -0,0 +1,78 @@ +--- +name: sensitive-logging-audit +description: Audit and fix sensitive-data exposure through Python runtime logging in openai-agents-python. Use when reviewing logging, print, warnings, stderr, traceback, MCP names, model or tool exceptions, redaction flags, or any diagnostic path that may retain user data. +--- + +# Sensitive Logging Audit + +## Objective + +Find candidate output sinks, trace their values manually, fix demonstrated leaks at shared runtime boundaries, and prove redaction with adversarial tests. + +The collector is only a syntax-based search aid. It does not resolve Python aliases or control flow, certify policy guards, or prove that an absent candidate is safe. + +## Workflow + +### 1. Establish the review surface + +- Work in the current checkout and preserve unrelated changes. +- Read `src/agents/_debug.py`, `src/agents/logger.py`, and the affected callers. +- Treat exception messages, arguments, tracebacks, causes, contexts, notes, names, URLs, and arbitrary values as potentially sensitive. +- Read [the Python redaction validation matrix](references/redaction-validation.md). + +Run the collector tests, then collect candidates: + +```bash +uv run python .agents/skills/sensitive-logging-audit/scripts/test_inventory.py +uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ + --format json --output /tmp/sensitive-logging-candidates.json +``` + +The report intentionally contains no `policy`, `safe`, or guard classification. + +### 2. Supplement the collector with source search + +The collector does not follow assignments such as `emit = logger.error`. Search the source directly and inspect aliases, callbacks, wrappers, and reflective dispatch: + +```bash +rg -n '\.(debug|info|warning|warn|error|exception|critical|fatal|log)\b' src/agents +rg -n '\b(print|pprint|pp|warn|warn_explicit|write|writelines|print_exc|print_exception)\b' src/agents +rg -n 'DONT_LOG_(MODEL|TOOL)_DATA|log_(model|tool|model_and_tool)_action' src/agents +``` + +Do not turn collector coverage or a textual guard into a security conclusion. Trace producers and callers. + +### 3. Classify manually + +Assign each reviewed path one disposition: + +- `model`: model requests, responses, Realtime events, or derived values. +- `tool`: tool arguments, outputs, MCP data, tool events, or derived values. +- `model+tool`: either class may reach the sink. +- `operational`: demonstrated to contain only non-sensitive SDK metadata. +- `intentional-output`: explicitly user-facing output rather than diagnostics. +- `uncertain`: source tracing is incomplete. + +Record evidence in the audit report. The script does not validate or inherit dispositions. + +### 4. Fix runtime boundaries + +Before changing runtime behavior, use `$implementation-strategy`. + +- Check the relevant `_debug.DONT_LOG_MODEL_DATA` and `_debug.DONT_LOG_TOOL_DATA` flags before formatting or inspecting sensitive values. +- Redact mixed model/tool values when either flag disables data logging. +- In redacted mode, emit a fixed message and omit sensitive `args`, `extra`, and `exc_info`. +- Build diagnostic-only context lazily so redacted mode never reads it. +- Preserve useful diagnostics when sensitive-data logging is explicitly enabled. +- Keep logging failure from changing fallback, cleanup, event, rejection, or cancellation behavior. +- For MCP URLs, remove credentials, query parameters, and fragments in diagnostic mode; never use sanitized names as a substitute for fixed redacted messages. + +### 5. Prove caller behavior + +Add tests at every changed caller boundary. Inspect the complete `LogRecord`, not only rendered text. Test both redacted policies, diagnostic mode, hostile objects, exception chains, and the caller's observable fallback or cleanup behavior as applicable. + +### 6. Re-run and close out + +Re-run the collector, the manual searches, focused tests, and applicable repository gates. Use `$code-change-verification` for runtime or test changes and `$pr-draft-summary` when required. + +Report candidate counts as search coverage only. Lead with confirmed leaks fixed, retained intentional output, reviewed uncertainty, and verification results. Never report a clean collector result as proof that no sensitive logging path exists. diff --git a/.agents/skills/sensitive-logging-audit/agents/openai.yaml b/.agents/skills/sensitive-logging-audit/agents/openai.yaml new file mode 100644 index 0000000000..1f0e17b1bd --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Sensitive Logging Audit" + short_description: "Audit and fix sensitive Python logging paths" + default_prompt: "Use $sensitive-logging-audit to inventory, verify, and fix sensitive logging leaks in this repository." diff --git a/.agents/skills/sensitive-logging-audit/references/redaction-validation.md b/.agents/skills/sensitive-logging-audit/references/redaction-validation.md new file mode 100644 index 0000000000..35b2ab6cba --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/references/redaction-validation.md @@ -0,0 +1,63 @@ +# Python sensitive logging validation + +The collector reports syntactic logging and raw-output candidates. It does not resolve aliases, prove receiver types, evaluate guards, classify payloads, or support a completeness claim. Review candidates together with direct source searches and runtime tests. + +## Required validation matrix + +Test every changed sensitive caller boundary in both redacted and diagnostic modes. Use a unique sentinel for each source and inspect both rendered output and the complete `LogRecord`. + +| Case | Model flag | Tool flag | Value | Required assertion | +| --- | --- | --- | --- | --- | +| Model redaction | on | off | `Exception(secret)` | No sentinel or exception object remains in the record | +| Tool redaction | off | on | `Exception(secret)` | No sentinel or exception object remains in the record | +| Both redacted | on | on | model and tool values | Neither sentinel remains anywhere in the record | +| Diagnostic mode | off | off | ordinary exception | Existing diagnostic detail and traceback behavior remain | +| Hostile string | applicable | applicable | object whose `__str__` raises or returns a secret | Logging does not fail or reveal the secret | +| Hostile repr | applicable | applicable | object whose `__repr__` raises or returns a secret | Logging does not fail or reveal the secret | +| Hostile class access | applicable | applicable | exception overriding `__getattribute__` | Redacted logging does not inspect the exception | +| Exception chain | applicable | applicable | `__cause__`, `__context__`, notes, or `ExceptionGroup` containing secrets | No chained secret is attached or rendered | +| Supplemental arguments | applicable | applicable | fixed message plus secret formatting argument | Formatting arguments are omitted in redacted mode | +| Extra payload | applicable | applicable | `extra={"detail": secret}` | Secret `LogRecord` attributes are omitted | +| Traceback payload | applicable | applicable | `exc_info=True` or an exception tuple | `exc_info` and `exc_text` are absent in redacted mode | +| MCP server or tool name | tool | on | path token or custom-name sentinel | Log uses a fixed message and does not read or attach the name | +| URL-derived MCP name | tool | off | URL credentials, query, and fragment | Log retains only scheme, host, port, and path; the runtime value is unchanged | + +Also test the observable caller behavior after logging. Redaction is incorrect if it prevents a fallback result, cleanup, event emission, rejection, or cancellation from completing. + +## Inspect the full LogRecord + +Do not assert only against `caplog.text` or a mock call converted to a string. In redacted mode, inspect at least: + +- `record.msg` +- `record.args` +- `record.exc_info` +- `record.exc_text` +- values added through `record.__dict__` +- the final output of a real `logging.Formatter` + +The sensitive object itself must not remain attached even when its string representation is absent. A custom handler or exporter may inspect raw record fields. + +## Review procedure + +1. Run the collector against all of `src/agents`. +2. Run the supplemental `rg` searches from `SKILL.md` and inspect aliases and dynamic dispatch. +3. Review raw output and ambiguous receivers first. +4. Review caught values, `logger.exception`, `exc_info`, `extra`, and formatting arguments. +5. Trace model, tool, Realtime, MCP, session, sandbox, voice, tracing, and cleanup values to their producers. +6. Classify intentional output separately from diagnostics; do not silently exempt `print` or warnings. +7. Add focused tests at every changed caller boundary. +8. Re-run the collector and source searches after the fix. + +An empty or unchanged collector report is not proof of safety. Assignment aliases, monkey-patched methods, dynamically installed handlers, non-constant reflection, and arbitrary runtime data flow require manual inspection. + +## Audit report expectations + +For each confirmed or uncertain path, record: + +- The source location and value producer. +- The manual disposition: `model`, `tool`, `model+tool`, `operational`, `intentional-output`, or `uncertain`. +- Concrete evidence for the disposition. +- The fix or reason for retaining the path. +- The caller-level regression test, when behavior changed. + +Do not reuse a disposition solely because a fingerprint or call text is unchanged. diff --git a/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py b/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py new file mode 100644 index 0000000000..da2bea1f20 --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import re +import sys +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +LOG_METHODS = { + "critical", + "debug", + "error", + "exception", + "fatal", + "info", + "log", + "warn", + "warning", +} +POLICY_HELPERS = { + "log_model_action_debug", + "log_model_action_error", + "log_model_action_warning", + "log_model_and_tool_action_debug", + "log_model_and_tool_action_error", + "log_model_and_tool_action_warning", + "log_tool_action_debug", + "log_tool_action_error", + "log_tool_action_warning", +} +RAW_OUTPUT_METHODS = { + "pp", + "pprint", + "print", + "print_exc", + "print_exception", + "warn", + "warn_explicit", + "write", + "writelines", +} +CALLBACK_KEYWORDS = {"callback", "handler"} + + +@dataclass(frozen=True) +class Candidate: + fingerprint: str + file: str + line: int + column: int + kind: str + method: str + context: str + call: str + reason: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def normalize_path(path: str | Path) -> str: + return str(path).replace("\\", "/") + + +def collect_source_files(roots: Sequence[str | Path]) -> list[Path]: + files: set[Path] = set() + for root_value in roots: + root = Path(root_value).resolve() + if root.is_file(): + if root.suffix == ".py": + files.add(root) + continue + if not root.is_dir(): + raise FileNotFoundError(f"Inventory root does not exist: {root_value}") + for path in root.rglob("*.py"): + relative_parts = path.relative_to(root).parts + if any(part.startswith(".") or part == "__pycache__" for part in relative_parts): + continue + files.add(path.resolve()) + return sorted(files) + + +def normalize_node(node: ast.AST, source: str) -> str: + segment = ast.get_source_segment(source, node) + if segment is None: + segment = ast.dump(node, annotate_fields=True, include_attributes=False) + return re.sub(r"\s+", " ", segment).strip() + + +def dotted_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + receiver = dotted_name(node.value) + return f"{receiver}.{node.attr}" if receiver else node.attr + return None + + +def terminal_name(node: ast.AST) -> str | None: + name = dotted_name(node) + return name.rsplit(".", 1)[-1] if name else None + + +def make_parent_map(tree: ast.AST) -> dict[ast.AST, ast.AST]: + return {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)} + + +def scope_context(node: ast.AST, parents: Mapping[ast.AST, ast.AST]) -> str: + parts: list[str] = [] + current = parents.get(node) + while current is not None: + if isinstance(current, ast.ClassDef): + parts.append(f"class:{current.name}") + elif isinstance(current, ast.FunctionDef | ast.AsyncFunctionDef): + parts.append(f"function:{current.name}") + elif isinstance(current, ast.Lambda): + parts.append("lambda") + current = parents.get(current) + return ">".join(reversed(parts)) or "" + + +def callback_arguments(call: ast.Call) -> Iterable[tuple[ast.AST, str | None]]: + yield from ((argument, None) for argument in call.args) + yield from ( + (keyword.value, keyword.arg) for keyword in call.keywords if keyword.arg is not None + ) + + +def looks_like_callback(node: ast.AST, keyword: str | None) -> bool: + method = terminal_name(node) + if method not in LOG_METHODS: + return False + if keyword is not None and ( + keyword.startswith("on_") + or keyword.endswith(("_callback", "_handler")) + or keyword in CALLBACK_KEYWORDS + ): + return True + if not isinstance(node, ast.Attribute): + return False + receiver = dotted_name(node.value) + receiver_name = receiver.rsplit(".", 1)[-1].lower() if receiver else "" + return receiver_name in {"log", "logger"} or receiver_name.endswith(("_log", "_logger")) + + +def selected_getattr_method(call: ast.Call) -> str | None: + if terminal_name(call.func) != "getattr" or len(call.args) < 2: + return None + attribute = call.args[1] + if not isinstance(attribute, ast.Constant) or not isinstance(attribute.value, str): + return None + if attribute.value in LOG_METHODS | RAW_OUTPUT_METHODS: + return attribute.value + return None + + +def classify_call(call: ast.Call) -> tuple[str, str, str] | None: + qualified_method = dotted_name(call.func) + method = terminal_name(call.func) + if method in POLICY_HELPERS: + return ( + "policy-helper-call", + method, + "Known redaction helper; review the caller's data classification and fixed message.", + ) + if method in LOG_METHODS and not ( + method == "warn" and qualified_method in {"warn", "warnings.warn"} + ): + return ( + "logging-call-candidate", + method, + "Logging-like method name; inspect the receiver and every attached value.", + ) + if method in RAW_OUTPUT_METHODS: + return ( + "raw-output-call-candidate", + method, + "Direct-output method name; verify its destination and whether values " + "can be sensitive.", + ) + selected = selected_getattr_method(call) + if selected is not None: + return ( + "getattr-sink-candidate", + selected, + "Constant getattr selects an output-like method; trace the receiver and later uses.", + ) + return None + + +def inventory_source(source: str, file_path: str = "fixture.py") -> list[Candidate]: + normalized_path = normalize_path(file_path) + tree = ast.parse(source, filename=normalized_path) + parents = make_parent_map(tree) + candidates: list[Candidate] = [] + recorded: set[tuple[int, str, str]] = set() + + def record(node: ast.AST, kind: str, method: str, call: str, reason: str) -> None: + key = (id(node), kind, method) + if key in recorded: + return + recorded.add(key) + line = getattr(node, "lineno", 1) + column = getattr(node, "col_offset", 0) + 1 + context = scope_context(node, parents) + fingerprint = hashlib.sha256( + f"{normalized_path}\0{line}\0{column}\0{kind}\0{method}\0{call}".encode() + ).hexdigest()[:12] + candidates.append( + Candidate( + fingerprint=fingerprint, + file=normalized_path, + line=line, + column=column, + kind=kind, + method=method, + context=context, + call=call, + reason=reason, + ) + ) + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + classification = classify_call(node) + if classification is not None: + kind, method, reason = classification + record(node, kind, method, normalize_node(node, source), reason) + for argument, keyword in callback_arguments(node): + if not looks_like_callback(argument, keyword): + continue + method = terminal_name(argument) + if method is None: + continue + record( + argument, + "logging-callback-candidate", + method, + normalize_node(argument, source), + "Logging-like callable passed to a callback-shaped argument; inspect " + "registration and payloads.", + ) + + candidates.sort(key=lambda item: (item.file, item.line, item.column, item.kind, item.method)) + return candidates + + +def summarize(candidates: Sequence[Candidate]) -> dict[str, int]: + kinds = Counter(candidate.kind for candidate in candidates) + return { + "totalCandidates": len(candidates), + "loggingCalls": kinds["logging-call-candidate"], + "rawOutputCalls": kinds["raw-output-call-candidate"], + "policyHelperCalls": kinds["policy-helper-call"], + "getattrSelections": kinds["getattr-sink-candidate"], + "callbackReferences": kinds["logging-callback-candidate"], + } + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Collect syntactic Python logging and raw-output candidates for manual review." + ) + ) + parser.add_argument("roots", nargs="*", default=["src/agents"]) + parser.add_argument("--format", choices=("json", "markdown"), default="markdown") + parser.add_argument("--summary-only", action="store_true") + parser.add_argument("--output", type=Path) + return parser.parse_args(argv) + + +def build_report(args: argparse.Namespace) -> dict[str, Any]: + cwd = Path.cwd().resolve() + candidates: list[Candidate] = [] + for path in collect_source_files(args.roots): + try: + display_path = path.relative_to(cwd) + except ValueError: + display_path = path + source = path.read_text(encoding="utf-8") + try: + candidates.extend(inventory_source(source, str(display_path))) + except SyntaxError as error: + raise SyntaxError( + f"Failed to parse {display_path}:{error.lineno}: {error.msg}" + ) from error + + report: dict[str, Any] = { + "contract": ( + "Syntactic candidates only. Manual review and runtime tests are required; " + "absence from this report is not proof of safety." + ), + "summary": summarize(candidates), + } + if not args.summary_only: + report["candidates"] = [candidate.to_dict() for candidate in candidates] + return report + + +def render_markdown(report: Mapping[str, Any], summary_only: bool) -> str: + summary = report["summary"] + lines = [ + "# Sensitive logging candidates", + "", + f"> {report['contract']}", + "", + f"- Total candidates: {summary['totalCandidates']}", + f"- Logging calls: {summary['loggingCalls']}", + f"- Raw-output calls: {summary['rawOutputCalls']}", + f"- Policy-helper calls: {summary['policyHelperCalls']}", + f"- Constant getattr selections: {summary['getattrSelections']}", + f"- Callback references: {summary['callbackReferences']}", + ] + if not summary_only: + lines.extend( + [ + "", + "| Location | Kind | Method | Context | Fingerprint |", + "| --- | --- | --- | --- | --- |", + ] + ) + for candidate in report.get("candidates", []): + location = f"{candidate['file']}:{candidate['line']}" + lines.append( + f"| {location} | {candidate['kind']} | {candidate['method']} | " + f"{candidate['context']} | {candidate['fingerprint']} |" + ) + return "\n".join(lines) + "\n" + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + try: + report = build_report(args) + output = ( + json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.format == "json" + else render_markdown(report, args.summary_only) + ) + if args.output: + args.output.write_text(output, encoding="utf-8") + else: + sys.stdout.write(output) + return 0 + except (OSError, SyntaxError, ValueError, json.JSONDecodeError) as error: + print(f"Sensitive logging candidate collection failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py new file mode 100644 index 0000000000..44a6b42a1a --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from inventory_logging import collect_source_files, inventory_source, summarize + + +class InventoryTests(unittest.TestCase): + def test_collects_direct_logging_calls_without_certifying_receivers(self) -> None: + candidates = inventory_source( + """ +from logging import error + +logger.debug("ready") +logger.error("failed: %s", secret) +error(secret) +task.exception() +""" + ) + + self.assertEqual( + [(item.kind, item.method) for item in candidates], + [ + ("logging-call-candidate", "debug"), + ("logging-call-candidate", "error"), + ("logging-call-candidate", "error"), + ("logging-call-candidate", "exception"), + ], + ) + + def test_collects_policy_helpers_without_claiming_their_callers_are_safe(self) -> None: + candidates = inventory_source( + """ +from agents.logger import log_model_action_error + +log_model_action_error(logger, "failed", error) +agents.logger.log_model_and_tool_action_warning(logger, "failed", error) +""" + ) + + self.assertEqual( + [(item.kind, item.method) for item in candidates], + [ + ("policy-helper-call", "log_model_action_error"), + ("policy-helper-call", "log_model_and_tool_action_warning"), + ], + ) + + def test_collects_raw_output_method_names(self) -> None: + candidates = inventory_source( + """ +import os +import pprint +import sys +import traceback +import warnings + +print(secret) +pprint.pp(secret) +warnings.warn(secret) +sys.stderr.buffer.write(secret_bytes) +sys.stdout.writelines([secret]) +traceback.print_exception(error) +os.write(2, secret_bytes) +""" + ) + + self.assertEqual( + [(item.kind, item.method) for item in candidates], + [ + ("raw-output-call-candidate", "print"), + ("raw-output-call-candidate", "pp"), + ("raw-output-call-candidate", "warn"), + ("raw-output-call-candidate", "write"), + ("raw-output-call-candidate", "writelines"), + ("raw-output-call-candidate", "print_exception"), + ("raw-output-call-candidate", "write"), + ], + ) + + def test_collects_constant_getattr_sink_selections(self) -> None: + candidates = inventory_source( + """ +emit = getattr(logger, "error") +writer = builtins.getattr(stream, "write") +ignored = getattr(logger, method_name) +""" + ) + + self.assertEqual( + [(item.kind, item.method) for item in candidates], + [ + ("getattr-sink-candidate", "error"), + ("getattr-sink-candidate", "write"), + ], + ) + + def test_collects_obvious_logging_callbacks(self) -> None: + candidates = inventory_source( + """ +register(log.warning) +register(on_error=service.error) +register(result=request.error) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.call) for item in candidates], + [ + ("logging-callback-candidate", "warning", "log.warning"), + ("logging-callback-candidate", "error", "service.error"), + ], + ) + + def test_keeps_the_output_schema_free_of_security_certification(self) -> None: + candidate = inventory_source('logger.error("failed", secret)')[0].to_dict() + + self.assertEqual( + set(candidate), + { + "fingerprint", + "file", + "line", + "column", + "kind", + "method", + "context", + "call", + "reason", + }, + ) + self.assertNotIn("policy", candidate) + self.assertNotIn("safe", candidate) + + def test_reports_enclosing_scope_as_review_context(self) -> None: + candidate = inventory_source( + """ +class Worker: + def report(self): + logger.error(secret) +""" + )[0] + + self.assertEqual(candidate.context, "class:Worker>function:report") + + def test_does_not_claim_to_follow_assignment_aliases(self) -> None: + candidates = inventory_source( + """ +emit = logger.error +emit(secret) +""" + ) + + self.assertEqual(candidates, []) + + def test_summary_counts_only_syntactic_candidate_categories(self) -> None: + candidates = inventory_source( + """ +logger.error(secret) +print(secret) +log_tool_action_error(logger, "failed", error) +register(on_error=service.error) +getattr(logger, "warning") +""" + ) + + self.assertEqual( + summarize(candidates), + { + "totalCandidates": 5, + "loggingCalls": 1, + "rawOutputCalls": 1, + "policyHelperCalls": 1, + "getattrSelections": 1, + "callbackReferences": 1, + }, + ) + + def test_collect_source_files_filters_hidden_children_relative_to_root(self) -> None: + with TemporaryDirectory(prefix=".hidden-parent-") as directory: + root = Path(directory) / "scan" + root.mkdir() + visible = root / "visible.py" + visible.write_text("print('visible')\n") + hidden = root / ".cache" + hidden.mkdir() + (hidden / "hidden.py").write_text("print('hidden')\n") + + self.assertEqual(collect_source_files([root]), [visible.resolve()]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/run_integration_tests.py b/.github/scripts/run_integration_tests.py new file mode 100644 index 0000000000..28aa259d71 --- /dev/null +++ b/.github/scripts/run_integration_tests.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +WORKSPACE = ROOT / ".tmp" / "integration-tests" +DIST = WORKSPACE / "dist" +TESTS = ROOT / "integration_tests" +EXTRAS = "any-llm,litellm,realtime,voice" +OPTIONAL_EXTRAS = ( + "any-llm", + "litellm", + "realtime", + "voice", + "sqlalchemy", + "encrypt", + "redis", + "viz", + "s3", +) +PROFILES = ( + "packaging", + "core", + "providers", + "realtime", + "voice", + "hosted", + "extras", + "full", + "release", + "nightly", + "manual", +) + + +def run(command: list[str], *, env: dict[str, str] | None = None) -> None: + print(f"[integration] {' '.join(command)}", flush=True) + subprocess.run(command, cwd=ROOT, env=env, check=True) + + +def build_distributions() -> tuple[Path, Path]: + DIST.mkdir(parents=True, exist_ok=True) + run(["uv", "build", "--out-dir", str(DIST)]) + wheels = sorted(DIST.glob("openai_agents-*.whl"), key=lambda path: path.stat().st_mtime) + sdists = sorted(DIST.glob("openai_agents-*.tar.gz"), key=lambda path: path.stat().st_mtime) + if not wheels or not sdists: + raise RuntimeError("uv build did not produce both an openai-agents wheel and sdist.") + return wheels[-1], sdists[-1] + + +def _any_llm_provider_extras( + *, external_providers_enabled: bool, direct_providers_enabled: bool +) -> list[str]: + provider_extras: set[str] = set() + configured_models = os.environ.get("OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", "") + for model in configured_models.split(","): + provider = model.strip().partition("/")[0] + if provider in {"anthropic", "openrouter"}: + provider_extras.add(provider) + elif provider in {"gemini", "google"}: + provider_extras.add("gemini") + + if external_providers_enabled: + if direct_providers_enabled and os.environ.get("ANTHROPIC_API_KEY"): + provider_extras.add("anthropic") + if direct_providers_enabled and ( + os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY") + ): + provider_extras.add("gemini") + if os.environ.get("OPENROUTER_API_KEY"): + provider_extras.add("openrouter") + + return sorted(provider_extras) + + +def create_environment( + name: str, distribution: Path, *, extras: bool = False, optional_extra: str | None = None +) -> Path: + environment = WORKSPACE / name + venv_command = ["uv", "venv", "--clear", str(environment)] + if python_version := os.environ.get("OPENAI_AGENTS_INTEGRATION_PYTHON"): + venv_command.extend(["--python", python_version]) + run(venv_command) + python = environment / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python") + selected_extra = EXTRAS if extras else optional_extra + requirement = f"{distribution}[{selected_extra}]" if selected_extra else str(distribution) + requirements = [requirement, "pytest", "pytest-asyncio", "pytest-timeout"] + external_providers_enabled = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "" + ).lower() in {"1", "true", "yes"} + direct_providers_enabled = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", "" + ).lower() in {"1", "true", "yes"} + if extras: + any_llm_extras = _any_llm_provider_extras( + external_providers_enabled=external_providers_enabled, + direct_providers_enabled=direct_providers_enabled, + ) + if any_llm_extras: + requirements.append(f"any-llm-sdk[{','.join(any_llm_extras)}]") + proxy_values = [ + os.environ.get(name, "") + for name in ( + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "all_proxy", + "http_proxy", + "https_proxy", + ) + ] + if any(value.lower().startswith("socks") for value in proxy_values): + requirements.append("httpx[socks]") + run(["uv", "pip", "install", "--python", str(python), *requirements]) + return python + + +def run_suite( + python: Path, + wheel: Path, + sdist: Path, + *, + selection: str, + environment_kind: str, +) -> None: + child_env = dict(os.environ) + child_env.pop("PYTHONPATH", None) + if child_env.get("OPENAI_AGENTS_INTEGRATION_DISABLE_PROXY", "").lower() in { + "1", + "true", + "yes", + }: + for variable in ( + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "all_proxy", + "http_proxy", + "https_proxy", + ): + child_env.pop(variable, None) + child_env["PYTHONNOUSERSITE"] = "1" + child_env["OPENAI_AGENTS_INTEGRATION_WHEEL"] = str(wheel) + child_env["OPENAI_AGENTS_INTEGRATION_SDIST"] = str(sdist) + child_env["OPENAI_AGENTS_INTEGRATION_ENVIRONMENT"] = environment_kind + if environment_kind.startswith("extra-"): + child_env["OPENAI_AGENTS_INTEGRATION_EXTRA"] = environment_kind.removeprefix("extra-") + if not os.environ.get("OPENAI_AGENTS_INTEGRATION_ENABLE_TRACING"): + child_env["OPENAI_AGENTS_DISABLE_TRACING"] = "1" + command = [ + str(python), + "-I", + "-m", + "pytest", + "-c", + str(TESTS / "pytest.ini"), + str(TESTS), + "-v", + "--tb=short", + "-m", + selection, + ] + run(command, env=child_env) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run packaged openai-agents integration tests.") + parser.add_argument("--profile", choices=PROFILES, default="full") + parser.add_argument( + "--all", + action="store_true", + help="Include configured direct Anthropic and Gemini providers alongside OpenRouter.", + ) + args = parser.parse_args() + if args.all: + os.environ["OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS"] = "1" + os.environ["OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS"] = "1" + wheel, sdist = build_distributions() + print(f"[integration] wheel={wheel.name} sdist={sdist.name} profile={args.profile}") + + if args.profile in {"packaging", "core", "hosted", "full", "release", "nightly", "manual"}: + python = create_environment("core", wheel) + selections = { + "packaging": "packaging", + "core": "packaging or core", + "hosted": "packaging or hosted", + "full": "packaging or ((core or hosted) and not nightly and not manual)", + "release": "packaging or ((core or hosted) and not nightly and not manual)", + "nightly": "packaging or ((core or hosted) and not manual)", + "manual": "packaging or core or hosted", + } + run_suite( + python, + wheel, + sdist, + selection=selections[args.profile], + environment_kind="core", + ) + + if args.profile in {"providers", "realtime", "voice", "full", "release", "nightly", "manual"}: + python = create_environment("extended", wheel, extras=True) + if args.profile in {"full", "release"}: + selection = "(providers or realtime or voice) and not nightly and not manual" + elif args.profile == "nightly": + selection = "(providers or realtime or voice) and not manual" + elif args.profile == "manual": + selection = "providers or realtime or voice" + else: + selection = args.profile + run_suite( + python, + wheel, + sdist, + selection=selection, + environment_kind="extended", + ) + + if args.profile in {"packaging", "full", "release", "nightly", "manual"}: + python = create_environment("sdist", sdist) + run_suite(python, wheel, sdist, selection="packaging", environment_kind="sdist") + + if args.profile in {"extras", "full", "release", "nightly", "manual"}: + for optional_extra in OPTIONAL_EXTRAS: + environment_kind = f"extra-{optional_extra}" + python = create_environment(environment_kind, wheel, optional_extra=optional_extra) + run_suite(python, wheel, sdist, selection="extras", environment_kind=environment_kind) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/update_rclone_pin.py b/.github/scripts/update_rclone_pin.py new file mode 100644 index 0000000000..d4cd0f29a3 --- /dev/null +++ b/.github/scripts/update_rclone_pin.py @@ -0,0 +1,420 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import cast +from urllib import error, parse, request + +_RCLONE_RELEASES_API = "https://api.github.com/repos/rclone/rclone/releases" +_DEFAULT_COOLDOWN_DAYS = 7 +_RUNTIME_PIN_PATH = Path("src/agents/extensions/sandbox/_rclone.py") +_DOCKER_PIN_PATH = Path("examples/sandbox/docker/Dockerfile.mount") +_PYTHON_PIN_BEGIN = "# BEGIN RCLONE RELEASE PIN" +_PYTHON_PIN_END = "# END RCLONE RELEASE PIN" +_DOCKER_PIN_BEGIN = "# BEGIN RCLONE RELEASE PIN" +_DOCKER_PIN_END = "# END RCLONE RELEASE PIN" +_RCLONE_ARCHES = ("386", "amd64", "arm", "arm-v6", "arm-v7", "arm64") +_DOCKER_ARCHES = ("amd64", "arm64") +_SHA256_LINE = re.compile(r"^([0-9a-fA-F]{64})\s+\*?(\S+)$") + + +@dataclass(frozen=True) +class RclonePin: + version: str + sha256_by_arch: dict[str, str] + + +def _headers(url: str) -> dict[str, str]: + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "openai-agents-python-rclone-pin-updater", + } + token = os.environ.get("GITHUB_TOKEN") + if token and parse.urlparse(url).hostname == "api.github.com": + headers["Authorization"] = f"Bearer {token}" + return headers + + +def _fetch_bytes(url: str) -> bytes: + req = request.Request(url, headers=_headers(url)) + try: + with request.urlopen(req, timeout=30) as response: + return response.read() + except error.HTTPError as exc: + raise RuntimeError(f"failed to fetch {url}: HTTP {exc.code}") from exc + except error.URLError as exc: + raise RuntimeError(f"failed to fetch {url}: {exc.reason}") from exc + + +def _fetch_json_object(url: str) -> dict[str, object]: + payload = json.loads(_fetch_bytes(url)) + if not isinstance(payload, dict): + raise RuntimeError(f"expected a JSON object from {url}") + return cast(dict[str, object], payload) + + +def _fetch_json_array(url: str) -> list[dict[str, object]]: + payload = json.loads(_fetch_bytes(url)) + if not isinstance(payload, list) or not all(isinstance(item, dict) for item in payload): + raise RuntimeError(f"expected a JSON array of objects from {url}") + return cast(list[dict[str, object]], payload) + + +def _normalized_version(value: str) -> str: + version = value.removeprefix("v") + if re.fullmatch(r"\d+\.\d+\.\d+", version) is None: + raise ValueError(f"invalid rclone version: {value}") + return version + + +def _release_url(version: str | None) -> str: + if version is None: + return f"{_RCLONE_RELEASES_API}?per_page=100" + tag = parse.quote(f"v{_normalized_version(version)}", safe="") + return f"{_RCLONE_RELEASES_API}/tags/{tag}" + + +def _release_version(release: dict[str, object]) -> str: + tag_name = release.get("tag_name") + if not isinstance(tag_name, str): + raise RuntimeError("rclone release metadata is missing tag_name") + return _normalized_version(tag_name) + + +def _release_published_at(release: dict[str, object]) -> datetime: + value = release.get("published_at") + if not isinstance(value, str): + raise RuntimeError("rclone release metadata is missing published_at") + try: + published_at = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise RuntimeError(f"rclone release has invalid published_at: {value}") from exc + if published_at.tzinfo is None: + raise RuntimeError(f"rclone release published_at has no timezone: {value}") + return published_at.astimezone(timezone.utc) + + +def _asset_observed_at(release: dict[str, object], asset_name: str) -> datetime: + asset = _asset(release, asset_name) + timestamps: list[datetime] = [] + for field in ("created_at", "updated_at"): + value = asset.get(field) + if not isinstance(value, str): + raise RuntimeError(f"rclone release asset {asset_name} is missing {field}") + try: + timestamp = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise RuntimeError( + f"rclone release asset {asset_name} has invalid {field}: {value}" + ) from exc + if timestamp.tzinfo is None: + raise RuntimeError( + f"rclone release asset {asset_name} {field} has no timezone: {value}" + ) + timestamps.append(timestamp.astimezone(timezone.utc)) + return max(timestamps) + + +def _required_asset_names(version: str) -> tuple[str, ...]: + archives = tuple(f"rclone-v{version}-linux-{arch}.zip" for arch in _RCLONE_ARCHES) + return ("SHA256SUMS", *archives) + + +def _validate_cooldown( + subject: str, + observed_at: datetime, + *, + cooldown_days: int, + now: datetime, +) -> None: + eligible_at = observed_at + timedelta(days=cooldown_days) + if eligible_at > now: + raise RuntimeError( + f"{subject} is still in its {cooldown_days}-day cooldown " + f"(eligible at {eligible_at.isoformat()})" + ) + + +def _validate_stable_release( + release: dict[str, object], + *, + cooldown_days: int, + now: datetime, +) -> None: + version = _release_version(release) + if release.get("draft") is not False or release.get("prerelease") is not False: + raise RuntimeError(f"rclone v{version} is not a stable published release") + _validate_cooldown( + f"rclone v{version}", + _release_published_at(release), + cooldown_days=cooldown_days, + now=now, + ) + for asset_name in _required_asset_names(version): + _validate_cooldown( + f"rclone v{version} asset {asset_name}", + _asset_observed_at(release, asset_name), + cooldown_days=cooldown_days, + now=now, + ) + + +def _latest_stable_release( + releases: list[dict[str, object]], + *, + cooldown_days: int, + now: datetime, +) -> dict[str, object]: + eligible: list[tuple[datetime, dict[str, object]]] = [] + for release in releases: + try: + _validate_stable_release(release, cooldown_days=cooldown_days, now=now) + except (RuntimeError, ValueError): + continue + eligible.append((_release_published_at(release), release)) + if not eligible: + raise RuntimeError( + f"no stable rclone release has completed the {cooldown_days}-day cooldown" + ) + return max(eligible, key=lambda item: item[0])[1] + + +def _asset(release: dict[str, object], asset_name: str) -> dict[str, object]: + assets = release.get("assets") + if not isinstance(assets, list): + raise RuntimeError("rclone release metadata is missing assets") + for asset in assets: + if isinstance(asset, dict) and asset.get("name") == asset_name: + return cast(dict[str, object], asset) + raise RuntimeError(f"rclone release is missing {asset_name}") + + +def _asset_url(release: dict[str, object], asset_name: str) -> str: + url = _asset(release, asset_name).get("browser_download_url") + if not isinstance(url, str): + raise RuntimeError(f"rclone release asset {asset_name} is missing its download URL") + return url + + +def _asset_sha256(release: dict[str, object], asset_name: str) -> str: + digest = _asset(release, asset_name).get("digest") + if not isinstance(digest, str) or re.fullmatch(r"sha256:[0-9a-fA-F]{64}", digest) is None: + raise RuntimeError(f"rclone release asset {asset_name} is missing its SHA256 digest") + return digest.removeprefix("sha256:").lower() + + +def _validate_download_sha256( + release: dict[str, object], + asset_name: str, + content: bytes, +) -> None: + expected = _asset_sha256(release, asset_name) + actual = hashlib.sha256(content).hexdigest() + if actual != expected: + raise RuntimeError( + f"downloaded rclone release asset {asset_name} does not match GitHub's digest" + ) + + +def _parse_sha256s(text: str, version: str) -> dict[str, str]: + filenames = {f"rclone-v{version}-linux-{arch}.zip": arch for arch in _RCLONE_ARCHES} + sha256_by_arch: dict[str, str] = {} + for line in text.splitlines(): + match = _SHA256_LINE.fullmatch(line.strip()) + if match is None: + continue + digest, filename = match.groups() + arch = filenames.get(filename) + if arch is not None: + sha256_by_arch[arch] = digest.lower() + + missing = [arch for arch in _RCLONE_ARCHES if arch not in sha256_by_arch] + if missing: + raise RuntimeError( + f"rclone v{version} SHA256SUMS is missing Linux archives for: {', '.join(missing)}" + ) + return sha256_by_arch + + +def _validate_asset_sha256s( + release: dict[str, object], + version: str, + sha256_by_arch: dict[str, str], +) -> None: + for arch in _RCLONE_ARCHES: + asset_name = f"rclone-v{version}-linux-{arch}.zip" + asset_sha256 = _asset_sha256(release, asset_name) + if asset_sha256 != sha256_by_arch[arch]: + raise RuntimeError( + f"rclone v{version} SHA256SUMS does not match GitHub's digest for {asset_name}" + ) + + +def fetch_pin( + version: str | None = None, + *, + cooldown_days: int = _DEFAULT_COOLDOWN_DAYS, + now: datetime | None = None, +) -> RclonePin: + if cooldown_days < 0: + raise ValueError("cooldown days must be zero or greater") + current_time = now or datetime.now(timezone.utc) + if current_time.tzinfo is None: + raise ValueError("current time must include a timezone") + current_time = current_time.astimezone(timezone.utc) + + if version is None: + releases = _fetch_json_array(_release_url(None)) + release = _latest_stable_release( + releases, + cooldown_days=cooldown_days, + now=current_time, + ) + else: + release = _fetch_json_object(_release_url(version)) + _validate_stable_release( + release, + cooldown_days=cooldown_days, + now=current_time, + ) + resolved_version = _release_version(release) + if version is not None and resolved_version != _normalized_version(version): + raise RuntimeError( + f"requested rclone v{_normalized_version(version)}, got v{resolved_version}" + ) + checksums_url = _asset_url(release, "SHA256SUMS") + checksums_content = _fetch_bytes(checksums_url) + _validate_download_sha256(release, "SHA256SUMS", checksums_content) + checksums = checksums_content.decode("utf-8") + sha256_by_arch = _parse_sha256s(checksums, resolved_version) + _validate_asset_sha256s(release, resolved_version, sha256_by_arch) + return RclonePin( + version=resolved_version, + sha256_by_arch=sha256_by_arch, + ) + + +def _python_pin_block(pin: RclonePin) -> str: + lines = [ + _PYTHON_PIN_BEGIN, + f'_RCLONE_VERSION = "{pin.version}"', + "_RCLONE_SHA256_BY_ARCH = {", + ] + for arch in _RCLONE_ARCHES: + lines.append(f' "{arch}": "{pin.sha256_by_arch[arch]}",') + lines.extend(["}", _PYTHON_PIN_END]) + return "\n".join(lines) + + +def _docker_pin_block(pin: RclonePin) -> str: + lines = [_DOCKER_PIN_BEGIN, f"ARG RCLONE_VERSION={pin.version}"] + for arch in _DOCKER_ARCHES: + variable_arch = arch.upper().replace("-", "_") + lines.append(f"ARG RCLONE_SHA256_LINUX_{variable_arch}={pin.sha256_by_arch[arch]}") + lines.append(_DOCKER_PIN_END) + return "\n".join(lines) + + +def _replace_marked_block(text: str, begin: str, end: str, replacement: str) -> str: + if text.count(begin) != 1 or text.count(end) != 1: + raise RuntimeError(f"expected exactly one pin block delimited by {begin!r} and {end!r}") + start = text.index(begin) + finish = text.index(end, start) + len(end) + return f"{text[:start]}{replacement}{text[finish:]}" + + +def apply_pin(repo_root: Path, pin: RclonePin, *, check: bool) -> list[Path]: + updates = ( + ( + repo_root / _RUNTIME_PIN_PATH, + _PYTHON_PIN_BEGIN, + _PYTHON_PIN_END, + _python_pin_block(pin), + ), + ( + repo_root / _DOCKER_PIN_PATH, + _DOCKER_PIN_BEGIN, + _DOCKER_PIN_END, + _docker_pin_block(pin), + ), + ) + changed: list[Path] = [] + for path, begin, end, replacement in updates: + current = path.read_text() + updated = _replace_marked_block(current, begin, end, replacement) + if updated == current: + continue + changed.append(path) + if not check: + path.write_text(updated) + return changed + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Update the verified rclone release pinned by sandbox installers." + ) + parser.add_argument( + "--version", + help="rclone version to pin, with or without a leading v (default: latest stable)", + ) + parser.add_argument( + "--check", + action="store_true", + help="report stale pin blocks without modifying files", + ) + parser.add_argument( + "--cooldown-days", + type=int, + default=_DEFAULT_COOLDOWN_DAYS, + help=( + "minimum age of a stable release before it is eligible " + f"(default: {_DEFAULT_COOLDOWN_DAYS})" + ), + ) + parser.add_argument( + "--repo-root", + type=Path, + default=Path(__file__).resolve().parents[2], + help=argparse.SUPPRESS, + ) + args = parser.parse_args() + + try: + pin = fetch_pin(args.version, cooldown_days=args.cooldown_days) + changed = apply_pin(args.repo_root.resolve(), pin, check=args.check) + except (OSError, RuntimeError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + if not changed: + print(f"rclone v{pin.version} pin is current") + return 0 + + relative_paths = [str(path.relative_to(args.repo_root.resolve())) for path in changed] + if args.check: + print( + f"::error title=Stale rclone pin::rclone v{pin.version} differs in " + f"{', '.join(relative_paths)}", + file=sys.stderr, + ) + print( + f"Run: python .github/scripts/update_rclone_pin.py --version {pin.version}", + file=sys.stderr, + ) + return 1 + + print(f"updated rclone v{pin.version} pin in {', '.join(relative_paths)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Makefile b/Makefile index fdb7abecaf..daa1745f56 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,10 @@ sync: uv sync --all-extras --all-packages --group dev +.PHONY: update-rclone-pin +update-rclone-pin: + uv run python .github/scripts/update_rclone_pin.py --cooldown-days $(or $(RCLONE_COOLDOWN_DAYS),7) $(if $(RCLONE_VERSION),--version $(RCLONE_VERSION)) + .PHONY: format format: uv run ruff format @@ -51,6 +55,62 @@ tests-parallel: tests-serial: uv run pytest -m serial +.PHONY: integration-tests +integration-tests: + uv run python .github/scripts/run_integration_tests.py --profile full $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-release +integration-tests-release: + uv run python .github/scripts/run_integration_tests.py --profile release $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-nightly +integration-tests-nightly: + uv run python .github/scripts/run_integration_tests.py --profile nightly $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-manual +integration-tests-manual: + uv run python .github/scripts/run_integration_tests.py --profile manual $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-packaging +integration-tests-packaging: + uv run python .github/scripts/run_integration_tests.py --profile packaging + +.PHONY: integration-tests-core +integration-tests-core: + uv run python .github/scripts/run_integration_tests.py --profile core + +.PHONY: integration-tests-providers +integration-tests-providers: + uv run python .github/scripts/run_integration_tests.py --profile providers $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-providers-external +integration-tests-providers-external: + OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS=1 uv run python .github/scripts/run_integration_tests.py --profile providers $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-providers-all +integration-tests-providers-all: + uv run python .github/scripts/run_integration_tests.py --profile providers --all + +.PHONY: --all +--all: + @: + +.PHONY: integration-tests-realtime +integration-tests-realtime: + uv run python .github/scripts/run_integration_tests.py --profile realtime + +.PHONY: integration-tests-voice +integration-tests-voice: + uv run python .github/scripts/run_integration_tests.py --profile voice + +.PHONY: integration-tests-hosted +integration-tests-hosted: + uv run python .github/scripts/run_integration_tests.py --profile hosted + +.PHONY: integration-tests-extras +integration-tests-extras: + uv run python .github/scripts/run_integration_tests.py --profile extras + .PHONY: coverage coverage: diff --git a/docs/ja/running_agents.md b/docs/ja/running_agents.md index dfb676923c..a45bd142b7 100644 --- a/docs/ja/running_agents.md +++ b/docs/ja/running_agents.md @@ -305,6 +305,8 @@ result = Runner.run_sync( [`RunResultBase.to_input_list()`][agents.result.RunResultBase.to_input_list] メソッドを使用して次のターンの入力を取得し、会話履歴を手動で管理できます。 ```python +from agents import Agent, Runner, trace + async def main(): agent = Agent(name="Assistant", instructions="Reply very concisely.") @@ -327,7 +329,7 @@ async def main(): より簡単な方法として、[Sessions](sessions/index.md) を使用すると、`.to_input_list()` を手動で呼び出すことなく、会話履歴を自動的に処理できます。 ```python -from agents import Agent, Runner, SQLiteSession +from agents import Agent, Runner, SQLiteSession, trace async def main(): agent = Agent(name="Assistant", instructions="Reply very concisely.") @@ -593,4 +595,4 @@ SDK は特定の状況で例外を発生させます。完全な一覧は [`agen - 予期しないツール関連の失敗：モデルが想定された方法でツールを使用できなかった場合です。 - [`ToolTimeoutError`][agents.exceptions.ToolTimeoutError]：関数ツール呼び出しが設定されたタイムアウトを超え、そのツールが `timeout_behavior="raise_exception"` を使用している場合に発生する例外です。 - [`UserError`][agents.exceptions.UserError]：SDK を使用するコードを作成しているユーザーが、SDK の使用時に誤りを犯した場合に発生する例外です。通常は、不適切なコード実装、無効な設定、SDK API の誤用によって発生します。 -- [`InputGuardrailTripwireTriggered`][agents.exceptions.InputGuardrailTripwireTriggered]、[`OutputGuardrailTripwireTriggered`][agents.exceptions.OutputGuardrailTripwireTriggered]：それぞれ、入力ガードレールまたは出力ガードレールの条件が満たされた場合に発生する例外です。入力ガードレールは処理前に受信メッセージをチェックし、出力ガードレールは配信前にエージェントの最終レスポンスをチェックします。 \ No newline at end of file +- [`InputGuardrailTripwireTriggered`][agents.exceptions.InputGuardrailTripwireTriggered]、[`OutputGuardrailTripwireTriggered`][agents.exceptions.OutputGuardrailTripwireTriggered]：それぞれ、入力ガードレールまたは出力ガードレールの条件が満たされた場合に発生する例外です。入力ガードレールは処理前に受信メッセージをチェックし、出力ガードレールは配信前にエージェントの最終レスポンスをチェックします。 diff --git a/docs/ja/tracing.md b/docs/ja/tracing.md index 4f70296e29..b6d27ec849 100644 --- a/docs/ja/tracing.md +++ b/docs/ja/tracing.md @@ -168,7 +168,7 @@ OpenAI 以外のモデルで OpenAI API キーを使用すると、トレーシ ```python import os -from agents import set_tracing_export_api_key, Agent, Runner +from agents import set_tracing_export_api_key, Agent from agents.extensions.models.any_llm_model import AnyLLMModel tracing_api_key = os.environ["OPENAI_API_KEY"] @@ -234,4 +234,4 @@ await Runner.run( - [Asqav](https://www.asqav.com/docs/integrations#openai-agents) - [Datadog](https://docs.datadoghq.com/llm_observability/instrumentation/auto_instrumentation/?tab=python#openai-agents) - [Latitude](https://docs.latitude.so/telemetry/frameworks/openai-agents) -- [DProvenanceKit](https://dprovenance.dev/openai-agents/) \ No newline at end of file +- [DProvenanceKit](https://dprovenance.dev/openai-agents/) diff --git a/docs/ko/running_agents.md b/docs/ko/running_agents.md index 5496bab61c..9cff77086b 100644 --- a/docs/ko/running_agents.md +++ b/docs/ko/running_agents.md @@ -305,6 +305,8 @@ SDK가 이전 출력에서 후속 입력을 구성하는 다중 턴 에이전트 [`RunResultBase.to_input_list()`][agents.result.RunResultBase.to_input_list] 메서드를 사용해 다음 턴의 입력을 가져오는 방식으로 대화 기록을 수동으로 관리할 수 있습니다. ```python +from agents import Agent, Runner, trace + async def main(): agent = Agent(name="Assistant", instructions="Reply very concisely.") @@ -327,7 +329,7 @@ async def main(): 더 간단한 방법으로 [Sessions](sessions/index.md)를 사용하면 `.to_input_list()`를 직접 호출하지 않고도 대화 기록을 자동으로 처리할 수 있습니다. ```python -from agents import Agent, Runner, SQLiteSession +from agents import Agent, Runner, SQLiteSession, trace async def main(): agent = Agent(name="Assistant", instructions="Reply very concisely.") @@ -593,4 +595,4 @@ SDK는 특정 상황에서 예외를 발생시킵니다. 전체 목록은 [`agen - 예상하지 못한 도구 관련 실패: 모델이 예상된 방식으로 도구를 사용하지 못하는 경우 - [`ToolTimeoutError`][agents.exceptions.ToolTimeoutError]: 함수 도구 호출이 구성된 제한 시간을 초과하고 도구에서 `timeout_behavior="raise_exception"`을 사용하는 경우 이 예외가 발생합니다. - [`UserError`][agents.exceptions.UserError]: SDK를 사용하는 코드를 작성한 사람이 SDK를 사용하는 중 오류를 범하면 이 예외가 발생합니다. 일반적으로 잘못된 코드 구현, 유효하지 않은 구성 또는 SDK API의 잘못된 사용으로 인해 발생합니다. -- [`InputGuardrailTripwireTriggered`][agents.exceptions.InputGuardrailTripwireTriggered], [`OutputGuardrailTripwireTriggered`][agents.exceptions.OutputGuardrailTripwireTriggered]: 각각 입력 가드레일 또는 출력 가드레일의 조건이 충족되면 이 예외가 발생합니다. 입력 가드레일은 처리 전에 수신 메시지를 검사하고, 출력 가드레일은 전달 전에 에이전트의 최종 응답을 검사합니다. \ No newline at end of file +- [`InputGuardrailTripwireTriggered`][agents.exceptions.InputGuardrailTripwireTriggered], [`OutputGuardrailTripwireTriggered`][agents.exceptions.OutputGuardrailTripwireTriggered]: 각각 입력 가드레일 또는 출력 가드레일의 조건이 충족되면 이 예외가 발생합니다. 입력 가드레일은 처리 전에 수신 메시지를 검사하고, 출력 가드레일은 전달 전에 에이전트의 최종 응답을 검사합니다. diff --git a/docs/ko/tracing.md b/docs/ko/tracing.md index dda6b6f916..c68e0f3930 100644 --- a/docs/ko/tracing.md +++ b/docs/ko/tracing.md @@ -168,7 +168,7 @@ OpenAI 이외 모델에서 OpenAI API 키를 사용하면 트레이싱을 비활 ```python import os -from agents import set_tracing_export_api_key, Agent, Runner +from agents import set_tracing_export_api_key, Agent from agents.extensions.models.any_llm_model import AnyLLMModel tracing_api_key = os.environ["OPENAI_API_KEY"] @@ -234,4 +234,4 @@ await Runner.run( - [Asqav](https://www.asqav.com/docs/integrations#openai-agents) - [Datadog](https://docs.datadoghq.com/llm_observability/instrumentation/auto_instrumentation/?tab=python#openai-agents) - [Latitude](https://docs.latitude.so/telemetry/frameworks/openai-agents) -- [DProvenanceKit](https://dprovenance.dev/openai-agents/) \ No newline at end of file +- [DProvenanceKit](https://dprovenance.dev/openai-agents/) diff --git a/docs/models/index.md b/docs/models/index.md index bd0db9db4f..930a7a1418 100644 --- a/docs/models/index.md +++ b/docs/models/index.md @@ -387,7 +387,7 @@ if __name__ == "__main__": 1. Sets the name of an OpenAI model directly. 2. Provides a [`Model`][agents.models.interface.Model] implementation. -When you want to further configure the model used for an agent, you can pass [`ModelSettings`][agents.models.interface.ModelSettings], which provides optional model configuration parameters such as temperature. +When you want to further configure the model used for an agent, you can pass [`ModelSettings`][agents.model_settings.ModelSettings], which provides optional model configuration parameters such as temperature. ```python from agents import Agent, ModelSettings diff --git a/docs/tracing.md b/docs/tracing.md index ad73c68329..88adfd1854 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -149,7 +149,7 @@ By default, `trace_include_sensitive_data` is `True`. You can set the default wi The high level architecture for tracing is: -- At initialization, we create a global [`TraceProvider`][agents.tracing.setup.TraceProvider], which is responsible for creating traces. +- At initialization, we create a global [`TraceProvider`][agents.tracing.provider.TraceProvider], which is responsible for creating traces. - We configure the `TraceProvider` with a [`BatchTraceProcessor`][agents.tracing.processors.BatchTraceProcessor] that sends traces/spans in batches to a [`BackendSpanExporter`][agents.tracing.processors.BackendSpanExporter], which exports the spans and traces to the OpenAI backend in batches. To customize this default setup, to send traces to alternative or additional backends or modifying exporter behavior, you have two options: diff --git a/docs/zh/running_agents.md b/docs/zh/running_agents.md index 5537a19b54..ce7a5f95aa 100644 --- a/docs/zh/running_agents.md +++ b/docs/zh/running_agents.md @@ -305,6 +305,8 @@ result = Runner.run_sync( 你可以使用 [`RunResultBase.to_input_list()`][agents.result.RunResultBase.to_input_list] 方法获取下一轮输入，从而手动管理对话历史记录： ```python +from agents import Agent, Runner, trace + async def main(): agent = Agent(name="Assistant", instructions="Reply very concisely.") @@ -327,7 +329,7 @@ async def main(): 若要采用更简单的方式，可以使用 [Sessions](sessions/index.md) 自动处理对话历史记录，而无需手动调用 `.to_input_list()`： ```python -from agents import Agent, Runner, SQLiteSession +from agents import Agent, Runner, SQLiteSession, trace async def main(): agent = Agent(name="Assistant", instructions="Reply very concisely.") @@ -593,4 +595,4 @@ SDK 会在某些情况下引发异常。完整列表请参阅 [`agents.exception - 意外的工具相关故障：模型未能按预期方式使用工具。 - [`ToolTimeoutError`][agents.exceptions.ToolTimeoutError]：当工具调用超过其配置的超时时间，并且该工具使用 `timeout_behavior="raise_exception"` 时，会引发此异常。 - [`UserError`][agents.exceptions.UserError]：当你（使用 SDK 编写代码的人）在使用 SDK 时出错，会引发此异常。这通常是由代码实现不正确、配置无效或误用 SDK API 导致的。 -- [`InputGuardrailTripwireTriggered`][agents.exceptions.InputGuardrailTripwireTriggered]、[`OutputGuardrailTripwireTriggered`][agents.exceptions.OutputGuardrailTripwireTriggered]：分别在满足输入安全防护措施或输出安全防护措施的触发条件时引发。输入安全防护措施会在处理前检查传入消息，而输出安全防护措施会在交付前检查智能体的最终响应。 \ No newline at end of file +- [`InputGuardrailTripwireTriggered`][agents.exceptions.InputGuardrailTripwireTriggered]、[`OutputGuardrailTripwireTriggered`][agents.exceptions.OutputGuardrailTripwireTriggered]：分别在满足输入安全防护措施或输出安全防护措施的触发条件时引发。输入安全防护措施会在处理前检查传入消息，而输出安全防护措施会在交付前检查智能体的最终响应。 diff --git a/docs/zh/tracing.md b/docs/zh/tracing.md index e0fee4e672..3236eb3e4d 100644 --- a/docs/zh/tracing.md +++ b/docs/zh/tracing.md @@ -168,7 +168,7 @@ async def main(): ```python import os -from agents import set_tracing_export_api_key, Agent, Runner +from agents import set_tracing_export_api_key, Agent from agents.extensions.models.any_llm_model import AnyLLMModel tracing_api_key = os.environ["OPENAI_API_KEY"] @@ -234,4 +234,4 @@ await Runner.run( - [Asqav](https://www.asqav.com/docs/integrations#openai-agents) - [Datadog](https://docs.datadoghq.com/llm_observability/instrumentation/auto_instrumentation/?tab=python#openai-agents) - [Latitude](https://docs.latitude.so/telemetry/frameworks/openai-agents) -- [DProvenanceKit](https://dprovenance.dev/openai-agents/) \ No newline at end of file +- [DProvenanceKit](https://dprovenance.dev/openai-agents/) diff --git a/examples/sandbox/docker/Dockerfile.mount b/examples/sandbox/docker/Dockerfile.mount index 576d909b45..7f48dcfb89 100644 --- a/examples/sandbox/docker/Dockerfile.mount +++ b/examples/sandbox/docker/Dockerfile.mount @@ -1,4 +1,11 @@ FROM ubuntu:22.04 + +# BEGIN RCLONE RELEASE PIN +ARG RCLONE_VERSION=1.74.4 +ARG RCLONE_SHA256_LINUX_AMD64=fe435e0c36228e7c2f116a8701f01127bb1f694005fc11d1f27186c8bca4115d +ARG RCLONE_SHA256_LINUX_ARM64=97685285c9ad6a0cf17d5844115d2a67245af6444db672187074bd9c358de419 +# END RCLONE RELEASE PIN + RUN set -eux \ && apt-get update \ && apt-get install -y --no-install-recommends \ @@ -38,8 +45,20 @@ RUN set -eux \ && mount-s3 --version \ && curl -fsSL https://amazon-efs-utils.aws.com/efs-utils-installer.sh | sh -s -- --install \ && mount.s3files --version \ - && curl -fsSL https://rclone.org/install.sh | bash \ + && arch="$(dpkg --print-architecture)" \ + && case "$arch" in \ + amd64) rclone_arch="amd64"; rclone_sha256="$RCLONE_SHA256_LINUX_AMD64" ;; \ + arm64) rclone_arch="arm64"; rclone_sha256="$RCLONE_SHA256_LINUX_ARM64" ;; \ + *) echo "unsupported rclone arch: $arch" >&2; exit 1 ;; \ + esac \ + && rclone_archive="rclone-v${RCLONE_VERSION}-linux-${rclone_arch}.zip" \ + && rclone_url="https://downloads.rclone.org/v${RCLONE_VERSION}/${rclone_archive}" \ + && curl --fail --location --silent --show-error --proto '=https' --tlsv1.2 \ + --output "/tmp/${rclone_archive}" "$rclone_url" \ + && printf '%s %s\n' "$rclone_sha256" "/tmp/${rclone_archive}" | sha256sum --check --strict - \ + && unzip -q "/tmp/${rclone_archive}" -d /tmp/rclone \ + && install -m 0755 "/tmp/rclone/${rclone_archive%.zip}/rclone" /usr/local/bin/rclone \ && rclone version \ && touch /etc/fuse.conf \ - && grep -qxF 'user_allow_other' /etc/fuse.conf || echo 'user_allow_other' >> /etc/fuse.conf \ - && rm -rf /var/lib/apt/lists/* /tmp/mount-s3.deb + && (grep -qxF 'user_allow_other' /etc/fuse.conf || echo 'user_allow_other' >> /etc/fuse.conf) \ + && rm -rf /var/lib/apt/lists/* /tmp/mount-s3.deb /tmp/rclone /tmp/rclone-v*.zip diff --git a/integration_tests/README.md b/integration_tests/README.md new file mode 100644 index 0000000000..4d5db77f50 --- /dev/null +++ b/integration_tests/README.md @@ -0,0 +1,28 @@ +# Packaged live integration tests + +These tests exercise the exact wheel produced by `uv build` after installing it into clean virtual environments. The `integration_tests/` directory, repository automation metadata, and local dependency/type-checking caches are excluded from published distributions. + +Run the complete release-oriented matrix with: + + export UV_DEFAULT_INDEX=https://pypi.org/simple + make integration-tests + +`make integration-tests-release` runs the same release-safe matrix explicitly. `make integration-tests-nightly` also includes extended capability and transport checks, while `make integration-tests-manual` includes checks reserved for an intentionally configured manual run. Focused entry points are `make integration-tests-packaging`, `make integration-tests-core`, `make integration-tests-providers`, `make integration-tests-providers-external`, `make integration-tests-providers-all`, `make integration-tests-realtime`, `make integration-tests-voice`, `make integration-tests-hosted`, and `make integration-tests-extras`. + +Invoke the repository-local `$integration-tests` skill to run the release profile with configured OpenRouter-backed provider checks. OpenRouter provides a single configured gateway for the standard multi-provider matrix; provider-specific direct connections are optional extensions selected explicitly. When a release review also requires runnable examples, run `$examples-auto-run` first and then `$integration-tests`. + +Set `OPENAI_API_KEY` for live OpenAI calls. Override `OPENAI_AGENTS_INTEGRATION_MODEL`, `OPENAI_AGENTS_INTEGRATION_REALTIME_MODEL`, `OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS`, and `OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS` when testing different models or configured providers. Provider model lists contain comma-separated adapter model names and require the credentials matching each selected provider. Set `OPENAI_AGENTS_INTEGRATION_MCP_SERVER_URL` to use another trusted DeepWiki-compatible hosted MCP server that exposes the `ask_question` tool and can answer questions about the `openai/openai-agents-python` repository. + +Run `make integration-tests-providers-external` with `OPENROUTER_API_KEY` to exercise current OpenAI, Anthropic, and Google models through one provider gateway. To extend the matrix with separately configured direct-provider credentials, use `make integration-tests-providers-external -- --all`, `make integration-tests-providers-all`, or `uv run python .github/scripts/run_integration_tests.py --profile providers --all`. Set `ANTHROPIC_API_KEY` and `GEMINI_API_KEY` or `GOOGLE_API_KEY` for the direct providers you want to include. Override `OPENAI_AGENTS_INTEGRATION_ANTHROPIC_MODEL`, `OPENAI_AGENTS_INTEGRATION_GEMINI_MODEL`, or the comma-separated `OPENAI_AGENTS_INTEGRATION_OPENROUTER_MODELS` to select provider models. + +The default general model is `gpt-5.6`, while LiteLLM function-tool cases use the Chat Completions-native `openai/gpt-4.1-mini`. This avoids LiteLLM's separate Responses API bridge and keeps the adapter regression focused on its actual Chat Completions contract. + +When the host requires a SOCKS proxy, the runner installs `httpx[socks]` as a test-harness dependency without changing the SDK's published requirements. Set `OPENAI_AGENTS_INTEGRATION_DISABLE_PROXY=1` when the selected environment should connect without inherited proxy settings. + +Set `OPENAI_AGENTS_INTEGRATION_STRICT=1` to fail rather than skip when a requested live feature is not configured. Integration tests never run as part of ordinary `make tests`. + +Each live test has a 75-second timeout so a stalled provider connection cannot block a release review indefinitely. + +Set `OPENAI_AGENTS_INTEGRATION_PYTHON` to choose the Python interpreter used for isolated environments. For example, `OPENAI_AGENTS_INTEGRATION_PYTHON=3.10 make integration-tests-packaging` verifies the minimum supported Python package and import boundary; use Python 3.11 or newer for the full adapter matrix because the AnyLLM extra requires Python 3.11. + +The release suite also covers canonical and supported legacy public-import identity, client-side handoffs, nested agents as tools, custom and shell tools, namespaced tool search, approval/rejection plus serialized `RunState` resume, durable SQLite sessions, explicit and server-managed conversation continuation, controlled retries, input/output and tool guardrails, explicit prompt caching, structured streaming output, provider token logprobs, hosted web search/MCP approval, hosted multi-agent streaming, programmatic-tool streaming/handoffs, multi-turn Realtime history, usage, handoffs, agent updates, voice failure propagation, and independent installation of each selected optional dependency group. The nightly profile adds extended approval matrices, parallel tool concurrency, stateless reasoning replay, reusable Responses WebSocket sessions, collected trace trees, streamed provider tool calls, Realtime audio/guardrails, and streamed-input voice pipelines. diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py new file mode 100644 index 0000000000..9eeeb2d7d6 --- /dev/null +++ b/integration_tests/conftest.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import importlib +import os +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +import pytest + +LIVE_MARKERS = frozenset({"core", "providers", "realtime", "voice", "hosted"}) + + +@dataclass(frozen=True) +class ExternalProvider: + name: str + model: str + api_key_name: str + + @property + def api_key(self) -> str: + return os.environ[self.api_key_name] + + +def _external_providers_enabled() -> bool: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "").lower() in { + "1", + "true", + "yes", + } + + +def _direct_providers_enabled() -> bool: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", "").lower() in { + "1", + "true", + "yes", + } + + +def _external_providers() -> list[ExternalProvider]: + if not _external_providers_enabled(): + return [] + + providers: list[ExternalProvider] = [] + if os.environ.get("OPENROUTER_API_KEY"): + configured = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_OPENROUTER_MODELS", + "openai/gpt-5.6-luna,anthropic/claude-sonnet-5,google/gemini-3.6-flash", + ) + for model in configured.split(","): + if model.strip(): + providers.append( + ExternalProvider( + name=f"openrouter-{model.strip().replace('/', '-')}", + model=f"openrouter/{model.strip()}", + api_key_name="OPENROUTER_API_KEY", + ) + ) + + if _direct_providers_enabled(): + if os.environ.get("ANTHROPIC_API_KEY"): + providers.append( + ExternalProvider( + name="anthropic", + model="anthropic/" + + os.environ.get( + "OPENAI_AGENTS_INTEGRATION_ANTHROPIC_MODEL", "claude-sonnet-5" + ), + api_key_name="ANTHROPIC_API_KEY", + ) + ) + + gemini_key = "GEMINI_API_KEY" if os.environ.get("GEMINI_API_KEY") else "GOOGLE_API_KEY" + if os.environ.get(gemini_key): + providers.append( + ExternalProvider( + name="gemini", + model="gemini/" + + os.environ.get("OPENAI_AGENTS_INTEGRATION_GEMINI_MODEL", "gemini-3.6-flash"), + api_key_name=gemini_key, + ) + ) + + return providers + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "external_provider" not in metafunc.fixturenames: + return + + providers = _external_providers() + if providers: + metafunc.parametrize("external_provider", providers, ids=[item.name for item in providers]) + return + + metafunc.parametrize("external_provider", [None], ids=["unconfigured"]) + + +def _strict() -> bool: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_STRICT", "").lower() in { + "1", + "true", + "yes", + } + + +def skip_or_fail(reason: str) -> None: + if _strict(): + pytest.fail(reason) + pytest.skip(reason) + + +def _provider_model_credentials(model: str) -> tuple[str, ...]: + provider = model.partition("/")[0] + if provider == "openrouter": + return ("OPENROUTER_API_KEY",) + if provider == "anthropic": + return ("ANTHROPIC_API_KEY",) + if provider in {"gemini", "google"}: + return ("GEMINI_API_KEY", "GOOGLE_API_KEY") + return ("OPENAI_API_KEY",) + + +def _has_provider_credential(credential: str) -> bool: + value = os.environ.get(credential) + if credential == "OPENAI_API_KEY": + return value not in {None, "", "test_key", "fake-for-tests"} + return bool(value) + + +def pytest_runtest_setup(item: pytest.Item) -> None: + if not any(item.get_closest_marker(marker) for marker in LIVE_MARKERS): + return + if item.get_closest_marker("providers"): + fixture_names = getattr(item, "fixturenames", ()) + if "external_provider" in fixture_names: + callspec = getattr(item, "callspec", None) + provider = getattr(callspec, "params", {}).get("external_provider") + if provider is None: + if _external_providers_enabled(): + skip_or_fail( + "External provider coverage requires OPENROUTER_API_KEY or explicitly " + "enabled direct-provider credentials." + ) + pytest.skip( + "Enable external provider coverage and set OPENROUTER_API_KEY, " + "or explicitly include configured direct providers." + ) + return + for fixture_name, environment_name in ( + ("any_llm_models", "OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS"), + ("litellm_models", "OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS"), + ): + if fixture_name not in fixture_names: + continue + configured_models = os.environ.get(environment_name, "") + if not configured_models.strip(): + break + for model in configured_models.split(","): + if not model.strip(): + continue + credentials = _provider_model_credentials(model.strip()) + if not any(_has_provider_credential(credential) for credential in credentials): + skip_or_fail( + f"Set {' or '.join(credentials)} to execute configured provider " + f"model {model.strip()!r}." + ) + return + if os.environ.get("OPENAI_API_KEY") in {None, "", "test_key", "fake-for-tests"}: + skip_or_fail("Set a real OPENAI_API_KEY to execute live integration tests.") + + +@pytest.fixture(scope="session", autouse=True) +def verify_installed_sdk() -> Iterator[None]: + agents = importlib.import_module("agents") + if agents.__file__ is None: + pytest.fail("agents does not expose an installed module path.") + installed_path = Path(agents.__file__).resolve() + environment = Path(sys.prefix).resolve() + if not installed_path.is_relative_to(environment): + pytest.fail(f"agents resolved outside the isolated environment: {installed_path}") + if "site-packages" not in installed_path.parts: + pytest.fail(f"agents did not resolve from an installed distribution: {installed_path}") + yield + + +@pytest.fixture(scope="session") +def integration_model() -> str: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_MODEL", "gpt-5.6") + + +@pytest.fixture(scope="session") +def integration_realtime_model() -> str: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_REALTIME_MODEL", "gpt-realtime-2.1") + + +@pytest.fixture(scope="session") +def any_llm_models(integration_model: str) -> list[str]: + configured = os.environ.get("OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", "") + return [model.strip() for model in configured.split(",") if model.strip()] or [ + f"openai/{integration_model}" + ] + + +@pytest.fixture(scope="session") +def litellm_models() -> list[str]: + configured = os.environ.get("OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS", "") + return [model.strip() for model in configured.split(",") if model.strip()] or [ + "openai/gpt-4.1-mini" + ] + + +@pytest.fixture(scope="session") +async def integration_pcm_audio() -> bytes: + from openai import AsyncOpenAI + + client = AsyncOpenAI() + audio = bytearray() + request = client.audio.speech.with_streaming_response.create( + model="gpt-4o-mini-tts", + voice="alloy", + input="Please say the words packaged voice ready.", + response_format="pcm", + ) + async with request as response: + async for chunk in response.iter_bytes(): + audio.extend(chunk) + + if len(audio) % 2: + audio.append(0) + return bytes(audio) diff --git a/integration_tests/hosted/test_code_interpreter.py b/integration_tests/hosted/test_code_interpreter.py new file mode 100644 index 0000000000..b102ed891c --- /dev/null +++ b/integration_tests/hosted/test_code_interpreter.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import pytest +from openai.types.responses import ResponseReasoningItem + +from agents import Agent, CodeInterpreterTool, RunConfig, Runner +from agents.items import ToolCallItem + +pytestmark = pytest.mark.hosted + + +async def test_code_interpreter_reasoning_items_survive_follow_up_replay( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged code interpreter agent", + model=integration_model, + instructions=( + "Before using any tools, reason through conditional arithmetic to determine which " + "calculation is required. Then use code interpreter for the calculation and " + "answer with RESULT:." + ), + tools=[ + CodeInterpreterTool( + tool_config={"type": "code_interpreter", "container": {"type": "auto"}} + ) + ], + model_settings={ + "max_tokens": 1024, + "reasoning": {"effort": "medium", "summary": "auto"}, + "response_include": ["reasoning.encrypted_content"], + "store": False, + }, + ) + first = await Runner.run( + agent, + "First determine whether the remainder of 4837 multiplied by 8291 divided by 97 " + "is odd. If it is odd, use the code interpreter to calculate 273 * 312821 + 1782; " + "otherwise calculate 19 * 83. Respond only with RESULT:.", + run_config=RunConfig(tracing_disabled=True, reasoning_item_id_policy="omit"), + ) + expected = str(273 * 312821 + 1782) + assert expected in str(first.final_output) + assert any( + isinstance(item, ToolCallItem) + and getattr(item.raw_item, "type", None) == "code_interpreter_call" + for item in first.new_items + ) + + reasoning_items = [ + output + for response in first.raw_responses + for output in response.output + if isinstance(output, ResponseReasoningItem) + ] + assert reasoning_items, [ + getattr(output, "type", type(output).__name__) + for response in first.raw_responses + for output in response.output + ] + + follow_up = first.to_input_list(mode="normalized") + replayed_reasoning = [item for item in follow_up if item.get("type") == "reasoning"] + assert len(replayed_reasoning) == len(reasoning_items) + assert all(isinstance(item.get("encrypted_content"), str) for item in replayed_reasoning) + follow_up.append({"role": "user", "content": "Repeat the calculated result exactly."}) + second = await Runner.run( + agent, + follow_up, + run_config=RunConfig(tracing_disabled=True, reasoning_item_id_policy="omit"), + ) + + assert expected in str(second.final_output) diff --git a/integration_tests/hosted/test_local_tool_families.py b/integration_tests/hosted/test_local_tool_families.py new file mode 100644 index 0000000000..495e86990d --- /dev/null +++ b/integration_tests/hosted/test_local_tool_families.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from agents import ( + Agent, + CustomTool, + ModelSettings, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + RunState, + ShellCommandRequest, + ShellTool, + ToolCallOutputItem, +) +from agents.tool_context import ToolContext + +pytestmark = pytest.mark.hosted + + +@pytest.mark.parametrize( + "streaming", + [False, pytest.param(True, marks=pytest.mark.nightly)], + ids=["nonstreaming", "streaming"], +) +async def test_custom_tools_preserve_raw_string_inputs_and_outputs( + integration_model: str, + streaming: bool, +) -> None: + raw_inputs: list[str] = [] + + async def format_release_word(_context: ToolContext[Any], raw_input: str) -> str: + raw_inputs.append(raw_input) + return raw_input.strip().upper() + + custom = CustomTool( + name="format_release_word", + description="Convert the raw release word to uppercase.", + on_invoke_tool=format_release_word, + ) + agent = Agent( + name="Packaged raw custom tool agent", + model=integration_model, + instructions=( + "Call format_release_word with exactly the raw string amber, " + "then reply exactly CUSTOM:AMBER." + ), + tools=[custom], + model_settings=ModelSettings(tool_choice="required", max_tokens=256), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "Format the release word.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Format the release word.", run_config=config) + + outputs = [item for item in result.new_items if isinstance(item, ToolCallOutputItem)] + assert len(raw_inputs) == 1 + assert raw_inputs[0].strip() == "amber" + assert result.final_output == "CUSTOM:AMBER" + assert len(outputs) == 1 + assert isinstance(outputs[0].raw_item, dict) + assert outputs[0].raw_item["type"] == "custom_tool_call_output" + + +@pytest.mark.nightly +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +async def test_custom_tool_approval_survives_serialized_resume( + integration_model: str, + approved: bool, +) -> None: + calls: list[str] = [] + + async def publish_release(_context: ToolContext[Any], raw_input: str) -> str: + calls.append(raw_input) + return "CUSTOM_APPROVED" + + custom = CustomTool( + name="publish_release_note", + description="Publish the raw release note after operator approval.", + on_invoke_tool=publish_release, + needs_approval=True, + ) + agent = Agent( + name="Packaged approval-gated custom tool agent", + model=integration_model, + instructions=( + "Call publish_release_note with the raw string amber. If approved reply exactly " + "CUSTOM_APPROVED; if rejected reply exactly CUSTOM_REJECTED." + ), + tools=[custom], + model_settings=ModelSettings(tool_choice="required", max_tokens=320), + ) + config = RunConfig(tracing_disabled=True) + first = await Runner.run(agent, "Publish the release note.", run_config=config, max_turns=5) + assert len(first.interruptions) == 1 + + state = await RunState.from_json(agent, first.to_state().to_json()) + if approved: + state.approve(state.get_interruptions()[0]) + else: + state.reject(state.get_interruptions()[0], rejection_message="Publication was declined.") + + resumed = await Runner.run(agent, state, run_config=config, max_turns=5) + outputs = [item for item in resumed.new_items if isinstance(item, ToolCallOutputItem)] + assert calls == (["amber"] if approved else []) + assert resumed.final_output == ("CUSTOM_APPROVED" if approved else "CUSTOM_REJECTED") + assert any( + isinstance(item.raw_item, dict) and item.raw_item.get("type") == "custom_tool_call_output" + for item in outputs + ) + + +@pytest.mark.parametrize( + "streaming", + [False, pytest.param(True, marks=pytest.mark.nightly)], + ids=["nonstreaming", "streaming"], +) +async def test_local_shell_tools_execute_only_the_supplied_safe_harness( + integration_model: str, + streaming: bool, +) -> None: + requested_commands: list[list[str]] = [] + + def execute_shell(request: ShellCommandRequest) -> str: + requested_commands.append(request.data.action.commands) + return "SHELL_CHECKPOINT_READY" + + agent = Agent( + name="Packaged local shell tool agent", + model=integration_model, + instructions=( + "Call the shell tool with exactly the command echo release, " + "then reply exactly SHELL_READY." + ), + tools=[ShellTool(executor=execute_shell)], + model_settings=ModelSettings(tool_choice="required", max_tokens=256), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "Check the release with shell.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Check the release with shell.", run_config=config) + + outputs = [item for item in result.new_items if isinstance(item, ToolCallOutputItem)] + assert requested_commands == [["echo release"]] + assert result.final_output == "SHELL_READY" + assert len(outputs) == 1 + assert isinstance(outputs[0].raw_item, dict) + assert outputs[0].raw_item["type"] == "shell_call_output" + + +@pytest.mark.nightly +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +async def test_local_shell_approval_survives_serialized_resume( + integration_model: str, + approved: bool, +) -> None: + requested_commands: list[list[str]] = [] + + def execute_shell(request: ShellCommandRequest) -> str: + requested_commands.append(request.data.action.commands) + return "SHELL_APPROVED" + + agent = Agent( + name="Packaged approval-gated shell agent", + model=integration_model, + instructions=( + "Call the shell tool with exactly the command echo release. If approved reply " + "exactly SHELL_APPROVED; if rejected reply exactly SHELL_REJECTED." + ), + tools=[ShellTool(executor=execute_shell, needs_approval=True)], + model_settings=ModelSettings(tool_choice="required", max_tokens=320), + ) + config = RunConfig(tracing_disabled=True) + first = await Runner.run(agent, "Check the release with shell.", run_config=config, max_turns=5) + assert len(first.interruptions) == 1 + + state = await RunState.from_json(agent, first.to_state().to_json()) + if approved: + state.approve(state.get_interruptions()[0]) + else: + state.reject(state.get_interruptions()[0], rejection_message="Shell access was declined.") + + resumed = await Runner.run(agent, state, run_config=config, max_turns=5) + assert requested_commands == ([["echo release"]] if approved else []) + assert resumed.final_output == ("SHELL_APPROVED" if approved else "SHELL_REJECTED") + assert any( + isinstance(item, ToolCallOutputItem) + and isinstance(item.raw_item, dict) + and item.raw_item.get("type") == "shell_call_output" + for item in resumed.new_items + ) diff --git a/integration_tests/hosted/test_mcp.py b/integration_tests/hosted/test_mcp.py new file mode 100644 index 0000000000..9b14a00ef3 --- /dev/null +++ b/integration_tests/hosted/test_mcp.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import os + +import pytest + +from agents import Agent, HostedMCPTool, ModelSettings, RunConfig, Runner, RunState +from agents.items import ( + MCPApprovalRequestItem, + MCPApprovalResponseItem, + MCPListToolsItem, + ToolCallItem, +) +from agents.model_settings import MCPToolChoice + +pytestmark = pytest.mark.hosted + + +async def test_hosted_mcp_lists_and_calls_a_trusted_remote_server(integration_model: str) -> None: + server_url = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_MCP_SERVER_URL", "https://mcp.deepwiki.com/mcp" + ) + agent = Agent( + name="Packaged hosted MCP agent", + model=integration_model, + instructions=( + "Use the DeepWiki MCP server to identify the main programming language of " + "openai/openai-agents-python." + ), + model_settings={"max_tokens": 768}, + tools=[ + HostedMCPTool( + tool_config={ + "type": "mcp", + "server_label": "packaged_deepwiki", + "server_url": server_url, + "require_approval": "never", + } + ) + ], + ) + result = await Runner.run( + agent, + "Which language is the openai/openai-agents-python repository mainly written in?", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + + assert "python" in str(result.final_output).lower() + assert any(isinstance(item, MCPListToolsItem) for item in result.new_items) + assert any( + isinstance(item, ToolCallItem) and getattr(item.raw_item, "type", None) == "mcp_call" + for item in result.new_items + ) + + +async def test_hosted_mcp_approval_survives_serialized_pause_and_resume( + integration_model: str, +) -> None: + server_url = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_MCP_SERVER_URL", "https://mcp.deepwiki.com/mcp" + ) + agent = Agent( + name="Packaged hosted MCP approval agent", + model=integration_model, + instructions="Use the DeepWiki MCP server to answer the repository language question.", + model_settings=ModelSettings( + max_tokens=768, + tool_choice=MCPToolChoice(server_label="packaged_mcp_approval", name="ask_question"), + ), + tools=[ + HostedMCPTool( + tool_config={ + "type": "mcp", + "server_label": "packaged_mcp_approval", + "server_url": server_url, + "require_approval": "always", + } + ) + ], + ) + first = await Runner.run( + agent, + "Which language is the openai/openai-agents-python repository mainly written in?", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + + assert len(first.interruptions) == 1 + assert any(isinstance(item, MCPApprovalRequestItem) for item in first.new_items) + state = await RunState.from_json(agent, first.to_state().to_json()) + state.approve(state.get_interruptions()[0]) + resumed = await Runner.run( + agent, + state, + run_config=RunConfig( + tracing_disabled=True, + model_settings=ModelSettings(tool_choice="auto"), + ), + max_turns=6, + ) + + assert "python" in str(resumed.final_output).lower() + assert any(isinstance(item, MCPApprovalResponseItem) for item in resumed.new_items) + + +@pytest.mark.nightly +async def test_hosted_mcp_rejection_survives_serialized_pause_and_resume( + integration_model: str, +) -> None: + server_url = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_MCP_SERVER_URL", "https://mcp.deepwiki.com/mcp" + ) + agent = Agent( + name="Packaged hosted MCP rejection agent", + model=integration_model, + instructions=( + "Use the DeepWiki MCP server to answer the repository language question. " + "If the request is rejected, reply exactly MCP_REJECTED." + ), + model_settings=ModelSettings( + max_tokens=512, + tool_choice=MCPToolChoice(server_label="packaged_mcp_rejection", name="ask_question"), + ), + tools=[ + HostedMCPTool( + tool_config={ + "type": "mcp", + "server_label": "packaged_mcp_rejection", + "server_url": server_url, + "require_approval": "always", + "allowed_tools": ["ask_question"], + } + ) + ], + ) + + first = await Runner.run( + agent, + "What is the main repository language?", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + assert len(first.interruptions) == 1 + restored = await RunState.from_json(agent, first.to_state().to_json()) + restored.reject(restored.get_interruptions()[0], rejection_message="Remote access declined.") + resumed = await Runner.run( + agent, + restored, + run_config=RunConfig( + tracing_disabled=True, + model_settings=ModelSettings(tool_choice="auto"), + ), + max_turns=6, + ) + + assert resumed.final_output == "MCP_REJECTED" + assert any(isinstance(item, MCPApprovalResponseItem) for item in resumed.new_items) diff --git a/integration_tests/hosted/test_multi_agent.py b/integration_tests/hosted/test_multi_agent.py new file mode 100644 index 0000000000..a20b6eb7f5 --- /dev/null +++ b/integration_tests/hosted/test_multi_agent.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import os +from typing import Any + +import pytest + +from agents import Agent, RunConfig, Runner, RunResult, RunResultStreaming +from agents.decorators import tool +from agents.extensions.experimental.hosted_multi_agent import ( + HostedMultiAgentConfig, + OpenAIHostedMultiAgentModel, + get_hosted_agent_metadata, +) +from agents.tool_context import ToolContext + +pytestmark = pytest.mark.hosted + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_hosted_multi_agent_preserves_subagent_tool_callers(streaming: bool) -> None: + model_name = os.environ.get("OPENAI_AGENTS_INTEGRATION_HOSTED_MODEL", "gpt-5.6-sol") + proposals = {"alpha": 6, "beta": 8} + callers: set[str] = set() + call_ids: set[str] = set() + + @tool + def inspect_proposal(ctx: ToolContext[Any], proposal: str) -> dict[str, object]: + """Return deterministic details for one proposal.""" + metadata = get_hosted_agent_metadata(ctx) + callers.add(metadata.agent_name if metadata else "/root") + call_ids.add(ctx.tool_call_id) + return {"proposal": proposal, "estimated_weeks": proposals[proposal]} + + agent = Agent( + name="Packaged hosted coordinator", + model=OpenAIHostedMultiAgentModel( + model=model_name, + config=HostedMultiAgentConfig(max_concurrent_subagents=2), + ), + instructions=( + "Create two subagents. Have one inspect proposal alpha and the other inspect " + "proposal beta. Each subagent must call inspect_proposal before you compare them." + ), + tools=[inspect_proposal], + ) + result: RunResult | RunResultStreaming + if streaming: + streamed = Runner.run_streamed( + agent, + "Compare proposal alpha and proposal beta.", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + event_types = [event.type async for event in streamed.stream_events()] + assert "raw_response_event" in event_types + result = streamed + else: + result = await Runner.run( + agent, + "Compare proposal alpha and proposal beta.", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + + assert result.final_output + assert len(call_ids) == 2 + assert len(callers) >= 2 + assert "/root" not in callers diff --git a/integration_tests/hosted/test_programmatic_tool_calling.py b/integration_tests/hosted/test_programmatic_tool_calling.py new file mode 100644 index 0000000000..48cf76c667 --- /dev/null +++ b/integration_tests/hosted/test_programmatic_tool_calling.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import json +from typing import Any, cast + +import pytest +from openai.types.responses import ResponseFunctionToolCall +from openai.types.responses.response_output_item import Program +from pydantic import BaseModel + +from agents import ( + Agent, + ModelSettings, + ProgrammaticToolCallingTool, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + RunState, + ToolCallItem, + ToolCallOutputItem, + ToolGuardrailFunctionOutput, + ToolInputGuardrailData, + ToolOutputGuardrailData, + handoff, +) +from agents.decorators import tool, tool_input_guardrail, tool_output_guardrail +from agents.extensions.handoff_filters import remove_all_tools +from agents.handoffs import HandoffInputData + +pytestmark = pytest.mark.hosted + + +class InventoryResult(BaseModel): + units: int + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_programmatic_tool_calling_retains_program_owned_calls_and_output( + integration_model: str, streaming: bool +) -> None: + calls: list[str] = [] + + @tool(allowed_callers=["programmatic"]) + def read_inventory(sku: str) -> InventoryResult: + """Return the deterministic available units for an item.""" + calls.append(sku) + return InventoryResult(units={"alpha": 7, "beta": 11}[sku]) + + agent = Agent( + name="Packaged programmatic tool agent", + model=integration_model, + instructions=( + "Use Programmatic Tool Calling. Generate a JavaScript program that calls " + "read_inventory('alpha') and read_inventory('beta') with Promise.all, adds the " + "units fields from their returned objects, and returns the result. Then answer " + "exactly TOTAL:18." + ), + model_settings=ModelSettings(tool_choice="programmatic_tool_calling", max_tokens=1024), + tools=[read_inventory, ProgrammaticToolCallingTool()], + ) + result: RunResult | RunResultStreaming + if streaming: + streamed = Runner.run_streamed( + agent, + "Calculate the total inventory.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + async for _event in streamed.stream_events(): + pass + result = streamed + else: + result = await Runner.run( + agent, + "Calculate the total inventory.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + program_calls = [ + item.raw_item + for item in result.new_items + if isinstance(item, ToolCallItem) + and isinstance(item.raw_item, ResponseFunctionToolCall) + and item.raw_item.caller is not None + and item.raw_item.caller.type == "program" + ] + + assert sorted(calls) == ["alpha", "beta"] + assert len(program_calls) == 2 + assert any( + isinstance(item, ToolCallItem) and isinstance(item.raw_item, Program) + for item in result.new_items + ) + assert any( + isinstance(item, ToolCallOutputItem) + and getattr(item.raw_item, "type", None) == "program_output" + for item in result.new_items + ) + assert result.final_output == "TOTAL:18" + + +async def test_programmatic_tool_history_survives_a_filtered_handoff( + integration_model: str, +) -> None: + calls: list[str] = [] + handoff_filter_inputs: list[tuple[HandoffInputData, HandoffInputData]] = [] + + def capture_filtered_handoff(input_data: HandoffInputData) -> HandoffInputData: + filtered = remove_all_tools(input_data) + handoff_filter_inputs.append((input_data, filtered)) + return filtered + + @tool(allowed_callers=["programmatic"]) + def inspect_inventory(sku: str) -> InventoryResult: + """Return deterministic inventory details to the hosted program.""" + calls.append(sku) + return InventoryResult(units=18) + + specialist = Agent( + name="Packaged program summary specialist", + model=integration_model, + instructions="Reply with exactly FILTERED_PROGRAM_HANDOFF_OK.", + model_settings={"max_tokens": 256}, + ) + coordinator = Agent( + name="Packaged program handoff coordinator", + model=integration_model, + instructions=( + "First use Programmatic Tool Calling to run inspect_inventory('alpha'). " + "After the program returns, immediately transfer to the summary specialist." + ), + tools=[inspect_inventory, ProgrammaticToolCallingTool()], + handoffs=[handoff(specialist, input_filter=capture_filtered_handoff)], + model_settings={"max_tokens": 1024}, + ) + result = await Runner.run( + coordinator, + "Inspect alpha with a program, then transfer the answer.", + run_config=RunConfig(tracing_disabled=True, nest_handoff_history=True), + max_turns=7, + ) + + assert calls == ["alpha"] + assert result.final_output == "FILTERED_PROGRAM_HANDOFF_OK" + assert result.last_agent is specialist + assert len(handoff_filter_inputs) == 1 + original_input, filtered_input = handoff_filter_inputs[0] + assert any( + isinstance(item, ToolCallItem | ToolCallOutputItem) + for item in (*original_input.pre_handoff_items, *original_input.new_items) + ) + assert not any( + isinstance(item, ToolCallItem | ToolCallOutputItem) + for item in (*filtered_input.pre_handoff_items, *filtered_input.new_items) + ) + assert any( + isinstance(output, Program) + for response in result.raw_responses + for output in response.output + ) + + +@pytest.mark.nightly +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +async def test_programmatic_tool_approval_preserves_caller_across_serialized_resume( + integration_model: str, approved: bool +) -> None: + calls: list[str] = [] + + @tool(allowed_callers=["programmatic"], needs_approval=True) + def approve_inventory(sku: str) -> InventoryResult: + """Read inventory only after the program's tool call is approved.""" + calls.append(sku) + return InventoryResult(units=18) + + agent = Agent( + name="Packaged programmatic approval agent", + model=integration_model, + instructions=( + "Use Programmatic Tool Calling to invoke approve_inventory('alpha'). " + "If it succeeds reply exactly PROGRAM_APPROVED; if it is rejected reply " + "exactly PROGRAM_REJECTED." + ), + model_settings=ModelSettings(tool_choice="programmatic_tool_calling", max_tokens=1024), + tools=[approve_inventory, ProgrammaticToolCallingTool()], + ) + config = RunConfig(tracing_disabled=True) + + first = await Runner.run(agent, "Read the protected inventory.", run_config=config, max_turns=6) + assert len(first.interruptions) == 1 + state = await RunState.from_json(agent, first.to_state().to_json()) + interruption = state.get_interruptions()[0] + if approved: + state.approve(interruption) + else: + state.reject(interruption, rejection_message="Inventory access was rejected.") + + resumed = await Runner.run(agent, state, run_config=config, max_turns=6) + outputs = [item for item in resumed.new_items if isinstance(item, ToolCallOutputItem)] + + assert calls == (["alpha"] if approved else []) + assert outputs + if approved: + assert resumed.final_output == "PROGRAM_APPROVED" + else: + rejected_item = next( + item + for item in outputs + if isinstance(item.raw_item, dict) + and item.raw_item.get("type") == "function_call_output" + ) + assert rejected_item.output == "Inventory access was rejected." + assert json.loads(cast(dict[str, Any], rejected_item.raw_item)["output"]) == { + "error": "Inventory access was rejected." + } + callers = [ + cast(dict[str, Any], item.raw_item).get("caller") + if isinstance(item.raw_item, dict) + else getattr(item.raw_item, "caller", None) + for item in outputs + ] + assert any( + (caller.get("type") if isinstance(caller, dict) else getattr(caller, "type", None)) + == "program" + for caller in callers + ) + + +@pytest.mark.nightly +@pytest.mark.parametrize("rejection_stage", ["input", "output"]) +async def test_programmatic_structured_tool_guardrail_errors_are_valid_json( + integration_model: str, rejection_stage: str +) -> None: + calls: list[str] = [] + rejection_message = f"Inventory {rejection_stage} was rejected." + + @tool_input_guardrail + def inspect_input(_data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + if rejection_stage == "input": + return ToolGuardrailFunctionOutput.reject_content(rejection_message) + return ToolGuardrailFunctionOutput.allow() + + @tool_output_guardrail + def inspect_output(_data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput: + if rejection_stage == "output": + return ToolGuardrailFunctionOutput.reject_content(rejection_message) + return ToolGuardrailFunctionOutput.allow() + + @tool( + allowed_callers=["programmatic"], + tool_input_guardrails=[inspect_input], + tool_output_guardrails=[inspect_output], + ) + def inspect_inventory(sku: str) -> InventoryResult: + """Read inventory after both programmatic tool guardrails allow the request.""" + calls.append(sku) + return InventoryResult(units=18) + + agent = Agent( + name="Packaged programmatic guardrail rejection agent", + model=integration_model, + instructions=( + "Use Programmatic Tool Calling. Write a JavaScript program that calls " + "inspect_inventory('alpha') and returns the resulting error field when present. " + "After the program finishes, reply exactly PROGRAM_GUARDRAIL_REJECTED." + ), + model_settings=ModelSettings(tool_choice="programmatic_tool_calling", max_tokens=1024), + tools=[inspect_inventory, ProgrammaticToolCallingTool()], + ) + + result = await Runner.run( + agent, + "Inspect the guarded inventory and report its rejection.", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + rejected_item = next( + item + for item in result.new_items + if isinstance(item, ToolCallOutputItem) + and isinstance(item.raw_item, dict) + and item.raw_item.get("type") == "function_call_output" + ) + + assert calls == ([] if rejection_stage == "input" else ["alpha"]) + assert rejected_item.output == rejection_message + assert json.loads(cast(dict[str, Any], rejected_item.raw_item)["output"]) == { + "error": rejection_message + } + assert result.final_output == "PROGRAM_GUARDRAIL_REJECTED" diff --git a/integration_tests/hosted/test_tool_search.py b/integration_tests/hosted/test_tool_search.py new file mode 100644 index 0000000000..55482c9809 --- /dev/null +++ b/integration_tests/hosted/test_tool_search.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import pytest + +from agents import ( + Agent, + ModelSettings, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + ToolCallItem, + ToolCallOutputItem, + ToolSearchCallItem, + ToolSearchOutputItem, + ToolSearchTool, + tool_namespace, +) +from agents.decorators import tool + +pytestmark = pytest.mark.hosted + + +@pytest.mark.parametrize( + "streaming", + [False, pytest.param(True, marks=pytest.mark.nightly)], + ids=["nonstreaming", "streaming"], +) +async def test_tool_search_loads_and_executes_a_deferred_namespaced_tool( + integration_model: str, streaming: bool +) -> None: + calls: list[str] = [] + + @tool(defer_loading=True) + def lookup_customer(customer_id: str) -> str: + """Find the customer's release readiness status.""" + calls.append(customer_id) + return "READY" + + namespaced = tool_namespace( + name="customer_support", + description="Look up customer release readiness and support records.", + tools=[lookup_customer], + ) + agent = Agent( + name="Packaged deferred tool search agent", + model=integration_model, + instructions=( + "Find the customer support tool, call lookup_customer with customer_id='customer-42', " + "and then reply with exactly SEARCH_READY." + ), + tools=[*namespaced, ToolSearchTool()], + model_settings=ModelSettings(max_tokens=512, parallel_tool_calls=False), + ) + result: RunResult | RunResultStreaming + if streaming: + streamed = Runner.run_streamed( + agent, + "Find and run the deferred customer lookup.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + events = [event async for event in streamed.stream_events()] + assert any(event.type == "raw_response_event" for event in events) + result = streamed + else: + result = await Runner.run( + agent, + "Find and run the deferred customer lookup.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + + assert calls == ["customer-42"] + assert result.final_output == "SEARCH_READY" + assert any(isinstance(item, ToolSearchCallItem) for item in result.new_items) + assert any(isinstance(item, ToolSearchOutputItem) for item in result.new_items) + assert any(isinstance(item, ToolCallItem) for item in result.new_items) + assert any(isinstance(item, ToolCallOutputItem) for item in result.new_items) + + +async def test_tool_search_routes_identically_named_tools_by_namespace( + integration_model: str, +) -> None: + calls: list[str] = [] + + @tool(name_override="lookup", defer_loading=True) + def lookup_billing(customer_id: str) -> str: + """Look up the customer's billing status.""" + calls.append(f"billing:{customer_id}") + return "BILLING_READY" + + @tool(name_override="lookup", defer_loading=True) + def lookup_shipping(customer_id: str) -> str: + """Look up the customer's package shipping status.""" + calls.append(f"shipping:{customer_id}") + return "SHIPPING_READY" + + agent = Agent( + name="Packaged namespaced tool routing agent", + model=integration_model, + instructions=( + "Find the shipping namespace tool named lookup and call it exactly once with " + "customer_id='customer-42'. Do not use billing. Reply exactly SHIPPING_READY." + ), + tools=[ + *tool_namespace(name="billing", description="Billing records", tools=[lookup_billing]), + *tool_namespace( + name="shipping", description="Package shipping records", tools=[lookup_shipping] + ), + ToolSearchTool(), + ], + model_settings=ModelSettings(max_tokens=512, parallel_tool_calls=False), + ) + + result = await Runner.run( + agent, + "Check the customer's shipping status.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + + assert calls == ["shipping:customer-42"] + assert result.final_output == "SHIPPING_READY" diff --git a/integration_tests/hosted/test_web_search.py b/integration_tests/hosted/test_web_search.py new file mode 100644 index 0000000000..98ec547b57 --- /dev/null +++ b/integration_tests/hosted/test_web_search.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from agents import Agent, RunConfig, Runner, ToolCallItem, WebSearchTool + +pytestmark = pytest.mark.hosted + + +async def test_web_search_emits_provider_owned_call_items(integration_model: str) -> None: + agent = Agent( + name="Packaged web search agent", + model=integration_model, + instructions=( + "Search the web before answering. Identify the organization that publishes " + "the OpenAI Agents Python SDK, then answer with only OPENAI." + ), + model_settings={"max_tokens": 768}, + tools=[WebSearchTool()], + ) + result = await Runner.run( + agent, + "Search for the official openai-agents-python GitHub repository publisher.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert result.final_output.strip().upper() == "OPENAI" + assert any( + isinstance(item, ToolCallItem) and getattr(item.raw_item, "type", None) == "web_search_call" + for item in result.new_items + ) diff --git a/integration_tests/openai/test_approval_resume.py b/integration_tests/openai/test_approval_resume.py new file mode 100644 index 0000000000..850fb76d18 --- /dev/null +++ b/integration_tests/openai/test_approval_resume.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from agents import ( + Agent, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + RunState, + SQLiteSession, + ToolCallOutputItem, +) +from agents.decorators import tool + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_tool_approval_survives_serialized_state_and_resume( + integration_model: str, approved: bool, streaming: bool +) -> None: + calls: list[str] = [] + + @tool(needs_approval=True) + def perform_action(action: str) -> str: + """Perform the deterministic action only after explicit approval.""" + calls.append(action) + return "completed" + + agent = Agent( + name="Packaged approval agent", + model=integration_model, + instructions=( + "Call perform_action with action='deploy'. If the tool succeeds, reply exactly " + "APPROVED. If the tool is rejected, reply exactly REJECTED." + ), + tools=[perform_action], + model_settings={"max_tokens": 384}, + ) + config = RunConfig(tracing_disabled=True) + first: RunResult | RunResultStreaming + resumed: RunResult | RunResultStreaming + + if streaming: + first_stream = Runner.run_streamed(agent, "Perform the deployment.", run_config=config) + async for _event in first_stream.stream_events(): + pass + first = first_stream + else: + first = await Runner.run(agent, "Perform the deployment.", run_config=config) + + assert len(first.interruptions) == 1 + interruption = first.interruptions[0] + assert interruption.name == "perform_action" + state_json = first.to_state().to_json() + restored = await RunState.from_json(agent, state_json) + restored_interruption = restored.get_interruptions()[0] + + if approved: + restored.approve(restored_interruption) + else: + restored.reject(restored_interruption, rejection_message="The operator rejected deploy.") + + if streaming: + resumed_stream = Runner.run_streamed(agent, restored, run_config=config) + async for _event in resumed_stream.stream_events(): + pass + resumed = resumed_stream + else: + resumed = await Runner.run(agent, restored, run_config=config) + + assert resumed.final_output == ("APPROVED" if approved else "REJECTED") + assert calls == (["deploy"] if approved else []) + assert any(isinstance(item, ToolCallOutputItem) for item in resumed.new_items) + + +async def test_approval_resume_preserves_durable_sqlite_tool_history( + integration_model: str, tmp_path: Path +) -> None: + calls: list[str] = [] + + @tool(needs_approval=True) + def confirm_release(version: str) -> str: + """Confirm a release after its approval decision is restored.""" + calls.append(version) + return "approved" + + agent = Agent( + name="Packaged durable approval agent", + model=integration_model, + instructions="Call confirm_release with version='1.0', then reply RELEASE_APPROVED.", + model_settings={"max_tokens": 384}, + tools=[confirm_release], + ) + session = SQLiteSession("packaged-approval", tmp_path / "approval.sqlite3") + config = RunConfig(tracing_disabled=True) + try: + first = await Runner.run( + agent, + "Approve the release.", + session=session, + run_config=config, + ) + restored = await RunState.from_json(agent, first.to_state().to_json()) + restored.approve(restored.get_interruptions()[0]) + resumed = await Runner.run(agent, restored, session=session, run_config=config) + saved_items = await session.get_items() + finally: + session.close() + + assert calls == ["1.0"] + assert resumed.final_output == "RELEASE_APPROVED" + assert sum(item.get("role") == "user" for item in saved_items) == 1 + assert sum(item.get("type") == "function_call_output" for item in saved_items) == 1 + + +async def test_parallel_tool_approvals_preserve_mixed_decisions_after_serialization( + integration_model: str, tmp_path: Path +) -> None: + calls: list[str] = [] + + @tool(needs_approval=True) + def approve_release(version: str) -> str: + """Approve a deterministic release version.""" + calls.append(f"release:{version}") + return "release-approved" + + @tool(needs_approval=True) + def notify_customer(customer: str) -> str: + """Notify a deterministic customer.""" + calls.append(f"customer:{customer}") + return "customer-notified" + + agent = Agent( + name="Packaged mixed approval agent", + model=integration_model, + instructions=( + "In the same turn, call approve_release with version='1.0' and notify_customer " + "with customer='customer-42'. After their approval decisions, reply exactly " + "MIXED_APPROVAL_READY." + ), + model_settings={"max_tokens": 512, "parallel_tool_calls": True}, + tools=[approve_release, notify_customer], + ) + session = SQLiteSession("packaged-mixed-approval", tmp_path / "mixed-approval.sqlite3") + config = RunConfig(tracing_disabled=True) + try: + first = await Runner.run( + agent, "Perform both requested actions.", session=session, run_config=config + ) + assert len(first.interruptions) == 2 + restored = await RunState.from_json(agent, first.to_state().to_json()) + for interruption in restored.get_interruptions(): + if interruption.name == "approve_release": + restored.approve(interruption) + else: + restored.reject(interruption, rejection_message="Customer notification declined.") + resumed = await Runner.run(agent, restored, session=session, run_config=config) + stored = await session.get_items() + finally: + session.close() + + assert calls == ["release:1.0"] + assert resumed.final_output == "MIXED_APPROVAL_READY" + assert sum(item.get("role") == "user" for item in stored) == 1 + outputs = [item for item in stored if item.get("type") == "function_call_output"] + assert len(outputs) == 2 diff --git a/integration_tests/openai/test_chat_completions.py b/integration_tests/openai/test_chat_completions.py new file mode 100644 index 0000000000..8a54eeb561 --- /dev/null +++ b/integration_tests/openai/test_chat_completions.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from typing import Any + +import pytest +from openai import AsyncOpenAI +from pydantic import BaseModel + +from agents import Agent, RunConfig, Runner, RunResult, RunResultStreaming +from agents.decorators import tool +from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel + +pytestmark = pytest.mark.core + + +class ChatCompletionStatus(BaseModel): + status: str + checkpoints: list[int] + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_chat_completions_tools_settings_and_usage( + integration_model: str, dictionary: bool, streaming: bool +) -> None: + from agents import ModelSettings + + calls: list[str] = [] + + @tool + def package_status(package: str) -> str: + """Return a deterministic package status.""" + calls.append(package) + return "ready" + + values: dict[str, Any] = { + "reasoning": {"effort": "none"}, + "include_usage": True, + "extra_args": {"max_completion_tokens": 512}, + } + settings = values if dictionary else ModelSettings(**values) + agent = Agent( + name="Packaged Chat Completions agent", + model=OpenAIChatCompletionsModel( + model=integration_model, + openai_client=AsyncOpenAI(), + ), + instructions=( + "Call package_status exactly once with package='openai-agents', then reply " + "exactly CHAT_READY." + ), + model_settings=settings, + tools=[package_status], + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + + if streaming: + result = Runner.run_streamed(agent, "Check the package.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Check the package.", run_config=config) + + assert calls == ["openai-agents"] + assert result.final_output == "CHAT_READY" + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_chat_completions_preserves_typed_structured_output( + integration_model: str, + streaming: bool, +) -> None: + agent = Agent( + name="Packaged structured Chat Completions agent", + model=OpenAIChatCompletionsModel( + model=integration_model, + openai_client=AsyncOpenAI(), + ), + instructions="Return status CHAT_STRUCTURED_READY and checkpoints [2, 4, 8].", + output_type=ChatCompletionStatus, + model_settings={"reasoning": {"effort": "none"}, "include_usage": True}, + ) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed( + agent, + "Return the requested typed release status.", + run_config=RunConfig(tracing_disabled=True), + ) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run( + agent, + "Return the requested typed release status.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert result.final_output == ChatCompletionStatus( + status="CHAT_STRUCTURED_READY", checkpoints=[2, 4, 8] + ) + assert result.context_wrapper.usage.total_tokens > 0 diff --git a/integration_tests/openai/test_execution_controls.py b/integration_tests/openai/test_execution_controls.py new file mode 100644 index 0000000000..f0807fa7a5 --- /dev/null +++ b/integration_tests/openai/test_execution_controls.py @@ -0,0 +1,492 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any, cast + +import pytest +from openai.types.responses import ResponseReasoningItem + +from agents import ( + Agent, + AgentHookContext, + ModelSettings, + RunConfig, + RunContextWrapper, + RunErrorHandlerInput, + RunErrorHandlerResult, + RunHooks, + Runner, + RunResult, + RunResultStreaming, + SQLiteSession, + Tool, + ToolCallOutputItem, + ToolExecutionConfig, +) +from agents.decorators import tool +from agents.items import ModelResponse, TResponseInputItem +from agents.run_config import CallModelData, ModelInputData + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_stop_on_first_tool_avoids_a_follow_up_model_request( + integration_model: str, + streaming: bool, +) -> None: + calls: list[str] = [] + + @tool + def resolve_checkpoint(checkpoint: str) -> str: + """Return the requested release checkpoint directly.""" + calls.append(checkpoint) + return "STOP_ON_FIRST_TOOL_READY" + + agent = Agent( + name="Packaged stop-on-tool agent", + model=integration_model, + instructions="Call resolve_checkpoint exactly once with checkpoint='release'.", + tools=[resolve_checkpoint], + tool_use_behavior="stop_on_first_tool", + model_settings={"max_tokens": 256, "tool_choice": "required"}, + ) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed( + agent, + "Return the checkpoint through the tool.", + run_config=RunConfig(tracing_disabled=True), + ) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run( + agent, + "Return the checkpoint through the tool.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["release"] + assert result.final_output == "STOP_ON_FIRST_TOOL_READY" + assert result.context_wrapper.usage.requests == 1 + assert len(result.raw_responses) == 1 + + +@pytest.mark.nightly +@pytest.mark.parametrize("max_concurrency", [1, 2], ids=["sequential", "bounded-parallel"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_parallel_function_tools_preserve_order_and_sdk_concurrency_limits( + integration_model: str, + max_concurrency: int, + streaming: bool, +) -> None: + active_calls = 0 + peak_concurrency = 0 + + async def run_checkpoint(name: str) -> str: + nonlocal active_calls, peak_concurrency + active_calls += 1 + peak_concurrency = max(peak_concurrency, active_calls) + try: + await asyncio.sleep(0.08) + return name.upper() + finally: + active_calls -= 1 + + @tool + async def checkpoint_alpha(checkpoint: str) -> str: + """Return the alpha release checkpoint.""" + return await run_checkpoint(checkpoint) + + @tool + async def checkpoint_beta(checkpoint: str) -> str: + """Return the beta release checkpoint.""" + return await run_checkpoint(checkpoint) + + settings = ModelSettings( + tool_choice="required", + parallel_tool_calls=True, + max_tokens=512, + ) + agent = Agent( + name="Packaged bounded tool concurrency agent", + model=integration_model, + instructions=( + "In the same turn, call checkpoint_alpha with checkpoint='alpha' and " + "checkpoint_beta with checkpoint='beta'. After both tools finish reply exactly " + "CONCURRENCY_READY." + ), + tools=[checkpoint_alpha, checkpoint_beta], + model_settings=settings, + ) + config = RunConfig( + tracing_disabled=True, + tool_execution=ToolExecutionConfig(max_function_tool_concurrency=max_concurrency), + ) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "Run both release checkpoints.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Run both release checkpoints.", run_config=config) + + outputs = [item.output for item in result.new_items if isinstance(item, ToolCallOutputItem)] + + assert result.final_output == "CONCURRENCY_READY" + assert outputs == ["ALPHA", "BETA"] + assert peak_concurrency == max_concurrency + assert result.context_wrapper.usage.requests == 2 + assert settings.tool_choice == "required" + + +async def test_session_merge_and_model_input_filter_have_distinct_persistence_boundaries( + integration_model: str, + tmp_path: Path, +) -> None: + callback_inputs: list[tuple[int, str]] = [] + filter_inputs: list[str] = [] + agent = Agent( + name="Packaged session input filtering agent", + model=integration_model, + instructions="Remember user-provided release words and reply exactly as requested.", + model_settings={"max_tokens": 256}, + ) + session = SQLiteSession("packaged-filtered-session", tmp_path / "filtered.sqlite3") + + try: + await Runner.run( + agent, + "Remember the release word JASPER and reply only STORED.", + session=session, + run_config=RunConfig(tracing_disabled=True), + ) + + def merge_session_input( + history: list[TResponseInputItem], + new_input: list[TResponseInputItem], + ) -> list[TResponseInputItem]: + callback_inputs.append((len(history), str(new_input[0].get("content")))) + rewritten = cast( + TResponseInputItem, + { + "role": "user", + "content": "What release word did I provide? Reply only with that word.", + }, + ) + return [*history, rewritten] + + def filter_model_input(data: CallModelData[Any]) -> ModelInputData: + latest = data.model_data.input[-1] + filter_inputs.append(str(latest.get("content"))) + return ModelInputData( + input=data.model_data.input, + instructions=(data.model_data.instructions or "") + + " Prefix the remembered word with FILTERED: and reply with nothing else.", + ) + + result = await Runner.run( + agent, + "PLACEHOLDER_NEW_INPUT", + session=session, + run_config=RunConfig( + tracing_disabled=True, + session_input_callback=merge_session_input, + call_model_input_filter=filter_model_input, + ), + ) + persisted = await session.get_items() + finally: + session.close() + + assert len(callback_inputs) == 1 + assert callback_inputs[0][0] >= 2 + assert callback_inputs[0][1] == "PLACEHOLDER_NEW_INPUT" + assert filter_inputs == ["What release word did I provide? Reply only with that word."] + assert result.final_output == "FILTERED:JASPER" + assert any("What release word" in str(item.get("content", "")) for item in persisted) + assert not any("PLACEHOLDER_NEW_INPUT" in str(item.get("content", "")) for item in persisted) + + +@pytest.mark.nightly +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_stateless_reasoning_replay_preserves_encrypted_content_when_returned( + integration_model: str, + streaming: bool, +) -> None: + stored_words: list[str] = [] + + @tool + def remember_word(word: str) -> str: + """Store the release word and return a deterministic acknowledgement.""" + stored_words.append(word) + return "WORD_STORED" + + agent = Agent( + name="Packaged stateless reasoning replay agent", + model=integration_model, + instructions=( + "Calculate the requested arithmetic before calling remember_word. Use the word " + "AMBER when the result is odd and COBALT when it is even. Once the tool succeeds " + "reply exactly STORED. Answer follow-up questions using the previous tool result." + ), + tools=[remember_word], + model_settings=ModelSettings( + store=False, + reasoning={"effort": "medium", "summary": "auto"}, + response_include=["reasoning.encrypted_content"], + max_tokens=1024, + ), + ) + config = RunConfig(tracing_disabled=True, reasoning_item_id_policy="omit") + first = await Runner.run( + agent, + "What is the remainder when 4837 multiplied by 8291 is divided by 97? " + "Follow the parity rule, call remember_word, and then reply only STORED.", + run_config=config, + ) + reasoning_items = [ + item + for response in first.raw_responses + for item in response.output + if isinstance(item, ResponseReasoningItem) + ] + assert reasoning_items, "The stateless response did not contain any reasoning items." + replay = first.to_input_list(mode="normalized") + replayed_reasoning = [item for item in replay if item.get("type") == "reasoning"] + replay.append( + cast( + TResponseInputItem, + {"role": "user", "content": "What release word did I provide? Reply only AMBER."}, + ) + ) + + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, replay, run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, replay, run_config=config) + + assert all(isinstance(item.encrypted_content, str) for item in reasoning_items) + assert len(replayed_reasoning) == len(reasoning_items) + assert all(isinstance(item.get("encrypted_content"), str) for item in replayed_reasoning) + assert all("id" not in item for item in replayed_reasoning) + assert first.context_wrapper.usage.requests == 2 + assert result.context_wrapper.usage.requests == 1 + assert stored_words == ["AMBER"] + assert result.final_output == "AMBER" + + +@pytest.mark.parametrize("use_session", [False, True], ids=["without-session", "sqlite-session"]) +async def test_cancel_after_turn_resumes_without_repeating_function_tools( + integration_model: str, + use_session: bool, +) -> None: + calls: list[str] = [] + + @tool + def checkpoint(value: str) -> str: + """Record one deterministic cancellation checkpoint.""" + calls.append(value) + return "CANCEL_CHECKPOINT_READY" + + session = SQLiteSession("packaged-cancel-after-turn") if use_session else None + agent = Agent( + name="Packaged streamed cancellation agent", + model=integration_model, + instructions=( + "Call checkpoint with value='release'. After the tool returns, reply exactly " + "CANCEL_RESUMED_READY." + ), + tools=[checkpoint], + model_settings={"max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + + try: + result = Runner.run_streamed( + agent, + "Run the release checkpoint.", + session=session, + run_config=config, + ) + async for event in result.stream_events(): + if getattr(event, "name", None) == "tool_called": + result.cancel(mode="after_turn") + + replay = result.to_input_list(mode="normalized") + resumed = await Runner.run(result.last_agent, replay, run_config=config) + persisted = await session.get_items() if session is not None else [] + finally: + if session is not None: + session.close() + + assert result.final_output is None + assert result.is_complete + assert calls == ["release"] + assert resumed.final_output == "CANCEL_RESUMED_READY" + assert result.context_wrapper.usage.requests == 1 + assert resumed.context_wrapper.usage.requests == 1 + if use_session: + assert any(item.get("type") == "function_call_output" for item in persisted) + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_max_turn_error_handler_preserves_tool_side_effects_and_history( + integration_model: str, + streaming: bool, +) -> None: + calls: list[str] = [] + handled: list[str] = [] + + @tool + def release_checkpoint(value: str) -> str: + """Return a release checkpoint before the model exceeds its turn limit.""" + calls.append(value) + return "CHECKPOINT_RECORDED" + + def handle_max_turns(data: RunErrorHandlerInput[Any]) -> RunErrorHandlerResult: + handled.append(type(data.error).__name__) + return RunErrorHandlerResult(final_output="MAX_TURNS_RECOVERED", include_in_history=False) + + session = SQLiteSession(f"packaged-max-turn-recovery-{streaming}") + agent = Agent( + name="Packaged max-turn recovery agent", + model=integration_model, + instructions="Call release_checkpoint with value='release', then explain the result.", + tools=[release_checkpoint], + model_settings={"tool_choice": "required", "max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + + try: + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed( + agent, + "Run the release checkpoint.", + session=session, + run_config=config, + max_turns=1, + error_handlers={"max_turns": handle_max_turns}, + ) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run( + agent, + "Run the release checkpoint.", + session=session, + run_config=config, + max_turns=1, + error_handlers={"max_turns": handle_max_turns}, + ) + persisted = await session.get_items() + finally: + session.close() + + assert calls == ["release"] + assert handled == ["MaxTurnsExceeded"] + assert result.final_output == "MAX_TURNS_RECOVERED" + assert any(item.get("type") == "function_call_output" for item in persisted) + assert not any("MAX_TURNS_RECOVERED" in str(item) for item in persisted) + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_live_run_hooks_preserve_model_and_function_tool_event_order( + integration_model: str, + streaming: bool, +) -> None: + observed: list[str] = [] + + class RecordingHooks(RunHooks[Any]): + async def on_agent_start(self, context: AgentHookContext[Any], agent: Agent[Any]) -> None: + del context, agent + observed.append("agent_start") + + async def on_agent_end( + self, + context: AgentHookContext[Any], + agent: Agent[Any], + output: Any, + ) -> None: + del context, agent, output + observed.append("agent_end") + + async def on_llm_start( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + system_prompt: str | None, + input_items: list[TResponseInputItem], + ) -> None: + del context, agent, system_prompt, input_items + observed.append("model_start") + + async def on_llm_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + response: ModelResponse, + ) -> None: + del context, agent, response + observed.append("model_end") + + async def on_tool_start( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + tool: Tool, + ) -> None: + del context, agent, tool + observed.append("tool_start") + + async def on_tool_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + tool: Tool, + result: object, + ) -> None: + del context, agent, tool, result + observed.append("tool_end") + + @tool + def inspect_release(value: str) -> str: + """Inspect a release checkpoint for lifecycle hook ordering.""" + observed.append(f"tool_call:{value}") + return "HOOK_CHECKPOINT_READY" + + agent = Agent( + name="Packaged run hooks agent", + model=integration_model, + instructions=("Call inspect_release with value='release', then reply exactly HOOKS_READY."), + tools=[inspect_release], + model_settings={"max_tokens": 256}, + ) + hooks = RecordingHooks() + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "Inspect the release.", hooks=hooks, run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Inspect the release.", hooks=hooks, run_config=config) + + assert result.final_output == "HOOKS_READY" + assert observed == [ + "agent_start", + "model_start", + "model_end", + "tool_start", + "tool_call:release", + "tool_end", + "model_start", + "model_end", + "agent_end", + ] diff --git a/integration_tests/openai/test_guardrails.py b/integration_tests/openai/test_guardrails.py new file mode 100644 index 0000000000..7d5235e2a4 --- /dev/null +++ b/integration_tests/openai/test_guardrails.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from agents import ( + Agent, + GuardrailFunctionOutput, + InputGuardrailTripwireTriggered, + OutputGuardrailTripwireTriggered, + RunConfig, + RunContextWrapper, + Runner, + ToolExecutionConfig, + ToolGuardrailFunctionOutput, + ToolInputGuardrailData, + ToolOutputGuardrailData, +) +from agents.decorators import ( + input_guardrail, + output_guardrail, + tool, + tool_input_guardrail, + tool_output_guardrail, +) + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("blocked", [False, True], ids=["accepted", "blocked"]) +async def test_output_guardrails_validate_real_model_results( + integration_model: str, blocked: bool +) -> None: + inspected: list[str] = [] + + @output_guardrail + async def inspect_result( + context: RunContextWrapper[Any], agent: Agent[Any], output: str + ) -> GuardrailFunctionOutput: + del context, agent + inspected.append(output) + return GuardrailFunctionOutput( + output_info={"checked": True}, + tripwire_triggered=blocked, + ) + + agent = Agent( + name="Packaged output guardrail agent", + model=integration_model, + instructions="Reply with exactly GUARDED_RESULT.", + output_guardrails=[inspect_result], + model_settings={"max_tokens": 256}, + ) + if blocked: + with pytest.raises(OutputGuardrailTripwireTriggered): + await Runner.run( + agent, + "Return the deterministic guarded result.", + run_config=RunConfig(tracing_disabled=True), + ) + else: + result = await Runner.run( + agent, + "Return the deterministic guarded result.", + run_config=RunConfig(tracing_disabled=True), + ) + assert result.final_output == "GUARDED_RESULT" + + assert inspected == ["GUARDED_RESULT"] + + +@pytest.mark.parametrize("blocked", [False, True], ids=["accepted", "blocked"]) +async def test_input_guardrails_validate_live_run_requests( + integration_model: str, blocked: bool +) -> None: + inspected: list[str] = [] + + @input_guardrail + async def inspect_input( + context: RunContextWrapper[Any], agent: Agent[Any], input: str | list[Any] + ) -> GuardrailFunctionOutput: + del context, agent + inspected.append(str(input)) + return GuardrailFunctionOutput(output_info={"checked": True}, tripwire_triggered=blocked) + + agent = Agent( + name="Packaged input guardrail agent", + model=integration_model, + instructions="Reply with exactly INPUT_GUARDRAIL_READY.", + input_guardrails=[inspect_input], + model_settings={"max_tokens": 256}, + ) + if blocked: + with pytest.raises(InputGuardrailTripwireTriggered): + await Runner.run( + agent, + "Check the input guardrail.", + run_config=RunConfig(tracing_disabled=True), + ) + else: + result = await Runner.run( + agent, + "Check the input guardrail.", + run_config=RunConfig(tracing_disabled=True), + ) + assert result.final_output == "INPUT_GUARDRAIL_READY" + + assert inspected == ["Check the input guardrail."] + + +async def test_tool_input_and_output_guardrails_preserve_live_execution_order( + integration_model: str, +) -> None: + observed: list[str] = [] + + @tool_input_guardrail + def inspect_input(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + observed.append(f"input:{data.context.tool_name}") + return ToolGuardrailFunctionOutput.allow() + + @tool_output_guardrail + def inspect_output(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput: + observed.append(f"output:{data.output}") + return ToolGuardrailFunctionOutput.allow() + + @tool( + tool_input_guardrails=[inspect_input], + tool_output_guardrails=[inspect_output], + ) + def guarded_lookup(value: int) -> str: + """Look up a deterministic guarded value.""" + observed.append(f"tool:{value}") + return "guarded-ready" + + agent = Agent( + name="Packaged tool guardrail agent", + model=integration_model, + instructions="Call guarded_lookup with value 42 and then reply TOOL_GUARDRAILS_READY.", + tools=[guarded_lookup], + model_settings={"max_tokens": 384}, + ) + + result = await Runner.run( + agent, + "Use the guarded lookup.", + run_config=RunConfig( + tracing_disabled=True, + tool_execution=ToolExecutionConfig(pre_approval_tool_input_guardrails=True), + ), + ) + + assert result.final_output == "TOOL_GUARDRAILS_READY" + assert observed == ["input:guarded_lookup", "tool:42", "output:guarded-ready"] + + +@pytest.mark.parametrize("blocked", [False, True], ids=["accepted", "blocked"]) +async def test_streaming_output_guardrails_validate_live_model_results( + integration_model: str, blocked: bool +) -> None: + inspected: list[str] = [] + + @output_guardrail + async def inspect_result( + context: RunContextWrapper[Any], agent: Agent[Any], output: str + ) -> GuardrailFunctionOutput: + del context, agent + inspected.append(output) + return GuardrailFunctionOutput(output_info={"checked": True}, tripwire_triggered=blocked) + + agent = Agent( + name="Packaged streamed output guardrail agent", + model=integration_model, + instructions="Reply with exactly STREAM_GUARDED_RESULT.", + output_guardrails=[inspect_result], + model_settings={"max_tokens": 256}, + ) + result = Runner.run_streamed( + agent, + "Return the deterministic streamed guarded result.", + run_config=RunConfig(tracing_disabled=True), + ) + if blocked: + with pytest.raises(OutputGuardrailTripwireTriggered): + async for _event in result.stream_events(): + pass + else: + async for _event in result.stream_events(): + pass + assert result.final_output == "STREAM_GUARDED_RESULT" + + assert inspected == ["STREAM_GUARDED_RESULT"] diff --git a/integration_tests/openai/test_handoffs.py b/integration_tests/openai/test_handoffs.py new file mode 100644 index 0000000000..35758011d2 --- /dev/null +++ b/integration_tests/openai/test_handoffs.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import pytest + +from agents import ( + Agent, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + ToolCallItem, + ToolCallOutputItem, + handoff, +) +from agents.decorators import tool +from agents.extensions.handoff_filters import remove_all_tools + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +@pytest.mark.parametrize("nested", [False, True], ids=["flat-history", "nested-history"]) +async def test_client_side_handoff_preserves_tool_ownership_and_filtered_history( + integration_model: str, streaming: bool, nested: bool +) -> None: + calls: list[str] = [] + + @tool + def lookup_ticket(ticket: str) -> str: + """Return the deterministic status for a support ticket.""" + calls.append(ticket) + return "resolved" + + specialist = Agent( + name="Packaged support specialist", + model=integration_model, + instructions=( + "Call lookup_ticket exactly once with ticket='CASE-42', then answer " + "exactly HANDOFF_RESOLVED." + ), + tools=[lookup_ticket], + model_settings={"max_tokens": 512}, + ) + coordinator = Agent( + name="Packaged handoff coordinator", + model=integration_model, + instructions="Immediately transfer this support ticket to the support specialist.", + handoffs=[handoff(specialist, input_filter=remove_all_tools)], + model_settings={"max_tokens": 512}, + ) + config = RunConfig(tracing_disabled=True, nest_handoff_history=nested) + result: RunResult | RunResultStreaming + + if streaming: + streamed = Runner.run_streamed( + coordinator, "Resolve support ticket CASE-42.", run_config=config + ) + event_types = [event.type async for event in streamed.stream_events()] + assert "agent_updated_stream_event" in event_types + result = streamed + else: + result = await Runner.run(coordinator, "Resolve support ticket CASE-42.", run_config=config) + + assert calls == ["CASE-42"] + assert result.final_output == "HANDOFF_RESOLVED" + assert result.last_agent is specialist + assert any( + isinstance(item, ToolCallItem) and item.agent is specialist for item in result.new_items + ) + assert any( + isinstance(item, ToolCallOutputItem) and item.agent is specialist + for item in result.new_items + ) + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_nested_agent_as_tool_runs_against_the_installed_distribution( + integration_model: str, streaming: bool +) -> None: + worker = Agent( + name="Packaged nested worker", + model=integration_model, + instructions="Reply with exactly INNER:42.", + model_settings={"max_tokens": 256}, + ) + coordinator = Agent( + name="Packaged nested coordinator", + model=integration_model, + instructions="Call ask_worker, then reply exactly OUTER:42.", + model_settings={"max_tokens": 384}, + tools=[ + worker.as_tool( + tool_name="ask_worker", + tool_description="Ask the nested worker for the deterministic answer.", + ) + ], + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + + if streaming: + streamed = Runner.run_streamed(coordinator, "Use the nested worker.", run_config=config) + async for _event in streamed.stream_events(): + pass + result = streamed + else: + result = await Runner.run(coordinator, "Use the nested worker.", run_config=config) + + assert result.final_output == "OUTER:42" + assert any(isinstance(item, ToolCallItem) for item in result.new_items) + assert any(isinstance(item, ToolCallOutputItem) for item in result.new_items) diff --git a/integration_tests/openai/test_model_settings.py b/integration_tests/openai/test_model_settings.py new file mode 100644 index 0000000000..0796e3a6e8 --- /dev/null +++ b/integration_tests/openai/test_model_settings.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from typing import Any + +import pytest +from openai.resources.responses import AsyncResponses +from openai.types.shared import Reasoning + +from agents import Agent, ModelSettings, RunConfig, Runner +from agents.retry import ModelRetryBackoffSettings, ModelRetrySettings + +pytestmark = pytest.mark.core + + +@pytest.fixture +def captured_response_requests(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: + requests: list[dict[str, Any]] = [] + original_create = AsyncResponses.create + + async def capture_request(responses: AsyncResponses, *args: Any, **kwargs: Any) -> Any: + requests.append(kwargs) + return await original_create(responses, *args, **kwargs) + + monkeypatch.setattr(AsyncResponses, "create", capture_request) + return requests + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +async def test_agent_model_settings_reach_the_live_responses_api( + integration_model: str, dictionary: bool, captured_response_requests: list[dict[str, Any]] +) -> None: + settings: ModelSettings | dict[str, Any] + if dictionary: + settings = {"reasoning": {"effort": "low"}, "max_tokens": 256} + else: + settings = ModelSettings(reasoning=Reasoning(effort="low"), max_tokens=256) + + agent = Agent( + name="Packaged settings agent", + model=integration_model, + instructions="Reply with exactly PACKAGED_SETTINGS_OK.", + model_settings=settings, + ) + result = await Runner.run(agent, "Confirm the packaged settings path.") + + assert isinstance(agent.model_settings, ModelSettings) + assert result.final_output == "PACKAGED_SETTINGS_OK" + assert result.context_wrapper.usage.total_tokens > 0 + assert len(captured_response_requests) == 1 + assert captured_response_requests[0]["max_output_tokens"] == 256 + assert captured_response_requests[0]["reasoning"].effort == "low" + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +async def test_run_config_model_settings_reach_the_live_responses_api( + integration_model: str, dictionary: bool, captured_response_requests: list[dict[str, Any]] +) -> None: + settings: ModelSettings | dict[str, Any] + if dictionary: + settings = {"reasoning": {"effort": "low"}, "max_tokens": 256} + else: + settings = ModelSettings(reasoning=Reasoning(effort="low"), max_tokens=256) + + config = RunConfig(model_settings=settings, tracing_disabled=True) + agent = Agent( + name="Packaged run configuration agent", + model=integration_model, + instructions="Reply with exactly RUN_CONFIG_OK.", + ) + result = await Runner.run(agent, "Confirm the packaged run configuration.", run_config=config) + + assert isinstance(config.model_settings, ModelSettings) + assert result.final_output == "RUN_CONFIG_OK" + assert len(captured_response_requests) == 1 + assert captured_response_requests[0]["max_output_tokens"] == 256 + assert captured_response_requests[0]["reasoning"].effort == "low" + + +async def test_nested_retry_settings_and_clone_dictionaries_reach_the_api( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged nested settings agent", + model=integration_model, + instructions="Reply with exactly NESTED_SETTINGS_OK.", + model_settings={ + "max_tokens": 256, + "reasoning": {"effort": "low"}, + "retry": { + "max_retries": 0, + "backoff": {"initial_delay": 0.0}, + }, + }, + ) + assert isinstance(agent.model_settings.retry, ModelRetrySettings) + assert isinstance(agent.model_settings.retry.backoff, ModelRetryBackoffSettings) + cloned = agent.clone( + model_settings={ + "max_tokens": 256, + "reasoning": {"effort": "low"}, + "retry": {"max_retries": 0, "backoff": {"initial_delay": 0.0}}, + } + ) + result = await Runner.run(cloned, "Confirm provider-specific settings normalization.") + + assert isinstance(cloned.model_settings, ModelSettings) + assert isinstance(cloned.model_settings.retry, ModelRetrySettings) + assert isinstance(cloned.model_settings.retry.backoff, ModelRetryBackoffSettings) + assert result.final_output == "NESTED_SETTINGS_OK" diff --git a/integration_tests/openai/test_responses.py b/integration_tests/openai/test_responses.py new file mode 100644 index 0000000000..c67effea7b --- /dev/null +++ b/integration_tests/openai/test_responses.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from typing import Any + +import pytest +from openai.resources.responses import AsyncResponses +from pydantic import BaseModel + +from agents import ( + Agent, + ModelSettings, + RunConfig, + Runner, + RunResult, + RunResultStreaming, +) +from agents.decorators import tool +from agents.items import ToolCallItem, ToolCallOutputItem + +pytestmark = pytest.mark.core + + +class StructuredStatus(BaseModel): + status: str + value: int + + +class NestedStructuredStatus(BaseModel): + result: StructuredStatus + note: str | None = None + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_responses_function_tools_preserve_calls_outputs_and_usage( + integration_model: str, streaming: bool +) -> None: + called: list[int] = [] + + @tool + def double_number(value: int) -> int: + """Double the supplied number.""" + called.append(value) + return value * 2 + + agent = Agent( + name="Packaged Responses tool agent", + model=integration_model, + instructions="Call double_number with value 21, then reply exactly RESULT:42.", + tools=[double_number], + model_settings=ModelSettings(max_tokens=512), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + + if streaming: + result = Runner.run_streamed(agent, "Use the tool now.", run_config=config) + events = [event async for event in result.stream_events()] + assert any(event.type == "raw_response_event" for event in events) + else: + result = await Runner.run(agent, "Use the tool now.", run_config=config) + + assert called == [21] + assert result.final_output == "RESULT:42" + assert any(isinstance(item, ToolCallItem) for item in result.new_items) + assert any(isinstance(item, ToolCallOutputItem) for item in result.new_items) + assert result.context_wrapper.usage.total_tokens > 0 + + +async def test_responses_structured_output_is_deserialized_from_the_installed_wheel( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged structured output agent", + model=integration_model, + instructions="Return status READY and value 42.", + output_type=StructuredStatus, + model_settings={"max_tokens": 256}, + ) + result = await Runner.run( + agent, + "Return the requested structured result.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert isinstance(result.final_output, StructuredStatus) + assert result.final_output.status == "READY" + assert result.final_output.value == 42 + + +async def test_previous_response_id_preserves_server_managed_conversation( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged server conversation agent", + model=integration_model, + model_settings={"max_tokens": 256}, + ) + first = await Runner.run( + agent, + "Remember that the secret verification word is ORCHID. Reply only STORED.", + run_config=RunConfig(tracing_disabled=True), + ) + assert first.last_response_id is not None + + second = await Runner.run( + agent, + "What verification word did I ask you to remember? Reply with only that word.", + previous_response_id=first.last_response_id, + run_config=RunConfig(tracing_disabled=True), + ) + + assert second.final_output.strip().upper() == "ORCHID" + assert second.last_response_id != first.last_response_id + + +async def test_streaming_structured_output_preserves_nested_optional_fields( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged streamed structured output agent", + model=integration_model, + instructions="Return result status READY, result value 42, and note null.", + output_type=NestedStructuredStatus, + model_settings={"max_tokens": 384}, + ) + + result = Runner.run_streamed( + agent, + "Return the nested structured status.", + run_config=RunConfig(tracing_disabled=True), + ) + event_types = [event.type async for event in result.stream_events()] + + assert isinstance(result.final_output, NestedStructuredStatus) + assert result.final_output.result == StructuredStatus(status="READY", value=42) + assert result.final_output.note is None + assert "raw_response_event" in event_types + + +async def test_explicit_prompt_cache_settings_reach_the_live_responses_api( + integration_model: str, monkeypatch: pytest.MonkeyPatch +) -> None: + captured_requests: list[dict[str, Any]] = [] + original_create = AsyncResponses.create + + async def capture_request(responses: AsyncResponses, *args: Any, **kwargs: Any) -> Any: + captured_requests.append(kwargs) + return await original_create(responses, *args, **kwargs) + + monkeypatch.setattr(AsyncResponses, "create", capture_request) + prefix = " ".join(f"release-checkpoint-{index}" for index in range(1100)) + agent = Agent( + name="Packaged prompt caching agent", + model=integration_model, + instructions="Reply with exactly PROMPT_CACHE_READY.", + model_settings=ModelSettings( + max_tokens=128, + prompt_cache_options={"mode": "explicit", "ttl": "30m"}, + extra_args={"prompt_cache_key": "packaged-integration-explicit-cache"}, + ), + ) + request_input: list[Any] = [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": prefix, + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + {"type": "input_text", "text": "Reply with PROMPT_CACHE_READY."}, + ], + } + ] + + result = await Runner.run( + agent, + request_input, + run_config=RunConfig(tracing_disabled=True), + ) + + assert result.final_output == "PROMPT_CACHE_READY" + assert result.context_wrapper.usage.input_tokens > 0 + assert len(captured_requests) == 1 + assert captured_requests[0]["prompt_cache_options"] == {"mode": "explicit", "ttl": "30m"} + assert captured_requests[0]["prompt_cache_key"] == "packaged-integration-explicit-cache" + assert captured_requests[0]["input"][0]["content"][0]["prompt_cache_breakpoint"] == { + "mode": "explicit" + } diff --git a/integration_tests/openai/test_retry.py b/integration_tests/openai/test_retry.py new file mode 100644 index 0000000000..85a0c1e480 --- /dev/null +++ b/integration_tests/openai/test_retry.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import httpx +import pytest +from openai import APIConnectionError, AsyncOpenAI + +from agents import ( + Agent, + ModelRetrySettings, + ModelSettings, + OpenAIResponsesModel, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + SQLiteSession, + retry_policies, +) + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_retry_reaches_real_api_without_rewinding_session_input( + integration_model: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, streaming: bool +) -> None: + model = OpenAIResponsesModel(model=integration_model, openai_client=AsyncOpenAI()) + original_fetch = model._fetch_response + attempts = 0 + + async def fail_once(*args: Any, **kwargs: Any) -> Any: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise APIConnectionError( + message="Controlled integration-test transport failure.", + request=httpx.Request("POST", "https://api.openai.com/v1/responses"), + ) + return await original_fetch(*args, **kwargs) + + monkeypatch.setattr(model, "_fetch_response", fail_once) + agent = Agent( + name="Packaged real retry agent", + model=model, + instructions="Reply with exactly RETRY_RECOVERED.", + model_settings=ModelSettings( + max_tokens=256, + retry=ModelRetrySettings( + max_retries=1, + backoff={"initial_delay": 0.0}, + policy=retry_policies.network_error(), + ), + ), + ) + session = SQLiteSession("packaged-retry", tmp_path / "retry.sqlite3") + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + + try: + if streaming: + streamed = Runner.run_streamed( + agent, "Recover exactly once.", session=session, run_config=config + ) + async for _event in streamed.stream_events(): + pass + result = streamed + else: + result = await Runner.run( + agent, "Recover exactly once.", session=session, run_config=config + ) + session_items = await session.get_items() + finally: + session.close() + + assert attempts == 2 + assert result.final_output == "RETRY_RECOVERED" + assert [item.get("role") for item in session_items] == ["user", "assistant"] diff --git a/integration_tests/openai/test_sessions.py b/integration_tests/openai/test_sessions.py new file mode 100644 index 0000000000..73d3132815 --- /dev/null +++ b/integration_tests/openai/test_sessions.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Literal + +import pytest +from openai import AsyncOpenAI + +from agents import ( + Agent, + ModelSettings, + OpenAIConversationsSession, + OpenAIResponsesCompactionSession, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + SQLiteSession, +) +from agents.decorators import tool + +pytestmark = pytest.mark.core + + +async def test_sqlite_session_persists_tool_history_across_reopened_instances( + integration_model: str, tmp_path: Path +) -> None: + calls: list[str] = [] + + @tool + def lookup_codeword(label: str) -> str: + """Look up the deterministic secret codeword.""" + calls.append(label) + return "MARIGOLD" + + agent = Agent( + name="Packaged SQLite session agent", + model=integration_model, + instructions="Use lookup_codeword when requested and remember its result.", + model_settings={"max_tokens": 384}, + tools=[lookup_codeword], + ) + database = tmp_path / "conversation.sqlite3" + session = SQLiteSession("packaged-live-session", database) + config = RunConfig(tracing_disabled=True) + try: + first = await Runner.run( + agent, + "Use lookup_codeword with label='release' and reply only STORED.", + session=session, + run_config=config, + ) + assert first.final_output == "STORED" + assert calls == ["release"] + finally: + session.close() + + reopened = SQLiteSession("packaged-live-session", database) + try: + second = await Runner.run( + agent, + "What exact codeword did the tool return? Answer with that word only.", + session=reopened, + run_config=config, + ) + saved_items = await reopened.get_items() + finally: + reopened.close() + + assert second.final_output.strip().upper() == "MARIGOLD" + assert calls == ["release"] + assert any(item.get("type") == "function_call_output" for item in saved_items) + + +async def test_explicit_input_replay_preserves_a_real_response_history( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged explicit replay agent", + model=integration_model, + model_settings={"max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + first = await Runner.run( + agent, + "Remember that the verification number is 907. Reply only STORED.", + run_config=config, + ) + replay = first.to_input_list() + assert any( + item.get("role") == "user" and "verification number is 907" in str(item.get("content")) + for item in replay + ) + second = await Runner.run( + agent, + replay + + [ + { + "role": "user", + "content": "What verification number did I provide? Reply with only the number.", + } + ], + run_config=config, + ) + + assert second.final_output.strip() == "907" + + +async def test_streamed_previous_response_id_continues_server_managed_history( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged streamed continuation agent", + model=integration_model, + model_settings={"max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + first = await Runner.run( + agent, + "Remember that the state token is IVORY. Reply only STORED.", + run_config=config, + ) + assert first.last_response_id is not None + + second = Runner.run_streamed( + agent, + "What state token did I provide? Answer with only the token.", + previous_response_id=first.last_response_id, + run_config=config, + ) + async for _event in second.stream_events(): + pass + + assert second.final_output.strip().upper() == "IVORY" + assert second.last_response_id != first.last_response_id + + +async def test_openai_conversation_id_preserves_server_owned_state( + integration_model: str, +) -> None: + client = AsyncOpenAI() + conversation = await client.conversations.create() + agent = Agent( + name="Packaged OpenAI conversation agent", + model=integration_model, + model_settings={"max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + + try: + await Runner.run( + agent, + "Remember the project color is CERULEAN. Reply only STORED.", + conversation_id=conversation.id, + run_config=config, + ) + second = await Runner.run( + agent, + "What is the project color? Reply with only the color.", + conversation_id=conversation.id, + run_config=config, + ) + finally: + await client.conversations.delete(conversation.id) + + assert second.final_output.strip().upper() == "CERULEAN" + + +async def test_auto_previous_response_id_preserves_tool_output_across_turns( + integration_model: str, +) -> None: + calls: list[str] = [] + + @tool + def read_checkpoint(name: str) -> str: + """Read a deterministic server-managed continuation checkpoint.""" + calls.append(name) + return "CHECKPOINT:84" + + agent = Agent( + name="Packaged automatic continuation agent", + model=integration_model, + instructions=( + "Call read_checkpoint with name='release', then reply exactly AUTO_CONTINUATION:84." + ), + model_settings={"max_tokens": 384}, + tools=[read_checkpoint], + ) + result = await Runner.run( + agent, + "Read the release checkpoint.", + auto_previous_response_id=True, + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["release"] + assert result.final_output == "AUTO_CONTINUATION:84" + assert result.last_response_id is not None + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_openai_conversations_session_preserves_server_managed_history( + integration_model: str, + streaming: bool, +) -> None: + session = OpenAIConversationsSession(session_settings={"limit": 20}) + agent = Agent( + name="Packaged OpenAI Conversations session agent", + model=integration_model, + instructions="Remember user-provided release words and follow exact output instructions.", + model_settings={"max_tokens": 192}, + ) + config = RunConfig(tracing_disabled=True) + + try: + first = await Runner.run( + agent, + "Remember the release word COBALT and reply only STORED.", + session=session, + run_config=config, + ) + second: RunResult | RunResultStreaming + if streaming: + second = Runner.run_streamed( + agent, + "What release word did I give you? Reply with that word only.", + session=session, + run_config=config, + ) + async for _event in second.stream_events(): + pass + else: + second = await Runner.run( + agent, + "What release word did I give you? Reply with that word only.", + session=session, + run_config=config, + ) + items = await session.get_items() + finally: + await session.clear_session() + + assert first.final_output == "STORED" + assert second.final_output == "COBALT" + assert len(items) >= 4 + assert any("COBALT" in str(item) for item in items) + + +@pytest.mark.nightly +@pytest.mark.parametrize( + ("compaction_mode", "store"), + [("auto", False), ("previous_response_id", True)], + ids=["stateless-input", "stored-previous-response"], +) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_responses_compaction_preserves_history_across_owner_modes( + integration_model: str, + compaction_mode: Literal["auto", "previous_response_id"], + store: bool, + streaming: bool, +) -> None: + underlying = SQLiteSession(f"packaged-compaction-{compaction_mode}-{streaming}") + compacted = OpenAIResponsesCompactionSession( + session_id=underlying.session_id, + underlying_session=underlying, + model=integration_model, + compaction_mode=compaction_mode, + should_trigger_compaction=lambda context: bool(context["compaction_candidate_items"]), + ) + agent = Agent( + name="Packaged Responses compaction agent", + model=integration_model, + instructions="Remember user-provided release words and follow exact output instructions.", + model_settings=ModelSettings(store=store, max_tokens=192), + ) + config = RunConfig(tracing_disabled=True) + + try: + first = await Runner.run( + agent, + "Remember the release word JASPER and reply only STORED.", + session=compacted, + run_config=config, + ) + first_items = await underlying.get_items() + second: RunResult | RunResultStreaming + if streaming: + second = Runner.run_streamed( + agent, + "What release word did I give you? Reply with that word only.", + session=compacted, + run_config=config, + ) + async for _event in second.stream_events(): + pass + else: + second = await Runner.run( + agent, + "What release word did I give you? Reply with that word only.", + session=compacted, + run_config=config, + ) + second_items = await underlying.get_items() + finally: + underlying.close() + + assert first.final_output == "STORED" + assert second.final_output == "JASPER" + assert any(item.get("type") == "compaction" for item in first_items) + assert any(item.get("type") == "compaction" for item in second_items) diff --git a/integration_tests/openai/test_tracing.py b/integration_tests/openai/test_tracing.py new file mode 100644 index 0000000000..7212d185d1 --- /dev/null +++ b/integration_tests/openai/test_tracing.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from typing import Any, cast + +import pytest + +from agents import ( + Agent, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + Span, + Trace, + TracingProcessor, + set_trace_processors, + set_tracing_disabled, +) +from agents.decorators import tool +from agents.tracing import get_trace_provider + +pytestmark = [pytest.mark.core, pytest.mark.nightly] + + +class CollectingTraceProcessor(TracingProcessor): + def __init__(self) -> None: + self.started_traces: list[Trace] = [] + self.finished_traces: list[Trace] = [] + self.started_spans: list[Span[Any]] = [] + self.finished_spans: list[Span[Any]] = [] + + def on_trace_start(self, trace: Trace) -> None: + self.started_traces.append(trace) + + def on_trace_end(self, trace: Trace) -> None: + self.finished_traces.append(trace) + + def on_span_start(self, span: Span[Any]) -> None: + self.started_spans.append(span) + + def on_span_end(self, span: Span[Any]) -> None: + self.finished_spans.append(span) + + def shutdown(self) -> None: + return None + + def force_flush(self) -> None: + return None + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_live_model_and_tool_spans_finish_without_exposing_sensitive_data( + integration_model: str, monkeypatch: pytest.MonkeyPatch, streaming: bool +) -> None: + calls: list[str] = [] + + @tool + def inspect_secret(value: str) -> str: + """Inspect a deterministic sensitive verification value.""" + calls.append(value) + return "TRACE_READY" + + agent = Agent( + name="Packaged traced agent", + model=integration_model, + instructions=( + "Call inspect_secret with value='secret-token-42', then reply with exactly TRACE_READY." + ), + tools=[inspect_secret], + model_settings={"max_tokens": 384}, + ) + processor = CollectingTraceProcessor() + provider = cast(Any, get_trace_provider()) + original_processors = list(provider._multi_processor._processors) + original_env_disabled = provider._env_disabled + original_manual_disabled = provider._manual_disabled + original_disabled = provider._disabled + monkeypatch.setenv("OPENAI_AGENTS_DISABLE_TRACING", "0") + set_trace_processors([processor]) + set_tracing_disabled(False) + result: RunResult | RunResultStreaming + try: + config = RunConfig( + tracing_disabled=False, + trace_include_sensitive_data=False, + workflow_name="Packaged tracing compatibility", + ) + if streaming: + result = Runner.run_streamed(agent, "Inspect the secret.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Inspect the secret.", run_config=config) + finally: + set_trace_processors(original_processors) + provider._env_disabled = original_env_disabled + provider._manual_disabled = original_manual_disabled + provider._disabled = original_disabled + + assert calls == ["secret-token-42"] + assert result.final_output == "TRACE_READY" + assert len(processor.started_traces) == len(processor.finished_traces) == 1 + assert len(processor.started_spans) == len(processor.finished_spans) + span_types = {span.span_data.type for span in processor.finished_spans} + assert "agent" in span_types + assert "response" in span_types + assert "function" in span_types + assert all(span.ended_at is not None for span in processor.finished_spans) + assert all("secret-token-42" not in str(span.export()) for span in processor.finished_spans) diff --git a/integration_tests/openai/test_websocket.py b/integration_tests/openai/test_websocket.py new file mode 100644 index 0000000000..949d00bfdd --- /dev/null +++ b/integration_tests/openai/test_websocket.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import pytest + +from agents import ( + Agent, + ModelSettings, + ToolCallOutputItem, + responses_websocket_session, +) +from agents.decorators import tool +from agents.models.openai_responses import OpenAIResponsesWSModel + +pytestmark = [pytest.mark.core, pytest.mark.nightly] + + +async def test_responses_websocket_session_reuses_a_connection_across_tool_turns( + integration_model: str, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[str] = [] + opened_connections: list[Any] = [] + original_open = OpenAIResponsesWSModel._open_websocket_connection + + async def capture_connection( + model: OpenAIResponsesWSModel, + url: str, + headers: Mapping[str, str], + *, + connect_timeout: float | None, + ) -> Any: + connection = await original_open(model, url, headers, connect_timeout=connect_timeout) + opened_connections.append(connection) + return connection + + monkeypatch.setattr(OpenAIResponsesWSModel, "_open_websocket_connection", capture_connection) + + @tool + def lookup_checkpoint(name: str) -> str: + """Return a deterministic websocket checkpoint.""" + calls.append(name) + return "WEBSOCKET:42" + + agent = Agent( + name="Packaged Responses websocket agent", + model=integration_model, + instructions=( + "When asked to check a checkpoint, call lookup_checkpoint with name='release'. " + "For a confirmation request, reply exactly WEBSOCKET_CONFIRMED." + ), + tools=[lookup_checkpoint], + model_settings=ModelSettings(max_tokens=384), + ) + + async with responses_websocket_session() as session: + first = await session.run( + agent, + "Check the checkpoint and include WEBSOCKET:42 in your answer.", + ) + second = session.run_streamed(agent, "Reply with exactly WEBSOCKET_CONFIRMED.") + event_types = [event.type async for event in second.stream_events()] + + assert calls == ["release"] + assert "WEBSOCKET:42" in str(first.final_output) + assert any(isinstance(item, ToolCallOutputItem) for item in first.new_items) + assert second.final_output == "WEBSOCKET_CONFIRMED" + assert "raw_response_event" in event_types + assert first.context_wrapper.usage.total_tokens > 0 + assert second.context_wrapper.usage.total_tokens > 0 + assert len(opened_connections) == 1 diff --git a/integration_tests/packaging/test_distribution_contents.py b/integration_tests/packaging/test_distribution_contents.py new file mode 100644 index 0000000000..b965aac5b4 --- /dev/null +++ b/integration_tests/packaging/test_distribution_contents.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import importlib +import importlib.metadata +import importlib.util +import os +import sys +import tarfile +import warnings +import zipfile +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.packaging + + +def test_wheel_excludes_integration_tests_and_contains_runtime_modules() -> None: + wheel = Path(os.environ["OPENAI_AGENTS_INTEGRATION_WHEEL"]) + with zipfile.ZipFile(wheel) as archive: + members = archive.namelist() + + assert not any(Path(member).parts[0] == "integration_tests" for member in members) + assert "agents/py.typed" in members + assert "agents/realtime/session.py" in members + assert "agents/voice/pipeline.py" in members + assert "agents/extensions/models/any_llm_model.py" in members + assert "agents/extensions/models/litellm_model.py" in members + assert "agents/extensions/experimental/hosted_multi_agent/model.py" in members + + +def test_source_distribution_excludes_repository_automation_and_local_caches() -> None: + source_distribution = Path(os.environ["OPENAI_AGENTS_INTEGRATION_SDIST"]) + with tarfile.open(source_distribution, "r:gz") as archive: + members = archive.getnames() + + assert not any( + len(Path(member).parts) > 1 + and ( + Path(member).parts[1] in {".agents", ".github", "integration_tests"} + or Path(member).parts[1].startswith((".tmp", ".uv")) + ) + for member in members + ) + assert any(member.endswith("/src/agents/py.typed") for member in members) + + +def test_installed_distribution_advertises_expected_optional_extras() -> None: + distribution = importlib.metadata.distribution("openai-agents") + extras = set(distribution.metadata.get_all("Provides-Extra") or []) + + assert { + "any-llm", + "encrypt", + "litellm", + "realtime", + "redis", + "s3", + "sqlalchemy", + "viz", + "voice", + }.issubset(extras) + assert distribution.version + + +@pytest.mark.parametrize( + "module_name", + [ + "agents", + "agents.models.openai_responses", + "agents.models.openai_chatcompletions", + "agents.decorators", + "agents.guardrail", + "agents.handoffs", + "agents.memory", + "agents.model_settings", + "agents.realtime", + "agents.responses_websocket_session", + "agents.run", + "agents.run_config", + "agents.tool", + "agents.tool_guardrails", + "agents.tracing", + "agents.extensions.experimental.hosted_multi_agent", + ], +) +def test_public_runtime_modules_import_from_the_distribution(module_name: str) -> None: + module = importlib.import_module(module_name) + + assert module.__file__ is not None + assert "site-packages" in Path(module.__file__).parts + + +@pytest.mark.parametrize( + ("module_name", "export_name", "canonical_module", "canonical_name"), + [ + ("agents.decorators", "function_tool", "agents", "function_tool"), + ("agents.decorators", "tool", "agents", "function_tool"), + ("agents.decorators", "input_guardrail", "agents", "input_guardrail"), + ("agents.decorators", "output_guardrail", "agents", "output_guardrail"), + ("agents.decorators", "tool_input_guardrail", "agents", "tool_input_guardrail"), + ("agents.decorators", "tool_output_guardrail", "agents", "tool_output_guardrail"), + ("agents.agent", "Agent", "agents", "Agent"), + ("agents.run", "Runner", "agents", "Runner"), + ("agents.run_config", "RunConfig", "agents", "RunConfig"), + ("agents.model_settings", "ModelSettings", "agents", "ModelSettings"), + ("agents.guardrail", "input_guardrail", "agents", "input_guardrail"), + ("agents.tool", "function_tool", "agents", "function_tool"), + ("agents.tool_guardrails", "tool_input_guardrail", "agents", "tool_input_guardrail"), + ("agents.memory", "SQLiteSession", "agents", "SQLiteSession"), + ("agents.memory.sqlite_session", "SQLiteSession", "agents", "SQLiteSession"), + ( + "agents.responses_websocket_session", + "ResponsesWebSocketSession", + "agents", + "ResponsesWebSocketSession", + ), + ("agents.tracing", "TracingProcessor", "agents", "TracingProcessor"), + ( + "agents.realtime.model_events", + "RealtimeModelUsageEvent", + "agents.realtime", + "RealtimeModelUsageEvent", + ), + ], +) +def test_supported_import_paths_resolve_to_canonical_runtime_objects( + module_name: str, + export_name: str, + canonical_module: str, + canonical_name: str, +) -> None: + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always", DeprecationWarning) + module = importlib.import_module(module_name) + canonical = importlib.import_module(canonical_module) + actual = getattr(module, export_name) + + assert actual is getattr(canonical, canonical_name) + assert not any(isinstance(warning.message, DeprecationWarning) for warning in captured) + + +def test_decorators_module_exports_supported_runtime_aliases() -> None: + decorators = importlib.import_module("agents.decorators") + + assert decorators.__all__ == [ + "function_tool", + "input_guardrail", + "output_guardrail", + "tool", + "tool_input_guardrail", + "tool_output_guardrail", + ] + assert decorators.tool is decorators.function_tool + + def legacy_status() -> str: + """Return the supported legacy decorator status.""" + return "LEGACY_DECORATOR_READY" + + decorated_status = decorators.tool(legacy_status) + assert decorated_status.name == "legacy_status" + + +@pytest.mark.parametrize( + ("module_name", "dependency_name", "expected_extra"), + [ + ("agents.extensions.models.any_llm_model", "any_llm", "any-llm"), + ("agents.extensions.models.litellm_model", "litellm", "litellm"), + ], +) +def test_optional_provider_modules_fail_with_actionable_install_guidance( + module_name: str, dependency_name: str, expected_extra: str +) -> None: + if importlib.util.find_spec(dependency_name) is not None: + pytest.skip(f"{dependency_name} is already installed in this isolated environment.") + + sys.modules.pop(module_name, None) + with pytest.raises(ImportError, match=rf"openai-agents\[{expected_extra}\]"): + importlib.import_module(module_name) diff --git a/integration_tests/packaging/test_optional_extras.py b/integration_tests/packaging/test_optional_extras.py new file mode 100644 index 0000000000..45e586a5d3 --- /dev/null +++ b/integration_tests/packaging/test_optional_extras.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import importlib +import os +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.extras + + +def test_requested_optional_extra_imports_from_its_standalone_environment() -> None: + optional_extra = os.environ["OPENAI_AGENTS_INTEGRATION_EXTRA"] + module_names = { + "any-llm": "agents.extensions.models.any_llm_model", + "encrypt": "agents.extensions.memory.encrypt_session", + "litellm": "agents.extensions.models.litellm_model", + "realtime": "agents.realtime", + "redis": "agents.extensions.memory.redis_session", + "s3": "boto3", + "sqlalchemy": "agents.extensions.memory.sqlalchemy_session", + "viz": "agents.extensions.visualization", + "voice": "agents.voice", + } + module = importlib.import_module(module_names[optional_extra]) + + assert module.__file__ is not None + assert "site-packages" in Path(module.__file__).parts + + +@pytest.mark.parametrize( + ("optional_extra", "package_symbol", "module_name", "module_symbol"), + [ + ( + "encrypt", + "EncryptedSession", + "agents.extensions.memory.encrypt_session", + "EncryptedSession", + ), + ("redis", "RedisSession", "agents.extensions.memory.redis_session", "RedisSession"), + ( + "sqlalchemy", + "SQLAlchemySession", + "agents.extensions.memory.sqlalchemy_session", + "SQLAlchemySession", + ), + ], +) +def test_memory_extra_lazy_exports_resolve_to_the_installed_backend( + optional_extra: str, + package_symbol: str, + module_name: str, + module_symbol: str, +) -> None: + if os.environ["OPENAI_AGENTS_INTEGRATION_EXTRA"] != optional_extra: + pytest.skip(f"This environment does not include the {optional_extra} extra.") + + memory = importlib.import_module("agents.extensions.memory") + module = importlib.import_module(module_name) + + assert getattr(memory, package_symbol) is getattr(module, module_symbol) diff --git a/integration_tests/packaging/test_provider_selection.py b/integration_tests/packaging/test_provider_selection.py new file mode 100644 index 0000000000..496487414a --- /dev/null +++ b/integration_tests/packaging/test_provider_selection.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import runpy +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +import pytest +from conftest import _external_providers, pytest_runtest_setup + +pytestmark = pytest.mark.packaging + + +def _configure_provider_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-test-key") + monkeypatch.setenv("GEMINI_API_KEY", "gemini-test-key") + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_OPENROUTER_MODELS", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_ANTHROPIC_MODEL", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_GEMINI_MODEL", raising=False) + + +def test_external_provider_coverage_is_explicitly_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", raising=False) + + assert _external_providers() == [] + + +@pytest.mark.parametrize( + "credential_name", ["OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY"] +) +def test_external_provider_tests_do_not_require_an_openai_api_key( + monkeypatch: pytest.MonkeyPatch, + credential_name: str, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setenv(credential_name, "provider-test-key") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=["external_provider"], + callspec=SimpleNamespace(params={"external_provider": object()}), + get_closest_marker=lambda name: name == "providers", + ) + + pytest_runtest_setup(cast(pytest.Item, item)) + + +def test_openai_backed_provider_tests_require_an_openai_api_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setenv("OPENROUTER_API_KEY", "provider-test-key") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=["integration_model"], + get_closest_marker=lambda name: name == "providers", + ) + + with pytest.raises(pytest.fail.Exception, match="Set a real OPENAI_API_KEY"): + pytest_runtest_setup(cast(pytest.Item, item)) + + +@pytest.mark.parametrize( + ("fixture_name", "model_environment", "model_name", "credential_name"), + [ + ( + "any_llm_models", + "OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", + "openrouter/openai/gpt-5.6-luna", + "OPENROUTER_API_KEY", + ), + ( + "any_llm_models", + "OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", + "anthropic/claude-sonnet-5", + "ANTHROPIC_API_KEY", + ), + ( + "any_llm_models", + "OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", + "gemini/gemini-3.6-flash", + "GEMINI_API_KEY", + ), + ( + "litellm_models", + "OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS", + "openrouter/google/gemini-3.6-flash", + "OPENROUTER_API_KEY", + ), + ( + "litellm_models", + "OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS", + "gemini/gemini-3.6-flash", + "GOOGLE_API_KEY", + ), + ], +) +def test_configured_provider_models_use_provider_specific_credentials( + monkeypatch: pytest.MonkeyPatch, + fixture_name: str, + model_environment: str, + model_name: str, + credential_name: str, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setenv(model_environment, model_name) + monkeypatch.setenv(credential_name, "provider-test-key") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=[fixture_name], + get_closest_marker=lambda name: name == "providers", + ) + + pytest_runtest_setup(cast(pytest.Item, item)) + + +def test_configured_provider_models_require_their_own_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", "openrouter/openai/gpt-5.6-luna") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=["any_llm_models"], + get_closest_marker=lambda name: name == "providers", + ) + + with pytest.raises(pytest.fail.Exception, match="Set OPENROUTER_API_KEY"): + pytest_runtest_setup(cast(pytest.Item, item)) + + +def test_configured_openai_provider_rejects_placeholder_api_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "test_key") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS", "openai/gpt-4.1-mini") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=["litellm_models"], + get_closest_marker=lambda name: name == "providers", + ) + + with pytest.raises(pytest.fail.Exception, match="Set OPENAI_API_KEY"): + pytest_runtest_setup(cast(pytest.Item, item)) + + +def test_strict_mode_requires_requested_external_provider_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", raising=False) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "1") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + + assert _external_providers() == [] + item = SimpleNamespace( + fixturenames=["external_provider"], + callspec=SimpleNamespace(params={"external_provider": None}), + get_closest_marker=lambda name: name == "providers", + ) + + with pytest.raises(pytest.fail.Exception, match="External provider coverage requires"): + pytest_runtest_setup(cast(pytest.Item, item)) + + +@pytest.mark.parametrize( + ("model", "expected_extra"), + [ + ("anthropic/claude-sonnet-5", "anthropic"), + ("gemini/gemini-3.6-flash", "gemini"), + ("google/gemini-3.6-flash", "gemini"), + ("openrouter/openai/gpt-5.6-luna", "openrouter"), + ], +) +def test_configured_any_llm_models_install_provider_extras_without_external_matrix( + monkeypatch: pytest.MonkeyPatch, model: str, expected_extra: str +) -> None: + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", model) + runner_path = Path(__file__).resolve().parents[2] / ".github/scripts/run_integration_tests.py" + runner = runpy.run_path(str(runner_path)) + + assert runner["_any_llm_provider_extras"]( + external_providers_enabled=False, direct_providers_enabled=False + ) == [expected_extra] + + +def test_strict_mode_does_not_require_unrequested_external_providers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", raising=False) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + + assert _external_providers() == [] + + +def test_strict_mode_accepts_explicit_direct_provider_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "1") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", "1") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + + providers = _external_providers() + + assert [provider.name for provider in providers] == ["anthropic"] + + +def test_external_provider_coverage_defaults_to_current_openrouter_models( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "1") + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", raising=False) + + providers = _external_providers() + + assert [provider.name for provider in providers] == [ + "openrouter-openai-gpt-5.6-luna", + "openrouter-anthropic-claude-sonnet-5", + "openrouter-google-gemini-3.6-flash", + ] + assert [provider.model for provider in providers] == [ + "openrouter/openai/gpt-5.6-luna", + "openrouter/anthropic/claude-sonnet-5", + "openrouter/google/gemini-3.6-flash", + ] + + +def test_all_provider_coverage_adds_explicit_direct_provider_models( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "1") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", "1") + + providers = _external_providers() + + assert [provider.name for provider in providers] == [ + "openrouter-openai-gpt-5.6-luna", + "openrouter-anthropic-claude-sonnet-5", + "openrouter-google-gemini-3.6-flash", + "anthropic", + "gemini", + ] + assert [provider.model for provider in providers[-2:]] == [ + "anthropic/claude-sonnet-5", + "gemini/gemini-3.6-flash", + ] diff --git a/integration_tests/providers/test_any_llm.py b/integration_tests/providers/test_any_llm.py new file mode 100644 index 0000000000..5291032d29 --- /dev/null +++ b/integration_tests/providers/test_any_llm.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from agents import Agent, ModelSettings, RunConfig, Runner, RunResult, RunResultStreaming +from agents.decorators import tool + +pytestmark = pytest.mark.providers + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +async def test_any_llm_configured_providers_execute_real_function_tools( + any_llm_models: list[str], dictionary: bool +) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + calls: list[int] = [] + + @tool + def lookup_number(value: int) -> int: + """Return the supplied deterministic number.""" + calls.append(value) + return value + + for model_name in any_llm_models: + calls.clear() + settings: ModelSettings | dict[str, Any] + settings = {"max_tokens": 512} if dictionary else ModelSettings(max_tokens=512) + agent = Agent( + name="Packaged AnyLLM agent", + model=AnyLLMModel(model=model_name), + instructions="Call lookup_number with 42 and then reply exactly ANY_LLM:42.", + model_settings=settings, + tools=[lookup_number], + ) + result = await Runner.run( + agent, + "Use the number tool.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == [42], model_name + assert result.final_output == "ANY_LLM:42", model_name + assert result.context_wrapper.usage.total_tokens > 0, model_name + + +@pytest.mark.parametrize("api", ["responses", "chat_completions"]) +async def test_any_llm_openai_supports_both_api_families(integration_model: str, api: str) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + agent = Agent( + name="Packaged AnyLLM API selector", + model=AnyLLMModel(model=f"openai/{integration_model}", api=api), # type: ignore[arg-type] + instructions="Reply with exactly ANY_LLM_API_OK.", + model_settings={"max_tokens": 256}, + ) + result = await Runner.run( + agent, + "Confirm the selected provider API.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert result.final_output == "ANY_LLM_API_OK" + + +@pytest.mark.filterwarnings( + "ignore:Inheritance class AiohttpClientSession from ClientSession is discouraged:" + r"DeprecationWarning:google\.genai\._api_client" +) +async def test_any_llm_major_external_providers_execute_function_tools( + external_provider: Any, +) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + calls: list[str] = [] + + @tool + def provider_status(provider: str) -> str: + """Return the deterministic provider readiness status.""" + calls.append(provider) + return "ready" + + agent = Agent( + name="Packaged AnyLLM external provider agent", + model=AnyLLMModel(model=external_provider.model, api_key=external_provider.api_key), + instructions=( + "Call provider_status exactly once with provider='external', " + "then reply exactly PROVIDER_READY." + ), + model_settings={"max_tokens": 512}, + tools=[provider_status], + ) + result = await Runner.run( + agent, + "Check the provider with its function tool.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["external"] + assert result.final_output == "PROVIDER_READY" + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.nightly +@pytest.mark.filterwarnings( + "ignore:Inheritance class AiohttpClientSession from ClientSession is discouraged:" + r"DeprecationWarning:google\.genai\._api_client" +) +async def test_any_llm_external_provider_streams_function_tool_results( + external_provider: Any, +) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + calls: list[str] = [] + + @tool + def check_streaming_provider(provider: str) -> str: + """Return the provider's deterministic streaming readiness.""" + calls.append(provider) + return "ready" + + agent = Agent( + name="Packaged AnyLLM streaming external provider agent", + model=AnyLLMModel(model=external_provider.model, api_key=external_provider.api_key), + instructions=( + "Call check_streaming_provider exactly once with provider='external', " + "then reply exactly STREAMING_PROVIDER_READY." + ), + model_settings={"max_tokens": 512}, + tools=[check_streaming_provider], + ) + result = Runner.run_streamed( + agent, + "Check the streamed external provider function tool.", + run_config=RunConfig(tracing_disabled=True), + ) + event_types = [event.type async for event in result.stream_events()] + + assert calls == ["external"] + assert result.final_output == "STREAMING_PROVIDER_READY" + assert "raw_response_event" in event_types + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_any_llm_chat_completions_preserves_real_token_logprobs( + streaming: bool, +) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + agent = Agent( + name="Packaged AnyLLM token logprob agent", + model=AnyLLMModel(model="openai/gpt-4.1-mini", api="chat_completions"), + instructions="Reply with exactly BLUE.", + model_settings=ModelSettings(top_logprobs=2, max_tokens=32), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "What color is the sky? Reply BLUE.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "What color is the sky? Reply BLUE.", run_config=config) + + texts = [ + content + for response in result.raw_responses + for item in response.output + for content in getattr(item, "content", []) + if getattr(content, "type", None) == "output_text" + ] + assert result.final_output == "BLUE" + assert texts + assert any(getattr(content, "logprobs", None) for content in texts) diff --git a/integration_tests/providers/test_litellm.py b/integration_tests/providers/test_litellm.py new file mode 100644 index 0000000000..fd1d2d371b --- /dev/null +++ b/integration_tests/providers/test_litellm.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from agents import Agent, ModelSettings, RunConfig, Runner, RunResult, RunResultStreaming +from agents.decorators import tool + +pytestmark = pytest.mark.providers + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +async def test_litellm_configured_providers_execute_real_function_tools( + litellm_models: list[str], dictionary: bool +) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + calls: list[str] = [] + + @tool + def lookup_package(name: str) -> str: + """Return the deterministic package health.""" + calls.append(name) + return "healthy" + + for model_name in litellm_models: + calls.clear() + settings: ModelSettings | dict[str, Any] + values: dict[str, Any] = {"max_tokens": 512} + settings = values if dictionary else ModelSettings(**values) + agent = Agent( + name="Packaged LiteLLM agent", + model=LitellmModel(model=model_name), + instructions=( + "Call lookup_package with name='openai-agents', then reply exactly LITELLM_OK." + ), + model_settings=settings, + tools=[lookup_package], + ) + result = await Runner.run( + agent, + "Check the installed package.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["openai-agents"], model_name + assert result.final_output == "LITELLM_OK", model_name + assert result.context_wrapper.usage.total_tokens > 0, model_name + + +@pytest.mark.filterwarnings( + "ignore:Accessing the 'model_(computed_)?fields' attribute on the instance is deprecated:" + "pydantic.warnings.PydanticDeprecatedSince211:" + r"litellm\.litellm_core_utils\.model_response_utils" +) +async def test_litellm_streaming_preserves_real_provider_usage(integration_model: str) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + agent = Agent( + name="Packaged LiteLLM streaming agent", + model=LitellmModel(model=f"openai/{integration_model}"), + instructions="Reply with exactly LITELLM_STREAM_OK.", + model_settings={"max_tokens": 256}, + ) + result = Runner.run_streamed( + agent, + "Confirm the streaming provider path.", + run_config=RunConfig(tracing_disabled=True), + ) + async for _event in result.stream_events(): + pass + + assert result.final_output == "LITELLM_STREAM_OK" + assert result.context_wrapper.usage.total_tokens > 0 + + +async def test_litellm_major_external_providers_execute_function_tools( + external_provider: Any, +) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + calls: list[str] = [] + + @tool + def provider_status(provider: str) -> str: + """Return the deterministic provider readiness status.""" + calls.append(provider) + return "ready" + + agent = Agent( + name="Packaged LiteLLM external provider agent", + model=LitellmModel(model=external_provider.model, api_key=external_provider.api_key), + instructions=( + "Call provider_status exactly once with provider='external', " + "then reply exactly PROVIDER_READY." + ), + model_settings={"max_tokens": 512}, + tools=[provider_status], + ) + result = await Runner.run( + agent, + "Check the provider with its function tool.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["external"] + assert result.final_output == "PROVIDER_READY" + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.nightly +@pytest.mark.filterwarnings( + "ignore:Accessing the 'model_(computed_)?fields' attribute on the instance is deprecated:" + "pydantic.warnings.PydanticDeprecatedSince211:" + r"litellm\.litellm_core_utils\.model_response_utils" +) +async def test_litellm_external_provider_streams_function_tool_results( + external_provider: Any, +) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + calls: list[str] = [] + + @tool + def check_streaming_provider(provider: str) -> str: + """Return the provider's deterministic streaming readiness.""" + calls.append(provider) + return "ready" + + agent = Agent( + name="Packaged LiteLLM streaming external provider agent", + model=LitellmModel(model=external_provider.model, api_key=external_provider.api_key), + instructions=( + "Call check_streaming_provider exactly once with provider='external', " + "then reply exactly STREAMING_PROVIDER_READY." + ), + model_settings={"max_tokens": 512, "include_usage": True}, + tools=[check_streaming_provider], + ) + result = Runner.run_streamed( + agent, + "Check the streamed external provider function tool.", + run_config=RunConfig(tracing_disabled=True), + ) + event_types = [event.type async for event in result.stream_events()] + + assert calls == ["external"] + assert result.final_output == "STREAMING_PROVIDER_READY" + assert "raw_response_event" in event_types + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.filterwarnings( + "ignore:Accessing the 'model_(computed_)?fields' attribute on the instance is deprecated:" + "pydantic.warnings.PydanticDeprecatedSince211:" + r"litellm\.litellm_core_utils\.model_response_utils" +) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_litellm_preserves_real_token_logprobs( + streaming: bool, +) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + agent = Agent( + name="Packaged LiteLLM token logprob agent", + model=LitellmModel(model="openai/gpt-4.1-mini"), + instructions="Reply with exactly BLUE.", + model_settings=ModelSettings(top_logprobs=2, max_tokens=32), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "What color is the sky? Reply BLUE.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "What color is the sky? Reply BLUE.", run_config=config) + + texts = [ + content + for response in result.raw_responses + for item in response.output + for content in getattr(item, "content", []) + if getattr(content, "type", None) == "output_text" + ] + assert result.final_output == "BLUE" + assert texts + assert any(getattr(content, "logprobs", None) for content in texts) diff --git a/integration_tests/pytest.ini b/integration_tests/pytest.ini new file mode 100644 index 0000000000..ab59a65e6e --- /dev/null +++ b/integration_tests/pytest.ini @@ -0,0 +1,16 @@ +[pytest] +asyncio_mode = auto +asyncio_default_fixture_loop_scope = session +asyncio_default_test_loop_scope = session +timeout = 75 +testpaths = . +markers = + packaging: Distribution contents and installed-package boundaries. + extras: Independently installed optional dependency groups. + core: Live OpenAI Responses and Chat Completions coverage. + providers: Live AnyLLM and LiteLLM provider-adapter coverage. + realtime: Live OpenAI Realtime WebSocket coverage. + voice: Live OpenAI speech-to-text and text-to-speech coverage. + hosted: Live hosted MCP, multi-agent, and programmatic tool coverage. + nightly: Extended integration coverage selected by the nightly and manual profiles. + manual: Expensive or externally provisioned integration coverage selected manually. diff --git a/integration_tests/realtime/test_realtime.py b/integration_tests/realtime/test_realtime.py new file mode 100644 index 0000000000..daa7c4ffb2 --- /dev/null +++ b/integration_tests/realtime/test_realtime.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from agents import GuardrailFunctionOutput, ToolGuardrailFunctionOutput, ToolInputGuardrailData +from agents.decorators import output_guardrail, tool, tool_input_guardrail +from agents.realtime import ( + AssistantMessageItem, + AssistantText, + InputAudio, + RealtimeAgent, + RealtimeGuardrailTripped, + RealtimeHandoffEvent, + RealtimeHistoryAdded, + RealtimeHistoryUpdated, + RealtimeModelUsageEvent, + RealtimeRawModelEvent, + RealtimeRunner, + RealtimeToolApprovalRequired, + UserMessageItem, +) + +pytestmark = pytest.mark.realtime + + +async def test_realtime_text_session_completes_and_updates_history( + integration_realtime_model: str, +) -> None: + agent = RealtimeAgent( + name="Packaged realtime agent", + instructions="Reply with exactly REALTIME_READY.", + ) + runner = RealtimeRunner(agent) + observed_events: list[str] = [] + assistant_text: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("Confirm the realtime connection.") + + async def receive() -> None: + async for event in session: + observed_events.append(event.type) + if isinstance(event, RealtimeHistoryAdded | RealtimeHistoryUpdated): + items = ( + [event.item] if isinstance(event, RealtimeHistoryAdded) else event.history + ) + for item in items: + if not isinstance(item, AssistantMessageItem): + continue + assistant_text.extend( + content.text + for content in item.content + if isinstance(content, AssistantText) and content.text + ) + if event.type == "agent_end": + return + + await asyncio.wait_for(receive(), timeout=45) + + assert "agent_start" in observed_events + assert "agent_end" in observed_events + assert any("REALTIME_READY" in text for text in assistant_text) + + +async def test_realtime_function_tool_emits_start_and_end_events( + integration_realtime_model: str, +) -> None: + calls: list[str] = [] + + @tool + def lookup_city(city: str) -> str: + """Return a deterministic city status.""" + calls.append(city) + return "sunny" + + agent = RealtimeAgent( + name="Packaged realtime tool agent", + instructions="Call lookup_city with Tokyo, then reply with TOKYO_SUNNY.", + tools=[lookup_city], + ) + runner = RealtimeRunner(agent) + observed_events: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("What is the weather in Tokyo? Use the function tool.") + + async def receive() -> None: + async for event in session: + observed_events.append(event.type) + if event.type == "agent_end" and "tool_end" in observed_events: + return + + await asyncio.wait_for(receive(), timeout=60) + + assert calls == ["Tokyo"] + assert "tool_start" in observed_events + assert "tool_end" in observed_events + + +async def test_realtime_session_preserves_history_across_text_turns( + integration_realtime_model: str, +) -> None: + agent = RealtimeAgent( + name="Packaged realtime conversation agent", + instructions="Remember user-provided verification words and answer concisely.", + ) + runner = RealtimeRunner(agent) + assistant_text: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + + async def receive_turn() -> None: + async for event in session: + if isinstance(event, RealtimeHistoryAdded | RealtimeHistoryUpdated): + items = ( + [event.item] if isinstance(event, RealtimeHistoryAdded) else event.history + ) + for item in items: + if isinstance(item, AssistantMessageItem): + assistant_text.extend( + content.text + for content in item.content + if isinstance(content, AssistantText) and content.text + ) + if event.type == "agent_end": + return + + await session.send_message("Remember the verification word SIERRA. Reply only STORED.") + await asyncio.wait_for(receive_turn(), timeout=45) + await session.send_message("What was the verification word? Reply only with that word.") + await asyncio.wait_for(receive_turn(), timeout=45) + + assert any("SIERRA" in text.upper() for text in assistant_text) + + +async def test_realtime_usage_events_accumulate_once_per_completed_turn( + integration_realtime_model: str, +) -> None: + agent = RealtimeAgent( + name="Packaged realtime usage agent", + instructions="Reply with exactly REALTIME_USAGE_READY.", + ) + runner = RealtimeRunner(agent) + observed_usage: list[int] = [] + completed_totals: list[int] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + + async def receive_turn() -> None: + async for event in session: + if isinstance(event, RealtimeRawModelEvent) and isinstance( + event.data, RealtimeModelUsageEvent + ): + observed_usage.append(event.data.usage.total_tokens) + if event.type == "agent_end": + completed_totals.append(event.info.context.usage.total_tokens) + return + + await session.send_message("Confirm realtime usage for turn one.") + await asyncio.wait_for(receive_turn(), timeout=45) + await session.send_message("Confirm realtime usage for turn two.") + await asyncio.wait_for(receive_turn(), timeout=45) + + assert len(observed_usage) == 2 + assert all(value > 0 for value in observed_usage) + assert completed_totals == [observed_usage[0], sum(observed_usage)] + + +async def test_realtime_handoff_updates_the_active_agent( + integration_realtime_model: str, +) -> None: + specialist = RealtimeAgent( + name="Packaged realtime specialist", + instructions="Reply with exactly REALTIME_HANDOFF_READY.", + ) + coordinator = RealtimeAgent( + name="Packaged realtime coordinator", + instructions="Immediately transfer to the packaged realtime specialist.", + handoffs=[specialist], + ) + runner = RealtimeRunner(coordinator) + handoffs: list[RealtimeHandoffEvent] = [] + ended_agents: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("Transfer me to the specialist.") + + async def receive() -> None: + async for event in session: + if isinstance(event, RealtimeHandoffEvent): + handoffs.append(event) + if event.type == "agent_end": + ended_agents.append(event.agent.name) + if event.agent is specialist: + return + + await asyncio.wait_for(receive(), timeout=60) + + assert len(handoffs) == 1 + assert handoffs[0].from_agent is coordinator + assert handoffs[0].to_agent is specialist + assert specialist.name in ended_agents + + +async def test_realtime_update_agent_replaces_instructions_and_tool_dispatch( + integration_realtime_model: str, +) -> None: + calls: list[str] = [] + + @tool + def replacement_checkpoint(checkpoint: str) -> str: + """Resolve the replacement agent's release checkpoint.""" + calls.append(checkpoint) + return "REALTIME_UPDATED_READY" + + initial = RealtimeAgent( + name="Packaged initial realtime agent", + instructions="Reply only INITIAL_AGENT_ACTIVE.", + ) + replacement = RealtimeAgent( + name="Packaged replacement realtime agent", + instructions=( + "Call replacement_checkpoint with checkpoint='updated', " + "then reply exactly REALTIME_UPDATED_READY." + ), + tools=[replacement_checkpoint], + ) + runner = RealtimeRunner(initial, config={"async_tool_calls": False}) + ended_agents: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.update_agent(replacement) + await session.send_message( + "You must call replacement_checkpoint with checkpoint='updated'. " + "Do not reply before calling the function." + ) + + async def receive() -> None: + async for event in session: + if event.type == "agent_end": + ended_agents.append(event.agent.name) + if event.agent is replacement: + if not calls and len(ended_agents) == 1: + await session.send_message( + "Call replacement_checkpoint now with checkpoint='updated'." + ) + continue + return + + await asyncio.wait_for(receive(), timeout=60) + + assert calls == ["updated"] + assert ended_agents + assert all(name == replacement.name for name in ended_agents) + + +@pytest.mark.nightly +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +async def test_realtime_function_tool_approval_controls_side_effects( + integration_realtime_model: str, + approved: bool, +) -> None: + calls: list[str] = [] + approvals: list[str] = [] + + @tool(needs_approval=True) + def publish_checkpoint(checkpoint: str) -> str: + """Publish a release checkpoint only after approval.""" + calls.append(checkpoint) + return "REALTIME_APPROVED" + + agent = RealtimeAgent( + name="Packaged realtime approval agent", + instructions=( + "Call publish_checkpoint with checkpoint='release'. If it succeeds reply " + "REALTIME_APPROVED. If it is rejected reply REALTIME_REJECTED." + ), + tools=[publish_checkpoint], + ) + runner = RealtimeRunner(agent) + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("Publish the release checkpoint with the tool.") + + async def receive() -> None: + async for event in session: + if isinstance(event, RealtimeToolApprovalRequired): + approvals.append(event.call_id) + if approved: + await session.approve_tool_call(event.call_id) + else: + await session.reject_tool_call( + event.call_id, + rejection_message="Publishing the checkpoint was rejected.", + ) + if approved and event.type == "tool_end" and approvals: + return + if not approved and event.type == "agent_end" and approvals: + return + + await asyncio.wait_for(receive(), timeout=60) + + assert len(approvals) == 1 + assert calls == (["release"] if approved else []) + + +@pytest.mark.nightly +async def test_realtime_accepts_committed_pcm_audio_input( + integration_realtime_model: str, integration_pcm_audio: bytes +) -> None: + agent = RealtimeAgent( + name="Packaged realtime audio input agent", + instructions="Respond to the user's speech with exactly REALTIME_AUDIO_READY.", + ) + runner = RealtimeRunner(agent) + assistant_text: list[str] = [] + received_audio = False + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_audio(integration_pcm_audio, commit=True) + await session.send_message("Respond to the committed user audio.") + + async def receive() -> None: + nonlocal received_audio + async for event in session: + if isinstance(event, RealtimeHistoryAdded | RealtimeHistoryUpdated): + items = ( + [event.item] if isinstance(event, RealtimeHistoryAdded) else event.history + ) + for item in items: + if isinstance(item, UserMessageItem): + received_audio = received_audio or any( + isinstance(content, InputAudio) for content in item.content + ) + if isinstance(item, AssistantMessageItem): + assistant_text.extend( + content.text + for content in item.content + if isinstance(content, AssistantText) and content.text + ) + if event.type == "agent_end": + return + + await asyncio.wait_for(receive(), timeout=60) + + assert received_audio + assert any("REALTIME_AUDIO_READY" in text for text in assistant_text) + + +@pytest.mark.nightly +async def test_realtime_output_guardrails_interrupt_audio_transcripts( + integration_realtime_model: str, +) -> None: + inspected: list[str] = [] + + @output_guardrail + async def reject_release_output( + _context: object, _agent: object, text: str + ) -> GuardrailFunctionOutput: + inspected.append(text) + return GuardrailFunctionOutput(output_info={"blocked": True}, tripwire_triggered=True) + + agent = RealtimeAgent( + name="Packaged guarded realtime output agent", + instructions="Reply with exactly BLOCKED_RELEASE_CONTENT.", + output_guardrails=[reject_release_output], + ) + runner = RealtimeRunner( + agent, + config={ + "output_guardrails": [reject_release_output], + "guardrails_settings": {"debounce_text_length": 5}, + }, + ) + tripped: list[RealtimeGuardrailTripped] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["audio"], + } + } + ) as session: + await session.send_message("Return the blocked release content.") + + async def receive() -> None: + async for event in session: + if isinstance(event, RealtimeGuardrailTripped): + tripped.append(event) + return + + await asyncio.wait_for(receive(), timeout=45) + + assert len(tripped) == 1 + assert len(inspected) == 1 + assert tripped[0].message == inspected[0] + assert tripped[0].guardrail_results[0].output.tripwire_triggered + + +@pytest.mark.nightly +@pytest.mark.parametrize("pre_approval", [False, True], ids=["after-approval", "before-approval"]) +async def test_realtime_tool_input_guardrails_control_approval_and_execution( + integration_realtime_model: str, + pre_approval: bool, +) -> None: + approvals: list[str] = [] + calls: list[str] = [] + checks: list[str] = [] + + @tool_input_guardrail + def block_checkpoint(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + checks.append(data.context.tool_name) + return ToolGuardrailFunctionOutput.reject_content("Release execution was blocked.") + + @tool(needs_approval=True, tool_input_guardrails=[block_checkpoint]) + def guarded_checkpoint(value: str) -> str: + """Execute a release checkpoint only when its guardrail allows it.""" + calls.append(value) + return "CHECKPOINT_READY" + + agent = RealtimeAgent( + name="Packaged guarded realtime tool agent", + instructions=( + "Call guarded_checkpoint with value='release'. If blocked, reply exactly " + "REALTIME_TOOL_BLOCKED." + ), + tools=[guarded_checkpoint], + ) + runner = RealtimeRunner( + agent, + config={"tool_execution": {"pre_approval_tool_input_guardrails": pre_approval}}, + ) + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("Execute the protected release checkpoint.") + + async def receive() -> None: + async for event in session: + if isinstance(event, RealtimeToolApprovalRequired): + approvals.append(event.call_id) + await session.approve_tool_call(event.call_id) + if event.type == "agent_end" and checks: + return + + await asyncio.wait_for(receive(), timeout=60) + + assert calls == [] + assert checks == ["guarded_checkpoint"] + assert len(approvals) == (0 if pre_approval else 1) diff --git a/integration_tests/voice/test_voice_pipeline.py b/integration_tests/voice/test_voice_pipeline.py new file mode 100644 index 0000000000..60b4af9e4d --- /dev/null +++ b/integration_tests/voice/test_voice_pipeline.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from agents import Agent + +pytestmark = pytest.mark.voice + + +@pytest.mark.parametrize("audio_dtype", ["int16", "float32"]) +async def test_static_voice_pipeline_transcribes_and_synthesizes_without_audio_devices( + integration_model: str, + integration_pcm_audio: bytes, + audio_dtype: str, +) -> None: + import numpy as np + + from agents.voice import ( + AudioInput, + SingleAgentVoiceWorkflow, + SingleAgentWorkflowCallbacks, + VoicePipeline, + VoiceStreamEventAudio, + VoiceStreamEventLifecycle, + ) + + pcm_audio = np.frombuffer(integration_pcm_audio, dtype=np.int16).copy() + audio = ( + pcm_audio.astype(np.float32) / np.float32(32767.0) + if audio_dtype == "float32" + else pcm_audio + ) + original_audio = audio.copy() + transcriptions: list[str] = [] + + class RecordingWorkflowCallbacks(SingleAgentWorkflowCallbacks): + def on_run(self, workflow: SingleAgentVoiceWorkflow, transcription: str) -> None: + transcriptions.append(transcription) + + agent: Agent[Any] = Agent( + name="Packaged voice workflow agent", + model=integration_model, + instructions="Reply with exactly VOICE READY.", + model_settings={"max_tokens": 256}, + ) + pipeline = VoicePipeline( + workflow=SingleAgentVoiceWorkflow(agent, callbacks=RecordingWorkflowCallbacks()), + config={ + "tracing_disabled": True, + "stt_settings": {"language": "en"}, + "tts_settings": {"voice": "alloy"}, + }, + ) + result = await pipeline.run(AudioInput(buffer=audio)) + lifecycle: list[str] = [] + audio_chunks = 0 + async for event in result.stream(): + if isinstance(event, VoiceStreamEventLifecycle): + lifecycle.append(event.event) + elif isinstance(event, VoiceStreamEventAudio) and event.data is not None: + audio_chunks += 1 + + assert audio.size > 0 + np.testing.assert_array_equal(audio, original_audio) + assert len(transcriptions) == 1 + assert all(word in transcriptions[0].lower() for word in ("packaged", "voice", "ready")) + assert audio_chunks > 0 + assert lifecycle == ["turn_started", "turn_ended", "session_ended"] + + +@pytest.mark.nightly +@pytest.mark.parametrize("audio_dtype", ["int16", "float32"]) +async def test_streamed_voice_pipeline_transcribes_chunked_input_and_runs_a_function_tool( + integration_model: str, + integration_pcm_audio: bytes, + audio_dtype: str, +) -> None: + import numpy as np + from openai import AsyncOpenAI + + from agents.decorators import tool + from agents.voice import ( + AudioInput, + SingleAgentVoiceWorkflow, + StreamedAudioInput, + StreamedTranscriptionSession, + STTModel, + STTModelSettings, + VoicePipeline, + VoiceStreamEventAudio, + VoiceStreamEventLifecycle, + ) + + class BoundedTranscriptionSession(StreamedTranscriptionSession): + def __init__(self, audio_input: StreamedAudioInput, client: AsyncOpenAI) -> None: + self.audio_input = audio_input + self.client = client + self.closed = False + + async def transcribe_turns(self) -> AsyncIterator[str]: + buffers: list[Any] = [] + while True: + chunk = await self.audio_input.queue.get() + if chunk is None: + break + buffers.append(chunk) + response = await self.client.audio.transcriptions.create( + model="gpt-4o-mini-transcribe", + file=AudioInput(buffer=np.concatenate(buffers)).to_audio_file(), + ) + yield response.text + + async def close(self) -> None: + self.closed = True + + class BoundedLiveSTTModel(STTModel): + def __init__(self) -> None: + self.client = AsyncOpenAI() + self.session: BoundedTranscriptionSession | None = None + + @property + def model_name(self) -> str: + return "gpt-4o-mini-transcribe" + + async def transcribe( + self, + input: AudioInput, + settings: STTModelSettings, + trace_include_sensitive_data: bool, + trace_include_sensitive_audio_data: bool, + ) -> str: + del settings, trace_include_sensitive_data, trace_include_sensitive_audio_data + response = await self.client.audio.transcriptions.create( + model=self.model_name, + file=input.to_audio_file(), + ) + return response.text + + async def create_session( + self, + input: StreamedAudioInput, + settings: STTModelSettings, + trace_include_sensitive_data: bool, + trace_include_sensitive_audio_data: bool, + ) -> StreamedTranscriptionSession: + del settings, trace_include_sensitive_data, trace_include_sensitive_audio_data + self.session = BoundedTranscriptionSession(input, self.client) + return self.session + + calls: list[str] = [] + + @tool + def voice_status(value: str) -> str: + """Return a deterministic streamed voice readiness status.""" + calls.append(value) + return "ready" + + stt_model = BoundedLiveSTTModel() + agent: Agent[Any] = Agent( + name="Packaged streamed voice workflow agent", + model=integration_model, + instructions=( + "Call voice_status with value='streamed', then reply exactly STREAMED_VOICE_READY." + ), + tools=[voice_status], + model_settings={"max_tokens": 384}, + ) + pipeline = VoicePipeline( + workflow=SingleAgentVoiceWorkflow(agent), + stt_model=stt_model, + config={"tracing_disabled": True, "tts_settings": {"voice": "alloy"}}, + ) + streamed_input = StreamedAudioInput() + pcm_audio = np.frombuffer(integration_pcm_audio, dtype=np.int16).copy() + audio = ( + pcm_audio.astype(np.float32) / np.float32(32767.0) + if audio_dtype == "float32" + else pcm_audio + ) + original_audio = audio.copy() + midpoint = len(audio) // 2 + await streamed_input.add_audio(audio[:midpoint]) + await streamed_input.add_audio(audio[midpoint:]) + await streamed_input.add_audio(None) + + result = await pipeline.run(streamed_input) + lifecycle: list[str] = [] + audio_chunks = 0 + + async def consume() -> None: + nonlocal audio_chunks + async for event in result.stream(): + if isinstance(event, VoiceStreamEventLifecycle): + lifecycle.append(event.event) + elif isinstance(event, VoiceStreamEventAudio) and event.data is not None: + audio_chunks += 1 + + await asyncio.wait_for(consume(), timeout=65) + + assert calls == ["streamed"] + np.testing.assert_array_equal(audio, original_audio) + assert audio_chunks > 0 + assert lifecycle == ["turn_started", "turn_ended", "session_ended"] + assert stt_model.session is not None and stt_model.session.closed + + +async def test_voice_pipeline_surfaces_tts_failures_without_hanging( + integration_model: str, + integration_pcm_audio: bytes, +) -> None: + import numpy as np + + from agents.voice import ( + AudioInput, + SingleAgentVoiceWorkflow, + TTSModel, + TTSModelSettings, + VoicePipeline, + VoiceStreamEventLifecycle, + ) + from agents.voice.events import VoiceStreamEventError + + class FailingTTSModel(TTSModel): + @property + def model_name(self) -> str: + return "failing-packaged-tts" + + async def run(self, text: str, settings: TTSModelSettings) -> AsyncIterator[bytes]: + del text, settings + raise RuntimeError("Packaged TTS synthesis failed.") + yield b"" # pragma: no cover + + agent: Agent[Any] = Agent( + name="Packaged failing voice workflow agent", + model=integration_model, + instructions="Reply with exactly VOICE_FAILURE_READY.", + model_settings={"max_tokens": 128}, + ) + pipeline = VoicePipeline( + workflow=SingleAgentVoiceWorkflow(agent), + tts_model=FailingTTSModel(), + config={"tracing_disabled": True}, + ) + audio = np.frombuffer(integration_pcm_audio, dtype=np.int16).copy() + result = await pipeline.run(AudioInput(buffer=audio)) + observed: list[str] = [] + + async def consume() -> None: + async for event in result.stream(): + if isinstance(event, VoiceStreamEventLifecycle): + observed.append(event.event) + elif isinstance(event, VoiceStreamEventError): + observed.append("error") + + with pytest.raises(RuntimeError, match="Packaged TTS synthesis failed"): + await asyncio.wait_for(consume(), timeout=25) + + assert observed[0] == "turn_started" diff --git a/pyproject.toml b/pyproject.toml index 50ee61fb6f..ba8fe15171 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,15 @@ agents = { workspace = true } requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.build] +exclude = [ + "/.agents", + "/.github", + "/.tmp*", + "/.uv*", + "/integration_tests", +] + [tool.hatch.build.targets.wheel] packages = ["src/agents"] diff --git a/src/agents/__init__.py b/src/agents/__init__.py index a0ab5e94ac..5284386a46 100644 --- a/src/agents/__init__.py +++ b/src/agents/__init__.py @@ -303,7 +303,7 @@ def set_default_openai_responses_transport(transport: Literal["http", "websocket def set_default_openai_agent_registration( - config: OpenAIAgentRegistrationConfig | None, + config: OpenAIAgentRegistrationConfig | dict[str, Any] | None, ) -> None: """Set the default OpenAI agent registration config. diff --git a/src/agents/_config.py b/src/agents/_config.py index e5bdd3d0d7..846debd43b 100644 --- a/src/agents/_config.py +++ b/src/agents/_config.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Any, Literal from openai import AsyncOpenAI @@ -40,7 +40,7 @@ def set_default_openai_responses_transport(transport: Literal["http", "websocket def set_default_openai_agent_registration( - config: OpenAIAgentRegistrationConfig | None, + config: OpenAIAgentRegistrationConfig | dict[str, Any] | None, ) -> None: set_default_openai_agent_registration_config(config) diff --git a/src/agents/_config_coercion.py b/src/agents/_config_coercion.py new file mode 100644 index 0000000000..2d0722f3d6 --- /dev/null +++ b/src/agents/_config_coercion.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from dataclasses import fields, is_dataclass +from types import UnionType +from typing import Any, TypeVar, Union, cast, get_args, get_origin, get_type_hints + +from pydantic import AliasChoices, BaseModel + +ConfigT = TypeVar("ConfigT") +DataclassConfigT = TypeVar("DataclassConfigT") +PydanticConfigT = TypeVar("PydanticConfigT", bound=BaseModel) + + +def _declared_dataclass_type( + owner_type: type[Any], + field_name: str, + default_type: type[DataclassConfigT], +) -> type[DataclassConfigT]: + try: + annotation = get_type_hints(owner_type).get(field_name) + except (NameError, TypeError): + return default_type + + candidates = ( + get_args(annotation) if get_origin(annotation) in (Union, UnionType) else (annotation,) + ) + for candidate in candidates: + if ( + isinstance(candidate, type) + and is_dataclass(candidate) + and issubclass(candidate, default_type) + ): + return candidate + return default_type + + +def _dataclass_input_values( + value: dict[str, Any], + config_type: type[Any], +) -> dict[str, Any]: + field_names = {config_field.name for config_field in fields(config_type)} + return {name: field_value for name, field_value in value.items() if name in field_names} + + +def coerce_dataclass_config( + value: ConfigT | dict[str, Any], + config_type: type[ConfigT], + *, + parameter_name: str, +) -> ConfigT: + """Normalize an SDK-owned dataclass configuration at its public input boundary.""" + if isinstance(value, config_type): + return value + if not isinstance(value, dict): + raise TypeError( + f"{parameter_name} must be a {config_type.__name__} instance or a dict, " + f"got {type(value).__name__}" + ) + + field_names = { + config_field.name for config_field in fields(cast(Any, config_type)) if config_field.init + } + unknown_fields = sorted(str(name) for name in value if name not in field_names) + if unknown_fields: + raise TypeError(f"Unknown {parameter_name} settings: {', '.join(unknown_fields)}") + return config_type(**value) + + +def coerce_pydantic_config( + value: PydanticConfigT | dict[str, Any], + config_type: type[PydanticConfigT], + *, + parameter_name: str, +) -> PydanticConfigT: + """Normalize an SDK-owned Pydantic configuration using its declared extra policy.""" + if isinstance(value, config_type): + return value + if not isinstance(value, dict): + raise TypeError( + f"{parameter_name} must be a {config_type.__name__} instance or a dict, " + f"got {type(value).__name__}" + ) + + if config_type.model_config.get("extra") != "allow": + accepted_fields: set[str] = set(config_type.model_fields) + for field_info in config_type.model_fields.values(): + if isinstance(field_info.validation_alias, str): + accepted_fields.add(field_info.validation_alias) + elif isinstance(field_info.validation_alias, AliasChoices): + accepted_fields.update( + alias for alias in field_info.validation_alias.choices if isinstance(alias, str) + ) + unknown_fields = sorted(str(name) for name in value if name not in accepted_fields) + if unknown_fields: + raise TypeError(f"Unknown {parameter_name} settings: {', '.join(unknown_fields)}") + + return config_type.model_validate(value) diff --git a/src/agents/agent.py b/src/agents/agent.py index 917aea4d65..57005c0ff7 100644 --- a/src/agents/agent.py +++ b/src/agents/agent.py @@ -29,9 +29,9 @@ from .exceptions import ModelBehaviorError, UserError from .guardrail import InputGuardrail, OutputGuardrail from .handoffs import Handoff -from .logger import logger +from .logger import log_model_and_tool_action_error, logger from .mcp import MCPUtil -from .model_settings import ModelSettings +from .model_settings import ModelSettings, _coerce_model_settings, _declared_model_settings_type from .models.default_models import ( get_default_model_settings, ) @@ -317,6 +317,8 @@ class Agent(AgentBase, Generic[TContext]): model_settings: ModelSettings = field(default_factory=get_default_model_settings) """Configures model-specific tuning parameters (e.g. temperature, top_p). + + Accepts a ``ModelSettings`` instance or a dictionary containing its fields. """ input_guardrails: list[InputGuardrail[TContext]] = field(default_factory=list) @@ -368,6 +370,39 @@ class Agent(AgentBase, Generic[TContext]): """Whether to reset the tool choice to the default value after a tool has been called. Defaults to True. This ensures that the agent doesn't enter an infinite loop of tool usage.""" + if TYPE_CHECKING: + + def __init__( + self, + name: str, + handoff_description: str | None = None, + tools: list[Tool] = ..., + mcp_servers: list[MCPServer] = ..., + mcp_config: MCPConfig = ..., + instructions: ( + str + | Callable[ + [RunContextWrapper[TContext], Agent[TContext]], + MaybeAwaitable[str], + ] + | None + ) = None, + prompt: Prompt | DynamicPromptFunction | None = None, + handoffs: list[Agent[Any] | Handoff[TContext, Any]] = ..., + model: str | Model | None = None, + model_settings: ModelSettings | dict[str, Any] = ..., + input_guardrails: list[InputGuardrail[TContext]] = ..., + output_guardrails: list[OutputGuardrail[TContext]] = ..., + output_type: type[Any] | AgentOutputSchemaBase | None = None, + hooks: AgentHooks[TContext] | None = None, + tool_use_behavior: ( + Literal["run_llm_again", "stop_on_first_tool"] + | StopAtTools + | ToolsToFinalOutputFunction + ) = "run_llm_again", + reset_tool_choice: bool = True, + ) -> None: ... + def __post_init__(self): from typing import get_origin @@ -424,11 +459,11 @@ def __post_init__(self): f"Agent model must be a string, Model, or None, got {type(self.model).__name__}" ) - if not isinstance(self.model_settings, ModelSettings): - raise TypeError( - f"Agent model_settings must be a ModelSettings instance, " - f"got {type(self.model_settings).__name__}" - ) + self.model_settings = _coerce_model_settings( + self.model_settings, + parameter_name="Agent model_settings", + model_settings_type=_declared_model_settings_type(type(self), "model_settings"), + ) if self.model is not None and self.model_settings == get_default_model_settings(): self.model_settings = _initial_model_settings_for_model(self.model) @@ -503,6 +538,13 @@ def clone(self, **kwargs: Any) -> Agent[TContext]: and _model_settings_match_implicit_model_defaults(self.model, self.model_settings) ): kwargs["model_settings"] = _initial_model_settings_for_model(kwargs["model"]) + if "model_settings" in kwargs: + kwargs["model_settings"] = _coerce_model_settings( + kwargs["model_settings"], + parameter_name="Agent model_settings", + model_settings_type=type(self.model_settings), + inherited_model_settings=self.model_settings, + ) return dataclasses.replace(self, **kwargs) def as_tool( @@ -515,7 +557,7 @@ def as_tool( is_enabled: bool | Callable[[RunContextWrapper[Any], AgentBase[Any]], MaybeAwaitable[bool]] = True, on_stream: Callable[[AgentToolStreamEvent], MaybeAwaitable[None]] | None = None, - run_config: RunConfig | None = None, + run_config: RunConfig | dict[str, Any] | None = None, max_turns: int | None = None, hooks: RunHooks[TContext] | None = None, previous_response_id: str | None = None, @@ -558,6 +600,11 @@ def as_tool( include_input_schema: Whether to include the full JSON schema in structured input. """ + if run_config is not None: + from .run_config import _coerce_run_config + + run_config = _coerce_run_config(run_config) + def _is_supported_parameters(value: Any) -> bool: if not isinstance(value, type): return False @@ -807,10 +854,16 @@ async def _run_handler(payload: AgentToolStreamEvent) -> None: maybe_result = stream_handler(payload) if inspect.isawaitable(maybe_result): await maybe_result - except Exception: - logger.exception( - "Error while handling on_stream event for agent tool %s.", - self.name, + except Exception as exc: + + def diagnostic_extra() -> dict[str, object]: + return {"agent_name": self.name} + + log_model_and_tool_action_error( + logger, + "Error while handling an agent tool on_stream event", + exc, + diagnostic_extra=diagnostic_extra, ) async def dispatch_stream_events() -> None: diff --git a/src/agents/decorators.py b/src/agents/decorators.py new file mode 100644 index 0000000000..8f40054377 --- /dev/null +++ b/src/agents/decorators.py @@ -0,0 +1,19 @@ +"""Public decorators for defining Agents SDK components. + +`tool` is an alias for `function_tool`. +""" + +from .guardrail import input_guardrail, output_guardrail +from .tool import function_tool +from .tool_guardrails import tool_input_guardrail, tool_output_guardrail + +tool = function_tool + +__all__ = [ + "function_tool", + "input_guardrail", + "output_guardrail", + "tool", + "tool_input_guardrail", + "tool_output_guardrail", +] diff --git a/src/agents/extensions/experimental/codex/codex_tool.py b/src/agents/extensions/experimental/codex/codex_tool.py index 534245dbd1..2c252e3d00 100644 --- a/src/agents/extensions/experimental/codex/codex_tool.py +++ b/src/agents/extensions/experimental/codex/codex_tool.py @@ -17,7 +17,7 @@ from agents import _debug from agents.exceptions import ModelBehaviorError, UserError -from agents.logger import logger +from agents.logger import log_model_and_tool_action_error, log_tool_action_error, logger from agents.models import _openai_shared from agents.run_context import RunContextWrapper from agents.strict_schema import ensure_strict_json_schema @@ -952,8 +952,12 @@ def _try_store_thread_id_in_run_context_after_error( try: _store_thread_id_in_run_context(ctx, key, thread_id) - except Exception: - logger.exception("Failed to store Codex thread id in run context after error.") + except Exception as exc: + log_tool_action_error( + logger, + "Failed to store Codex thread id in run context after error", + exc, + ) def _set_pydantic_context_value(context: BaseModel, key: str, value: str) -> bool: @@ -1047,8 +1051,12 @@ async def _run_handler(payload: CodexToolStreamEvent) -> None: maybe_result = on_stream(payload) if inspect.isawaitable(maybe_result): await maybe_result - except Exception: - logger.exception("Error while handling Codex on_stream event.") + except Exception as exc: + log_model_and_tool_action_error( + logger, + "Error while handling Codex on_stream event", + exc, + ) async def _dispatch() -> None: assert event_queue is not None diff --git a/src/agents/extensions/memory/advanced_sqlite_session.py b/src/agents/extensions/memory/advanced_sqlite_session.py index 98a54e3123..822c123570 100644 --- a/src/agents/extensions/memory/advanced_sqlite_session.py +++ b/src/agents/extensions/memory/advanced_sqlite_session.py @@ -11,8 +11,14 @@ from agents.result import RunResult from agents.usage import Usage +from ... import _debug from ..._tool_identity import is_reserved_synthetic_tool_namespace, tool_qualified_name from ...items import TResponseInputItem +from ...logger import ( + log_model_action_error, + log_model_action_warning, + log_model_and_tool_action_error, +) from ...memory import SQLiteSession from ...memory.session_settings import SessionSettings, resolve_session_limit @@ -41,7 +47,7 @@ def __init__( db_path: str | Path = ":memory:", create_tables: bool = False, logger: logging.Logger | None = None, - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, **kwargs, ): """Initialize the AdvancedSQLiteSession. @@ -172,9 +178,11 @@ def _add_items_sync(): self._insert_items(conn, items) self._insert_structure_metadata(conn, items) conn.commit() - except Exception: + except Exception as exc: conn.rollback() - self._logger.exception("Failed to add items for session %s", self.session_id) + log_model_and_tool_action_error( + self._logger, "Failed to add session items", exc + ) raise await asyncio.to_thread(_add_items_sync) @@ -462,7 +470,16 @@ async def store_run_usage(self, result: RunResult) -> None: turn_anchor=turn_anchor, ) except Exception as e: - self._logger.error("Failed to store usage for session %s: %s", self.session_id, e) + + def diagnostic_extra() -> dict[str, object]: + return {"session_id": self.session_id} + + log_model_action_error( + self._logger, + "Failed to store session usage", + e, + diagnostic_extra=diagnostic_extra, + ) def _capture_current_turn(self) -> tuple[int, str, int | None]: """Return (current_turn, branch_id, turn_anchor) in one locked read. @@ -581,15 +598,19 @@ def _add_structure_sync(): try: await asyncio.to_thread(_add_structure_sync) - except Exception: - self._logger.exception( - "Failed to add structure metadata for session %s", self.session_id + except Exception as exc: + log_model_and_tool_action_error( + self._logger, + "Failed to add session structure metadata", + exc, ) # Try to clean up any orphaned messages to maintain consistency. try: await self._cleanup_orphaned_messages() - except Exception: - self._logger.exception("Failed to cleanup orphaned messages") + except Exception as cleanup_exc: + log_model_and_tool_action_error( + self._logger, "Failed to cleanup orphaned session messages", cleanup_exc + ) raise def _insert_structure_metadata( @@ -870,13 +891,21 @@ def _validate_turn(): old_branch = self._current_branch_id await asyncio.to_thread(self._commit_branch_pointer, branch_name, generation) - self._logger.debug( - "Created branch '%s' from turn %s ('%s') in '%s'", - branch_name, - turn_number, - turn_content, - old_branch, - ) + if _debug.DONT_LOG_MODEL_DATA: + self._logger.debug( + "Created branch '%s' from turn %s in '%s'", + branch_name, + turn_number, + old_branch, + ) + else: + self._logger.debug( + "Created branch '%s' from turn %s ('%s') in '%s'", + branch_name, + turn_number, + turn_content, + old_branch, + ) return branch_name async def create_branch_from_content( @@ -1580,7 +1609,9 @@ def _update_sync(): try: input_details_json = json.dumps(usage_data.input_tokens_details.__dict__) except (TypeError, ValueError) as e: - self._logger.warning("Failed to serialize input tokens details: %s", e) + log_model_action_warning( + self._logger, "Failed to serialize input token details", e + ) input_details_json = None if ( @@ -1590,7 +1621,9 @@ def _update_sync(): try: output_details_json = json.dumps(usage_data.output_tokens_details.__dict__) except (TypeError, ValueError) as e: - self._logger.warning("Failed to serialize output tokens details: %s", e) + log_model_action_warning( + self._logger, "Failed to serialize output token details", e + ) output_details_json = None with closing(conn.cursor()) as cursor: diff --git a/src/agents/extensions/memory/async_sqlite_session.py b/src/agents/extensions/memory/async_sqlite_session.py index 27a23b1cbe..63ae77081b 100644 --- a/src/agents/extensions/memory/async_sqlite_session.py +++ b/src/agents/extensions/memory/async_sqlite_session.py @@ -5,13 +5,17 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager from pathlib import Path -from typing import cast +from typing import Any, cast import aiosqlite from ...items import TResponseInputItem from ...memory import SessionABC -from ...memory.session_settings import SessionSettings, resolve_session_limit +from ...memory.session_settings import ( + SessionSettings, + coerce_session_settings, + resolve_session_limit, +) class AsyncSQLiteSession(SessionABC): @@ -30,7 +34,7 @@ def __init__( db_path: str | Path = ":memory:", sessions_table: str = "agent_sessions", messages_table: str = "agent_messages", - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, ): """Initialize the async SQLite session. @@ -44,7 +48,11 @@ def __init__( retrieving items. If None, uses default SessionSettings(). """ self.session_id = session_id - self.session_settings = session_settings or SessionSettings() + self.session_settings = ( + coerce_session_settings(session_settings) + if session_settings is not None + else SessionSettings() + ) self.db_path = db_path self.sessions_table = sessions_table self.messages_table = messages_table diff --git a/src/agents/extensions/memory/dapr_session.py b/src/agents/extensions/memory/dapr_session.py index 6ac68f6020..e923940f11 100644 --- a/src/agents/extensions/memory/dapr_session.py +++ b/src/agents/extensions/memory/dapr_session.py @@ -43,9 +43,13 @@ ) from ...items import TResponseInputItem -from ...logger import logger +from ...logger import log_model_and_tool_action_error, logger from ...memory.session import SessionABC -from ...memory.session_settings import SessionSettings, resolve_session_limit +from ...memory.session_settings import ( + SessionSettings, + coerce_session_settings, + resolve_session_limit, +) # Type alias for consistency levels ConsistencyLevel = Literal["eventual", "strong"] @@ -72,7 +76,7 @@ def __init__( dapr_client: DaprClient, ttl: int | None = None, consistency: ConsistencyLevel = DAPR_CONSISTENCY_EVENTUAL, - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, ): """Initializes a new DaprSession. @@ -90,7 +94,11 @@ def __init__( default limit for retrieving items. If None, uses default SessionSettings(). """ self.session_id = session_id - self.session_settings = session_settings or SessionSettings() + self.session_settings = ( + coerce_session_settings(session_settings) + if session_settings is not None + else SessionSettings() + ) self._dapr_client = dapr_client self._state_store_name = state_store_name self._ttl = ttl @@ -109,7 +117,7 @@ def from_address( *, state_store_name: str, dapr_address: str = "localhost:50001", - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, **kwargs: Any, ) -> DaprSession: """Create a session from a Dapr sidecar address. @@ -453,5 +461,5 @@ async def ping(self) -> bool: ) return True except Exception: - logger.error("Dapr connection failed: %s", initial_error) + log_model_and_tool_action_error(logger, "Dapr connection failed", initial_error) return False diff --git a/src/agents/extensions/memory/mongodb_session.py b/src/agents/extensions/memory/mongodb_session.py index 07354577d6..98f7f26008 100644 --- a/src/agents/extensions/memory/mongodb_session.py +++ b/src/agents/extensions/memory/mongodb_session.py @@ -60,7 +60,11 @@ from ...items import TResponseInputItem from ...memory.session import SessionABC -from ...memory.session_settings import SessionSettings, resolve_session_limit +from ...memory.session_settings import ( + SessionSettings, + coerce_session_settings, + resolve_session_limit, +) # Identifies this library in the MongoDB handshake for server-side telemetry. _DRIVER_INFO = DriverInfo(name="openai-agents", version=_VERSION) @@ -110,7 +114,7 @@ def __init__( database: str = "agents", sessions_collection: str = "agent_sessions", messages_collection: str = "agent_messages", - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, ): """Initialize a new MongoDBSession. @@ -128,7 +132,11 @@ def __init__( is used (no item limit). """ self.session_id = session_id - self.session_settings = session_settings or SessionSettings() + self.session_settings = ( + coerce_session_settings(session_settings) + if session_settings is not None + else SessionSettings() + ) self._client = client self._owns_client = False @@ -153,7 +161,7 @@ def from_uri( uri: str, database: str = "agents", client_kwargs: dict[str, Any] | None = None, - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, **kwargs: Any, ) -> MongoDBSession: """Create a session from a MongoDB URI string. diff --git a/src/agents/extensions/memory/redis_session.py b/src/agents/extensions/memory/redis_session.py index 11e2dd838b..3ad261b28e 100644 --- a/src/agents/extensions/memory/redis_session.py +++ b/src/agents/extensions/memory/redis_session.py @@ -41,7 +41,11 @@ from ...items import TResponseInputItem from ...memory.session import SessionABC -from ...memory.session_settings import SessionSettings, resolve_session_limit +from ...memory.session_settings import ( + SessionSettings, + coerce_session_settings, + resolve_session_limit, +) class RedisSession(SessionABC): @@ -56,7 +60,7 @@ def __init__( redis_client: Redis, key_prefix: str = "agents:session", ttl: int | None = None, - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, ): """Initializes a new RedisSession. @@ -71,7 +75,11 @@ def __init__( default limit for retrieving items. If None, uses default SessionSettings(). """ self.session_id = session_id - self.session_settings = session_settings or SessionSettings() + self.session_settings = ( + coerce_session_settings(session_settings) + if session_settings is not None + else SessionSettings() + ) self._redis = redis_client self._key_prefix = key_prefix self._ttl = ttl @@ -90,7 +98,7 @@ def from_url( *, url: str, redis_kwargs: dict[str, Any] | None = None, - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, **kwargs: Any, ) -> RedisSession: """Create a session from a Redis URL string. diff --git a/src/agents/extensions/memory/sqlalchemy_session.py b/src/agents/extensions/memory/sqlalchemy_session.py index 89467ad2d2..3fc793d328 100644 --- a/src/agents/extensions/memory/sqlalchemy_session.py +++ b/src/agents/extensions/memory/sqlalchemy_session.py @@ -50,7 +50,11 @@ from ...items import TResponseInputItem from ...memory.session import SessionABC -from ...memory.session_settings import SessionSettings, resolve_session_limit +from ...memory.session_settings import ( + SessionSettings, + coerce_session_settings, + resolve_session_limit, +) class SQLAlchemySession(SessionABC): @@ -135,7 +139,7 @@ def __init__( create_tables: bool = False, sessions_table: str = "agent_sessions", messages_table: str = "agent_messages", - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, ensure_ascii: bool = True, ): """Initializes a new SQLAlchemySession. @@ -155,7 +159,11 @@ def __init__( session items to JSON. Defaults to True to preserve the historical storage format. """ self.session_id = session_id - self.session_settings = session_settings or SessionSettings() + self.session_settings = ( + coerce_session_settings(session_settings) + if session_settings is not None + else SessionSettings() + ) self._engine = engine self._ensure_ascii = ensure_ascii self._configure_sqlite_engine(engine) @@ -225,7 +233,7 @@ def from_url( *, url: str, engine_kwargs: dict[str, Any] | None = None, - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, **kwargs: Any, ) -> SQLAlchemySession: """Create a session from a database URL string. diff --git a/src/agents/extensions/models/any_llm_model.py b/src/agents/extensions/models/any_llm_model.py index 95a0b86688..72f930dbab 100644 --- a/src/agents/extensions/models/any_llm_model.py +++ b/src/agents/extensions/models/any_llm_model.py @@ -731,6 +731,31 @@ async def _fetch_chat_response( extra_kwargs = self._build_chat_extra_kwargs(model_settings) extra_kwargs.pop("reasoning_effort", None) + headers = self._merge_headers(model_settings) + if self._provider_name in {"gemini", "vertexai"}: + http_options = extra_kwargs.get("http_options") + if isinstance(http_options, BaseModel): + existing_headers = getattr(http_options, "headers", None) or {} + extra_kwargs["http_options"] = http_options.model_copy( + update={"headers": {**existing_headers, **headers}} + ) + elif isinstance(http_options, dict): + existing_headers = http_options.get("headers") or {} + extra_kwargs["http_options"] = { + **http_options, + "headers": {**existing_headers, **headers}, + } + elif http_options is None: + extra_kwargs["http_options"] = {"headers": headers} + else: + extra_kwargs["extra_headers"] = headers + + # The Chat Completions API requires logprobs=True whenever top_logprobs is set. Defer to a + # caller-supplied logprobs (via extra_args, already merged into extra_kwargs) to avoid a + # duplicate-key collision. + if model_settings.top_logprobs is not None and "logprobs" not in extra_kwargs: + extra_kwargs["logprobs"] = True + ret = await self._get_provider().acompletion( model=self._provider_model, messages=converted_messages, @@ -747,7 +772,6 @@ async def _fetch_chat_response( stream_options=stream_options, reasoning_effort=reasoning_effort, top_logprobs=model_settings.top_logprobs, - extra_headers=self._merge_headers(model_settings), **extra_kwargs, ) @@ -953,6 +977,8 @@ def _get_provider(self) -> Any: api_key=self.api_key, api_base=self.base_url, ) + if self._provider_name in {"gemini", "vertexai"}: + self._normalize_google_tool_result_roles(base_provider) self._provider_cache[False] = base_provider if disable_provider_retries: @@ -962,6 +988,30 @@ def _get_provider(self) -> Any: return base_provider + @staticmethod + def _normalize_google_tool_result_roles(provider: Any) -> None: + convert_completion_params = getattr(provider, "_convert_completion_params", None) + if not callable(convert_completion_params): + return + + def convert_with_supported_tool_result_roles(*args: Any, **kwargs: Any) -> Any: + converted = convert_completion_params(*args, **kwargs) + contents = converted.get("contents") + if not isinstance(contents, list): + return converted + + converted["contents"] = [ + content.model_copy(update={"role": "user"}) + if isinstance(content, BaseModel) and getattr(content, "role", None) == "function" + else {**content, "role": "user"} + if isinstance(content, dict) and content.get("role") == "function" + else content + for content in contents + ] + return converted + + provider._convert_completion_params = convert_with_supported_tool_result_roles + def _clone_provider_without_retries(self, provider: Any) -> Any: client = getattr(provider, "client", None) with_options = getattr(client, "with_options", None) @@ -975,9 +1025,28 @@ def _clone_provider_without_retries(self, provider: Any) -> Any: def _normalize_response(self, response: Any) -> Response: if isinstance(response, Response): return response - if isinstance(response, BaseModel): - return Response.model_validate(response.model_dump()) - return Response.model_validate(response) + + payload = response.model_dump() if isinstance(response, BaseModel) else response + if isinstance(payload, dict): + usage = payload.get("usage") + if isinstance(usage, dict): + input_tokens_details = usage.get("input_tokens_details") + if ( + isinstance(input_tokens_details, dict) + and "cache_write_tokens" not in input_tokens_details + ): + payload = { + **payload, + "usage": { + **usage, + "input_tokens_details": { + **input_tokens_details, + "cache_write_tokens": 0, + }, + }, + } + + return Response.model_validate(payload) def _normalize_chat_completion_response(self, response: Any) -> ChatCompletion: if isinstance(response, ChatCompletion): diff --git a/src/agents/extensions/models/litellm_model.py b/src/agents/extensions/models/litellm_model.py index 0e1d3ae9cc..90eb81865b 100644 --- a/src/agents/extensions/models/litellm_model.py +++ b/src/agents/extensions/models/litellm_model.py @@ -324,12 +324,35 @@ async def get_response( else [] ) + # LiteLLM's Choices omits the logprobs attribute entirely when it was not requested, + # so access it defensively (mirrors the finish_reason handling above). + logprob_models = None + choice_logprobs = getattr(first_choice, "logprobs", None) if first_choice else None + if choice_logprobs is not None and getattr(choice_logprobs, "content", None): + logprob_models = ChatCmplHelpers.convert_logprobs_for_output_text( + choice_logprobs.content + ) + + if logprob_models: + self._attach_logprobs_to_output(items, logprob_models) + return ModelResponse( output=items, usage=usage, response_id=None, ) + def _attach_logprobs_to_output(self, output_items: list[Any], logprobs: list[Any]) -> None: + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + for output_item in output_items: + if not isinstance(output_item, ResponseOutputMessage): + continue + for content in output_item.content: + if isinstance(content, ResponseOutputText): + content.logprobs = logprobs + return + async def stream_response( self, system_instructions: str | None, @@ -540,9 +563,26 @@ async def _fetch_response( if model_settings.extra_args: extra_kwargs.update(model_settings.extra_args) + if converted_tools: + # SDK tools are already converted to ordinary function tools, so LiteLLM's proxy-only + # MCP discovery would add unsupported server dependencies without handling them. + extra_kwargs.setdefault("_skip_mcp_handler", True) + + if should_disable_provider_managed_retries(): + # Preserve provider-managed retries on the first attempt, but make runner retries the + # sole retry layer by forcing LiteLLM's retry knobs off on replay attempts. + extra_kwargs["num_retries"] = 0 + extra_kwargs["max_retries"] = 0 + # Prevent duplicate reasoning_effort kwargs when it was promoted to a top-level argument. extra_kwargs.pop("reasoning_effort", None) + # The Chat Completions API requires logprobs=True whenever top_logprobs is set. Defer to a + # caller-supplied logprobs (via extra_args, already merged into extra_kwargs) to avoid a + # duplicate-key collision. + if model_settings.top_logprobs is not None and "logprobs" not in extra_kwargs: + extra_kwargs["logprobs"] = True + ret = await litellm.acompletion( model=self.model, messages=converted_messages, diff --git a/src/agents/extensions/sandbox/__init__.py b/src/agents/extensions/sandbox/__init__.py index d7b082ba1f..ebf9e9aeeb 100644 --- a/src/agents/extensions/sandbox/__init__.py +++ b/src/agents/extensions/sandbox/__init__.py @@ -99,6 +99,7 @@ try: from .vercel import ( + VercelCloudBucketMountStrategy as VercelCloudBucketMountStrategy, VercelSandboxClient as VercelSandboxClient, VercelSandboxClientOptions as VercelSandboxClientOptions, VercelSandboxSession as VercelSandboxSession, @@ -180,6 +181,7 @@ if _HAS_VERCEL: __all__.extend( [ + "VercelCloudBucketMountStrategy", "VercelSandboxClient", "VercelSandboxClientOptions", "VercelSandboxSession", diff --git a/src/agents/extensions/sandbox/_rclone.py b/src/agents/extensions/sandbox/_rclone.py index e3d652db97..02327a0db3 100644 --- a/src/agents/extensions/sandbox/_rclone.py +++ b/src/agents/extensions/sandbox/_rclone.py @@ -6,13 +6,90 @@ _APT = "DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0" _RCLONE_CHECK = "command -v rclone >/dev/null 2>&1 || test -x /usr/local/bin/rclone" -_INSTALL_RCLONE_COMMANDS = ( +_RCLONE_CHECKSUM_MISMATCH_EXIT = 86 + +# BEGIN RCLONE RELEASE PIN +_RCLONE_VERSION = "1.74.4" +_RCLONE_SHA256_BY_ARCH = { + "386": "7feee086d7ff72652c5a91ef4b4a576941ccd33b2929772a2d70471904e516f0", + "amd64": "fe435e0c36228e7c2f116a8701f01127bb1f694005fc11d1f27186c8bca4115d", + "arm": "8135524b9b85111fa512f10a3fa191736a8d4d6ac3b3169af0763503744e95c9", + "arm-v6": "c9e1048feb597938884c0fff314d5d9a002599933cb94ce17fee19599cbfa3f1", + "arm-v7": "75844809d25d2534da96220727e7746a300e30ec8c676ca98c47affe5a752e7b", + "arm64": "97685285c9ad6a0cf17d5844115d2a67245af6444db672187074bd9c358de419", +} +# END RCLONE RELEASE PIN + +_INSTALL_RCLONE_PREREQUISITES = ( f"{_APT} update -qq", - f"{_APT} install -y -qq curl unzip ca-certificates", - "curl -fsSL https://rclone.org/install.sh | bash", + f"{_APT} install -y -qq ca-certificates coreutils curl unzip", ) +def _rclone_arch(machine: str) -> str | None: + normalized = machine.strip().lower() + if normalized in {"x86_64", "amd64"}: + return "amd64" + if normalized == "x86" or ( + len(normalized) == 4 + and normalized[0] == "i" + and normalized[1] in {"3", "4", "5", "6"} + and normalized[2:] == "86" + ): + return "386" + if normalized in {"aarch64", "arm64"}: + return "arm64" + if normalized.startswith("armv7"): + return "arm-v7" + if normalized.startswith("armv6"): + return "arm-v6" + if normalized.startswith("arm"): + return "arm" + return None + + +def _rclone_install_command(arch: str, sha256: str) -> str: + archive = f"rclone-v{_RCLONE_VERSION}-linux-{arch}.zip" + url = f"https://downloads.rclone.org/v{_RCLONE_VERSION}/{archive}" + return "\n".join( + [ + "set -eu", + 'tmp_dir="$(mktemp -d)"', + 'target_tmp=""', + "cleanup() {", + ' rm -rf "$tmp_dir"', + ' if [ -n "$target_tmp" ]; then rm -f "$target_tmp"; fi', + "}", + "trap cleanup EXIT", + "trap 'exit 1' HUP INT TERM", + f"archive='{archive}'", + f"expected_sha256='{sha256}'", + f"url='{url}'", + ( + "curl --fail --location --silent --show-error --proto '=https' " + '--tlsv1.2 --output "$tmp_dir/$archive" "$url"' + ), + ( + 'if ! printf \'%s %s\\n\' "$expected_sha256" "$tmp_dir/$archive" ' + "| sha256sum --check --strict -; then" + ), + f" exit {_RCLONE_CHECKSUM_MISMATCH_EXIT}", + "fi", + 'unzip -q "$tmp_dir/$archive" -d "$tmp_dir/unpacked"', + "install -d -m 0755 /usr/local/bin", + 'target_tmp="$(mktemp /usr/local/bin/.rclone.XXXXXX)"', + ('install -m 0755 "$tmp_dir/unpacked/${archive%.zip}/rclone" "$target_tmp"'), + 'version_output="$("$target_tmp" version)"', + ( + f"printf '%s\\n' \"$version_output\" | head -n 1 " + f"| grep -Fx 'rclone v{_RCLONE_VERSION}'" + ), + 'mv -f "$target_tmp" /usr/local/bin/rclone', + 'target_tmp=""', + ] + ) + + async def ensure_rclone(session: BaseSandboxSession) -> None: rclone = await session.exec("sh", "-lc", _RCLONE_CHECK, shell=False) if rclone.ok(): @@ -25,7 +102,16 @@ async def ensure_rclone(session: BaseSandboxSession) -> None: context={"package": "rclone"}, ) - for command in _INSTALL_RCLONE_COMMANDS: + machine_result = await session.exec("uname", "-m", shell=False, timeout=30) + machine = machine_result.stdout.decode("utf-8", errors="replace").strip() + arch = _rclone_arch(machine) if machine_result.ok() else None + if arch is None: + raise MountConfigError( + message="rclone is not installed and this architecture is unsupported", + context={"package": "rclone", "architecture": machine or "unknown"}, + ) + + for command in _INSTALL_RCLONE_PREREQUISITES: install = await session.exec( "sh", "-lc", @@ -40,11 +126,35 @@ async def ensure_rclone(session: BaseSandboxSession) -> None: context={"package": "rclone", "exit_code": install.exit_code}, ) + install = await session.exec( + "sh", + "-lc", + _rclone_install_command(arch, _RCLONE_SHA256_BY_ARCH[arch]), + shell=False, + timeout=300, + user="root", + ) + if install.exit_code == _RCLONE_CHECKSUM_MISMATCH_EXIT: + raise MountConfigError( + message="rclone archive checksum verification failed", + context={"package": "rclone", "version": _RCLONE_VERSION, "architecture": arch}, + ) + if not install.ok(): + raise MountConfigError( + message="failed to install rclone", + context={ + "package": "rclone", + "version": _RCLONE_VERSION, + "architecture": arch, + "exit_code": install.exit_code, + }, + ) + rclone = await session.exec("sh", "-lc", _RCLONE_CHECK, shell=False) if not rclone.ok(): raise MountConfigError( message="rclone was installed but is still not available on PATH", - context={"package": "rclone"}, + context={"package": "rclone", "version": _RCLONE_VERSION, "architecture": arch}, ) diff --git a/src/agents/extensions/sandbox/blaxel/mounts.py b/src/agents/extensions/sandbox/blaxel/mounts.py index 1476a1eb5d..d31a61edb0 100644 --- a/src/agents/extensions/sandbox/blaxel/mounts.py +++ b/src/agents/extensions/sandbox/blaxel/mounts.py @@ -25,6 +25,7 @@ from pathlib import Path from typing import Any, Literal +from ....logger import log_tool_action_warning from ....sandbox.entries import GCSMount, Mount, R2Mount, S3Mount from ....sandbox.entries.mounts.base import MountStrategyBase from ....sandbox.errors import MountConfigError @@ -668,7 +669,12 @@ async def _detach_drive(sandbox: Any, mount_path: str) -> None: try: await drives.unmount(mount_path) except Exception as e: - logger.warning("drive detach failed for %s (non-fatal): %s", mount_path, e) + log_tool_action_warning( + logger, + "Drive detach failed (non-fatal)", + e, + diagnostic_extra=lambda: {"mount_path": mount_path}, + ) __all__ = [ diff --git a/src/agents/extensions/sandbox/blaxel/sandbox.py b/src/agents/extensions/sandbox/blaxel/sandbox.py index 02c8e87b38..02d41a9712 100644 --- a/src/agents/extensions/sandbox/blaxel/sandbox.py +++ b/src/agents/extensions/sandbox/blaxel/sandbox.py @@ -29,6 +29,7 @@ from pydantic import BaseModel, Field +from ....logger import log_tool_action_debug, log_tool_action_warning from ....sandbox.entries import Mount from ....sandbox.errors import ( ExecTimeoutError, @@ -453,7 +454,9 @@ async def start(self) -> None: } ) except Exception as e: - logger.debug("workspace root mkdir failed (will retry during materialization): %s", e) + log_tool_action_debug( + logger, "Workspace root mkdir failed; retrying during materialization", e + ) await super().start() async def stop(self) -> None: @@ -467,7 +470,7 @@ async def shutdown(self) -> None: # When pause_on_exit is True the sandbox is kept alive. Blaxel # automatically resumes it on the next connection. except Exception as e: - logger.warning("sandbox delete failed during shutdown: %s", e) + log_tool_action_warning(logger, "Sandbox delete failed during shutdown", e) async def _validate_path_access(self, path: Path | str, *, for_write: bool = False) -> Path: return await self._validate_remote_path_access(path, for_write=for_write) @@ -626,7 +629,7 @@ async def running(self) -> bool: await asyncio.wait_for(self._sandbox.fs.ls("/"), timeout=10.0) return True except Exception as e: - logger.debug("sandbox health check failed: %s", e) + log_tool_action_debug(logger, "Sandbox health check failed", e) return False # -- workspace persistence ----------------------------------------------- @@ -690,7 +693,7 @@ async def persist_workspace(self) -> io.IOBase: "rm", "-f", "--", tar_path, timeout=self.state.timeouts.cleanup_s ) except Exception as e: - logger.debug("persist cleanup rm failed (non-fatal): %s", e) + log_tool_action_debug(logger, "Persist cleanup failed (non-fatal)", e) remount_error: WorkspaceArchiveReadError | None = None for mount_entry, mount_path in reversed(unmounted_mounts): @@ -764,7 +767,7 @@ async def hydrate_workspace(self, data: io.IOBase) -> None: "rm", "-f", "--", tar_path, timeout=self.state.timeouts.cleanup_s ) except Exception as e: - logger.debug("hydrate cleanup rm failed (non-fatal): %s", e) + log_tool_action_debug(logger, "Hydrate cleanup failed (non-fatal)", e) # -- PTY ----------------------------------------------------------------- @@ -947,7 +950,7 @@ async def _pty_ws_reader(self, entry: _BlaxelPtySessionEntry) -> None: ): break except Exception as e: - logger.debug("PTY ws reader terminated with error: %s", e) + log_tool_action_debug(logger, "PTY WebSocket reader terminated with an error", e) finally: entry.done = True entry.output_notify.set() @@ -1018,14 +1021,14 @@ async def _terminate_pty_entry(self, entry: _BlaxelPtySessionEntry) -> None: try: await entry.ws.close() except Exception as e: - logger.debug("PTY ws close error (non-fatal): %s", e) + log_tool_action_debug(logger, "PTY WebSocket close failed (non-fatal)", e) if entry.http_session is not None: try: await entry.http_session.close() except Exception as e: - logger.debug("PTY http session close error (non-fatal): %s", e) + log_tool_action_debug(logger, "PTY HTTP session close failed (non-fatal)", e) except Exception as e: - logger.debug("PTY entry termination error (non-fatal): %s", e) + log_tool_action_debug(logger, "PTY entry termination failed (non-fatal)", e) # --------------------------------------------------------------------------- @@ -1126,7 +1129,7 @@ async def delete(self, session: SandboxSession) -> SandboxSession: try: await inner.shutdown() except Exception as e: - logger.warning("shutdown error during delete (non-fatal): %s", e) + log_tool_action_warning(logger, "Shutdown failed during delete (non-fatal)", e) return session async def resume( @@ -1152,7 +1155,7 @@ async def resume( blaxel_sandbox = await SandboxInstance.get(state.sandbox_name) reconnected = True except Exception as e: - logger.debug("sandbox get() failed, will recreate: %s", e) + log_tool_action_debug(logger, "Sandbox lookup failed; recreating", e) if not reconnected or blaxel_sandbox is None: create_config = _build_create_config( diff --git a/src/agents/extensions/sandbox/cloudflare/sandbox.py b/src/agents/extensions/sandbox/cloudflare/sandbox.py index a3f94ec591..ab492ff823 100644 --- a/src/agents/extensions/sandbox/cloudflare/sandbox.py +++ b/src/agents/extensions/sandbox/cloudflare/sandbox.py @@ -29,6 +29,8 @@ import aiohttp +from .... import _debug +from ....logger import log_tool_action_debug from ....sandbox.errors import ( ConfigurationError, ErrorCode, @@ -693,13 +695,16 @@ async def _shutdown_backend(self) -> None: async with http.delete(url) as resp: if resp.status < 400 or resp.status == 404: return - detail = await _read_cloudflare_response_body(resp) - logger.debug( - "Failed to delete Cloudflare sandbox on shutdown: %s", - _cloudflare_http_error_message("DELETE /sandbox", resp.status, detail), - ) - except Exception: - logger.debug("Failed to delete Cloudflare sandbox on shutdown", exc_info=True) + if _debug.DONT_LOG_TOOL_DATA: + logger.debug("Failed to delete Cloudflare sandbox on shutdown") + else: + detail = await _read_cloudflare_response_body(resp) + logger.debug( + "Failed to delete Cloudflare sandbox on shutdown: %s", + _cloudflare_http_error_message("DELETE /sandbox", resp.status, detail), + ) + except Exception as exc: + log_tool_action_debug(logger, "Failed to delete Cloudflare sandbox on shutdown", exc) async def _after_shutdown(self) -> None: await self._close_http() @@ -846,7 +851,10 @@ async def _pump_ws_output(self, entry: _CloudflarePtyProcessEntry) -> None: try: payload = json.loads(msg.data) except json.JSONDecodeError: - logger.debug("Ignoring non-JSON PTY text frame: %s", msg.data) + if _debug.DONT_LOG_TOOL_DATA: + logger.debug("Ignoring non-JSON PTY text frame") + else: + logger.debug("Ignoring non-JSON PTY text frame: %s", msg.data) continue msg_type = payload.get("type") @@ -859,7 +867,10 @@ async def _pump_ws_output(self, entry: _CloudflarePtyProcessEntry) -> None: entry.output_notify.set() break if msg_type == "error": - logger.warning("Cloudflare PTY error frame: %s", payload.get("message")) + if _debug.DONT_LOG_TOOL_DATA: + logger.warning("Cloudflare PTY error frame") + else: + logger.warning("Cloudflare PTY error frame: %s", payload.get("message")) entry.output_closed.set() entry.output_notify.set() break @@ -875,8 +886,8 @@ async def _pump_ws_output(self, entry: _CloudflarePtyProcessEntry) -> None: break except asyncio.CancelledError: raise - except Exception: - logger.debug("Cloudflare PTY pump ended with an exception", exc_info=True) + except Exception as exc: + log_tool_action_debug(logger, "Cloudflare PTY pump ended with an exception", exc) entry.output_closed.set() entry.output_notify.set() diff --git a/src/agents/extensions/sandbox/daytona/sandbox.py b/src/agents/extensions/sandbox/daytona/sandbox.py index 294afc6cf3..988d5a2778 100644 --- a/src/agents/extensions/sandbox/daytona/sandbox.py +++ b/src/agents/extensions/sandbox/daytona/sandbox.py @@ -26,6 +26,7 @@ from pydantic import BaseModel, Field +from ....logger import log_tool_action_debug from ....sandbox.entries import Mount from ....sandbox.errors import ( ExecTimeoutError, @@ -1335,7 +1336,7 @@ async def resume( await daytona_sandbox.start(timeout=state.start_timeout) reconnected = True except Exception as e: - logger.debug("daytona sandbox get() failed, will recreate: %s", e) + log_tool_action_debug(logger, "Daytona sandbox lookup failed; recreating", e) if not reconnected or daytona_sandbox is None: params = await self._build_create_params( diff --git a/src/agents/extensions/sandbox/e2b/sandbox.py b/src/agents/extensions/sandbox/e2b/sandbox.py index 324d08bfdf..ecbe8bc0bf 100644 --- a/src/agents/extensions/sandbox/e2b/sandbox.py +++ b/src/agents/extensions/sandbox/e2b/sandbox.py @@ -34,6 +34,7 @@ from pydantic import BaseModel, Field +from ....logger import log_tool_action_warning from ....sandbox.entries import Mount from ....sandbox.errors import ( ExecNonZeroError, @@ -858,6 +859,12 @@ async def _after_start_failed(self) -> None: async def _shutdown_backend(self) -> None: # Best-effort kill of the remote sandbox. + def diagnostic_extra() -> dict[str, object]: + return { + "sandbox_id": self.state.sandbox_id, + "pause_on_exit": self.state.pause_on_exit, + } + try: if self.state.pause_on_exit: await _sandbox_pause(self._sandbox) @@ -865,33 +872,27 @@ async def _shutdown_backend(self) -> None: await _sandbox_kill(self._sandbox) except Exception as e: if self.state.pause_on_exit: - logger.warning( + log_tool_action_warning( + logger, "Failed to pause E2B sandbox on shutdown; falling back to kill.", - extra={ - "sandbox_id": self.state.sandbox_id, - "pause_on_exit": self.state.pause_on_exit, - }, - exc_info=e, + e, + diagnostic_extra=diagnostic_extra, ) try: await _sandbox_kill(self._sandbox) except Exception as kill_exc: - logger.warning( + log_tool_action_warning( + logger, "Failed to kill E2B sandbox after pause fallback failure.", - extra={ - "sandbox_id": self.state.sandbox_id, - "pause_on_exit": self.state.pause_on_exit, - }, - exc_info=kill_exc, + kill_exc, + diagnostic_extra=diagnostic_extra, ) else: - logger.warning( + log_tool_action_warning( + logger, "Failed to kill E2B sandbox on shutdown.", - extra={ - "sandbox_id": self.state.sandbox_id, - "pause_on_exit": self.state.pause_on_exit, - }, - exc_info=e, + e, + diagnostic_extra=diagnostic_extra, ) async def _exec_internal( diff --git a/src/agents/extensions/sandbox/modal/sandbox.py b/src/agents/extensions/sandbox/modal/sandbox.py index da2bd2939c..b70f840119 100644 --- a/src/agents/extensions/sandbox/modal/sandbox.py +++ b/src/agents/extensions/sandbox/modal/sandbox.py @@ -33,6 +33,7 @@ from modal.config import config as modal_config from modal.container_process import ContainerProcess +from ....logger import log_tool_action_warning from ....sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE from ....sandbox.entries import Mount from ....sandbox.errors import ( @@ -1327,8 +1328,9 @@ async def restore_ephemeral_paths() -> WorkspaceArchiveReadError | None: if not rm_out.ok(): cleanup_restore_error = await restore_ephemeral_paths() if cleanup_restore_error is not None: - logger.warning( - "Failed to restore Modal ephemeral paths after cleanup failure: %s", + log_tool_action_warning( + logger, + "Failed to restore Modal ephemeral paths after cleanup failure", cleanup_restore_error, ) raise WorkspaceArchiveReadError( @@ -1352,8 +1354,9 @@ async def restore_ephemeral_paths() -> WorkspaceArchiveReadError | None: except Exception as e: restore_error = await restore_ephemeral_paths() if restore_error is not None: - logger.warning( - "Failed to restore Modal ephemeral paths after snapshot failure: %s", + log_tool_action_warning( + logger, + "Failed to restore Modal ephemeral paths after snapshot failure", restore_error, ) raise WorkspaceArchiveReadError( @@ -1652,7 +1655,11 @@ async def _persist_workspace_via_tar(self) -> io.IOBase: excludes: list[str] = [] for rel in sorted(skip, key=lambda p: p.as_posix()): - excludes.extend(["--exclude", f"./{rel.as_posix().lstrip('./')}"]) + # Strip a leading "./" prefix only. `lstrip("./")` strips a *set* of + # characters, which would eat leading dots of dot-prefixed skip paths + # (e.g. ".venv" -> "venv"), producing a wrong exclude pattern. Match the + # Cloudflare backend, which uses removeprefix here. + excludes.extend(["--exclude", f"./{rel.as_posix().removeprefix('./')}"]) cmd: list[str] = [ "tar", diff --git a/src/agents/extensions/sandbox/vercel/__init__.py b/src/agents/extensions/sandbox/vercel/__init__.py index fd525ae62f..7861f58e80 100644 --- a/src/agents/extensions/sandbox/vercel/__init__.py +++ b/src/agents/extensions/sandbox/vercel/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +from .mounts import VercelCloudBucketMountStrategy from .sandbox import ( VercelSandboxClient, VercelSandboxClientOptions, @@ -8,6 +9,7 @@ ) __all__ = [ + "VercelCloudBucketMountStrategy", "VercelSandboxClient", "VercelSandboxClientOptions", "VercelSandboxSession", diff --git a/src/agents/extensions/sandbox/vercel/mounts.py b/src/agents/extensions/sandbox/vercel/mounts.py new file mode 100644 index 0000000000..b11954f112 --- /dev/null +++ b/src/agents/extensions/sandbox/vercel/mounts.py @@ -0,0 +1,583 @@ +"""Create-time-only S3 mounts for Vercel sandboxes.""" + +from __future__ import annotations + +import asyncio +import shlex +from pathlib import Path +from typing import Literal, NoReturn + +from ....sandbox.entries import Mount, S3Mount +from ....sandbox.entries.mounts.base import MountStrategyBase +from ....sandbox.errors import MountCommandError, MountConfigError +from ....sandbox.materialization import MaterializedFile +from ....sandbox.session.base_sandbox_session import BaseSandboxSession +from ....sandbox.session.runtime_helpers import RESOLVE_WORKSPACE_PATH_HELPER +from ....sandbox.types import ExecResult +from ....sandbox.workspace_paths import sandbox_path_str +from .sandbox import VercelSandboxSession + +_MOUNTPOINT_BINARY = "/usr/bin/mount-s3" +_MOUNTPOINT_PACKAGE = "mount-s3" +_MOUNTPOINT_SOURCE = "mountpoint-s3" +_MOUNTPOINT_MINIMUM_VERSION = (1, 21, 0) +_MOUNTPOINT_INSTALL_TIMEOUT_S = 300.0 +_MOUNTPOINT_COMMAND_TIMEOUT_S = 120.0 + + +def _require_vercel_session(session: BaseSandboxSession) -> VercelSandboxSession: + if not isinstance(session, VercelSandboxSession): + raise MountConfigError( + message=( + "Vercel S3 mount topology is fixed when the sandbox is created; " + "dynamic manifest application is not supported" + ), + context={"backend": "vercel", "session_type": type(session).__name__}, + ) + return session + + +def _redact_sensitive_values(text: str, values: tuple[str, ...]) -> str: + redacted = text + for value in sorted({value for value in values if value}, key=len, reverse=True): + redacted = redacted.replace(value, "REDACTED") + return redacted + + +async def _run_vercel_command( + session: VercelSandboxSession, + command: str, + args: list[str], + *, + sudo: bool = False, + timeout: float = _MOUNTPOINT_COMMAND_TIMEOUT_S, +) -> ExecResult: + command_text = shlex.join([command, *args]) + try: + sandbox = await session._ensure_sandbox() + + async def run_and_collect_output() -> ExecResult: + finished = await sandbox.run_command( + command, + args, + sudo=sudo, + ) + stdout = (await finished.stdout()).encode("utf-8") + stderr = (await finished.stderr()).encode("utf-8") + return ExecResult(stdout=stdout, stderr=stderr, exit_code=finished.exit_code) + + return await asyncio.wait_for(run_and_collect_output(), timeout=timeout) + except Exception as exc: + raise MountCommandError( + command=command_text, + stderr=f"{type(exc).__name__}: {exc}", + context={"backend": "vercel"}, + retryable=session._runtime_provider_retryability(exc), + ) from None + + +def _raise_command_failure( + command: str, + args: list[str], + result: ExecResult, + *, + context: dict[str, object] | None = None, +) -> NoReturn: + raise MountCommandError( + command=shlex.join([command, *args]), + stderr=result.stderr.decode("utf-8", errors="replace"), + context={ + "backend": "vercel", + "exit_code": result.exit_code, + **(context or {}), + }, + ) + + +async def _run_required_command( + session: VercelSandboxSession, + command: str, + args: list[str], + *, + sudo: bool = False, + timeout: float = _MOUNTPOINT_COMMAND_TIMEOUT_S, + context: dict[str, object] | None = None, +) -> ExecResult: + result = await _run_vercel_command( + session, + command, + args, + sudo=sudo, + timeout=timeout, + ) + if not result.ok(): + _raise_command_failure(command, args, result, context=context) + return result + + +async def _run_credentialed_mount_command( + session: VercelSandboxSession, + mount_path: Path, + args: list[str], + *, + context: dict[str, object], +) -> ExecResult | MountCommandError | asyncio.CancelledError: + env = session._runtime_s3_mount_environment(mount_path) + sensitive_values = tuple(env.values()) + command_text = shlex.join([_MOUNTPOINT_BINARY, *args]) + try: + sandbox = await session._ensure_sandbox() + + async def run_and_collect_output() -> ExecResult: + finished = await sandbox.run_command( + _MOUNTPOINT_BINARY, + args, + env=env, + sudo=True, + ) + stdout = (await finished.stdout()).encode("utf-8") + stderr = (await finished.stderr()).encode("utf-8") + return ExecResult(stdout=stdout, stderr=stderr, exit_code=finished.exit_code) + + result = await asyncio.wait_for( + run_and_collect_output(), + timeout=_MOUNTPOINT_COMMAND_TIMEOUT_S, + ) + except (Exception, asyncio.CancelledError) as exc: + cancelled = isinstance(exc, asyncio.CancelledError) + retryable = session._runtime_provider_retryability(exc) + failure_message = _redact_sensitive_values( + f"{type(exc).__name__}: {exc}", + sensitive_values, + ) + exc.__traceback__ = None + exc.__context__ = None + exc.__cause__ = None + if cancelled: + return asyncio.CancelledError() + return MountCommandError( + command=command_text, + stderr=failure_message, + context={"backend": "vercel", **context}, + retryable=retryable, + ) + + if result.ok(): + return result + + failure_message = _redact_sensitive_values( + result.stderr.decode("utf-8", errors="replace"), + sensitive_values, + ) + return MountCommandError( + command=command_text, + stderr=failure_message, + context={ + "backend": "vercel", + "exit_code": result.exit_code, + **context, + }, + ) + + +def _parse_mountpoint_version(raw: str) -> tuple[int, int, int] | None: + parts = raw.strip().split(".") + if len(parts) != 3 or not all(part.isdecimal() for part in parts): + return None + return int(parts[0]), int(parts[1]), int(parts[2]) + + +async def _ensure_mountpoint(session: VercelSandboxSession) -> None: + version_args = ["--query", "--queryformat", "%{VERSION}", _MOUNTPOINT_PACKAGE] + version_result = await _run_vercel_command( + session, + "/usr/bin/rpm", + version_args, + ) + version_text = version_result.stdout.decode("utf-8", errors="replace").strip() + version = _parse_mountpoint_version(version_text) if version_result.ok() else None + binary_check = await _run_vercel_command( + session, + "/usr/bin/test", + ["-x", _MOUNTPOINT_BINARY], + ) + supported = ( + version is not None + and version[0] == _MOUNTPOINT_MINIMUM_VERSION[0] + and version >= _MOUNTPOINT_MINIMUM_VERSION + ) + if not binary_check.ok() or not supported: + await _run_required_command( + session, + "/usr/bin/dnf", + [ + "install", + "-y", + "--setopt=gpgcheck=1", + "fuse", + _MOUNTPOINT_PACKAGE, + ], + sudo=True, + timeout=_MOUNTPOINT_INSTALL_TIMEOUT_S, + context={"package": _MOUNTPOINT_PACKAGE}, + ) + await _run_required_command( + session, + "/usr/bin/test", + ["-x", _MOUNTPOINT_BINARY], + context={"package": _MOUNTPOINT_PACKAGE}, + ) + version_result = await _run_required_command( + session, + "/usr/bin/rpm", + version_args, + context={"package": _MOUNTPOINT_PACKAGE}, + ) + version_text = version_result.stdout.decode("utf-8", errors="replace").strip() + version = _parse_mountpoint_version(version_text) + supported = ( + version is not None + and version[0] == _MOUNTPOINT_MINIMUM_VERSION[0] + and version >= _MOUNTPOINT_MINIMUM_VERSION + ) + + if not supported: + raise MountConfigError( + message="unsupported Mountpoint for Amazon S3 version", + context={ + "backend": "vercel", + "actual_version": version_text, + "minimum_version": ".".join(map(str, _MOUNTPOINT_MINIMUM_VERSION)), + }, + ) + + +def _validate_s3_mount(mount: Mount) -> S3Mount: + if not isinstance(mount, S3Mount): + raise MountConfigError( + message="VercelCloudBucketMountStrategy only supports S3Mount", + context={"backend": "vercel", "mount_type": mount.type}, + ) + if not mount.ephemeral: + raise MountConfigError( + message="Vercel S3 mounts must be ephemeral", + context={"backend": "vercel", "mount_type": mount.type}, + ) + if (mount.access_key_id is None) != (mount.secret_access_key is None): + raise MountConfigError( + message="Vercel S3 mounts require both access_key_id and secret_access_key", + context={"backend": "vercel", "mount_type": mount.type}, + ) + if mount.session_token is not None and mount.access_key_id is None: + raise MountConfigError( + message=( + "Vercel S3 mounts require access_key_id and secret_access_key " + "when session_token is provided" + ), + context={"backend": "vercel", "mount_type": mount.type}, + ) + for name, value in ( + ("access_key_id", mount.access_key_id), + ("secret_access_key", mount.secret_access_key), + ("session_token", mount.session_token), + ): + if value is not None and not value.strip(): + raise MountConfigError( + message=f"Vercel S3 mount {name} must not be blank", + context={"backend": "vercel", "mount_type": mount.type}, + ) + return mount + + +async def _command_user_ids(session: VercelSandboxSession) -> tuple[str, str]: + uid_result = await _run_required_command(session, "/usr/bin/id", ["-u"]) + gid_result = await _run_required_command(session, "/usr/bin/id", ["-g"]) + uid = uid_result.stdout.decode("utf-8", errors="replace").strip() + gid = gid_result.stdout.decode("utf-8", errors="replace").strip() + if not uid.isdecimal() or not gid.isdecimal(): + raise MountCommandError( + command="/usr/bin/id", + stderr="Vercel returned a non-numeric user or group ID", + context={"backend": "vercel"}, + ) + return uid, gid + + +def _mount_args( + mount: S3Mount, + mount_path: Path, + *, + authenticated: bool, + user_ids: tuple[str, str] | None, +) -> list[str]: + args = [mount.bucket, sandbox_path_str(mount_path), "--allow-other"] + if not authenticated: + args.append("--no-sign-request") + if mount.read_only: + args.append("--read-only") + else: + args.extend(["--allow-overwrite", "--allow-delete"]) + if user_ids is not None: + uid, gid = user_ids + args.extend(["--uid", uid, "--gid", gid]) + if mount.region is not None: + args.extend(["--region", mount.region]) + if mount.endpoint_url is not None: + args.extend(["--endpoint-url", mount.endpoint_url]) + if mount.prefix: + prefix = mount.prefix if mount.prefix.endswith("/") else f"{mount.prefix}/" + args.extend(["--prefix", prefix]) + return args + + +async def _assert_empty_mount_directory( + session: VercelSandboxSession, + mount_path: Path, +) -> None: + mount_path_text = sandbox_path_str(mount_path) + result = await _run_required_command( + session, + "/usr/bin/find", + [mount_path_text, "-mindepth", "1", "-maxdepth", "1", "-print", "-quit"], + context={"mount_path": mount_path_text}, + ) + if result.stdout.strip(): + raise MountConfigError( + message="Vercel S3 mounts require an empty mount directory", + context={"backend": "vercel", "mount_path": mount_path_text}, + ) + + +async def _assert_canonical_mount_path( + session: VercelSandboxSession, + mount_path: Path, +) -> None: + mount_path_text = sandbox_path_str(mount_path) + helper_path = await session._ensure_runtime_helper_installed(RESOLVE_WORKSPACE_PATH_HELPER) + root_path_text = sandbox_path_str(session._workspace_root_path()) + result = await _run_required_command( + session, + str(helper_path), + [root_path_text, mount_path_text, "1"], + context={"mount_path": mount_path_text}, + ) + resolved_path_text = result.stdout.decode("utf-8", errors="replace").strip() + if resolved_path_text != mount_path_text: + raise MountConfigError( + message="Vercel S3 mount paths must not resolve through symlinks", + context={ + "backend": "vercel", + "mount_path": mount_path_text, + "resolved_path": resolved_path_text, + }, + ) + + +async def _mount_s3( + mount: S3Mount, + session: VercelSandboxSession, + mount_path: Path, +) -> None: + normalized_path = await session._validate_path_access(mount_path, for_write=True) + await _assert_canonical_mount_path(session, normalized_path) + mount_path_text = sandbox_path_str(normalized_path) + await _ensure_mountpoint(session) + await _run_required_command( + session, + "/usr/bin/mkdir", + ["-p", "--", mount_path_text], + context={"mount_path": mount_path_text}, + ) + await _assert_empty_mount_directory(session, normalized_path) + user_ids = await _command_user_ids(session) if not mount.read_only else None + await _assert_canonical_mount_path(session, normalized_path) + outcome = await _run_credentialed_mount_command( + session, + normalized_path, + _mount_args( + mount, + normalized_path, + authenticated=session._runtime_s3_mount_is_authenticated(normalized_path), + user_ids=user_ids, + ), + context={"bucket": mount.bucket, "mount_path": mount_path_text}, + ) + if isinstance(outcome, BaseException): + raise outcome from None + + +async def _is_mounted(session: VercelSandboxSession, mount_path: Path) -> bool: + mount_path_text = sandbox_path_str(mount_path) + args = ["--noheadings", "--output", "SOURCE", "--mountpoint", mount_path_text] + result = await _run_vercel_command(session, "/usr/bin/findmnt", args) + if result.exit_code == 1: + return False + if not result.ok(): + _raise_command_failure( + "/usr/bin/findmnt", + args, + result, + context={"mount_path": mount_path_text}, + ) + source = result.stdout.decode("utf-8", errors="replace").strip() + if source != _MOUNTPOINT_SOURCE: + raise MountConfigError( + message="refusing to manage an unexpected filesystem at the Vercel S3 mount path", + context={ + "backend": "vercel", + "mount_path": mount_path_text, + "expected_source": _MOUNTPOINT_SOURCE, + "actual_source": source, + }, + ) + return True + + +async def _unmount_s3(session: VercelSandboxSession, mount_path: Path) -> None: + if not await _is_mounted(session, mount_path): + raise MountConfigError( + message="tracked Vercel S3 mount is missing from its configured path", + context={ + "backend": "vercel", + "mount_path": sandbox_path_str(mount_path), + }, + ) + + mount_path_text = sandbox_path_str(mount_path) + args = [mount_path_text] + result = await _run_vercel_command( + session, + "/usr/bin/umount", + args, + sudo=True, + ) + if result.ok(): + return + if not await _is_mounted(session, mount_path): + # A mount can move with a renamed ancestor, so disappearance after umount is ambiguous. + raise MountConfigError( + message="Vercel S3 mount state became ambiguous during unmount", + context={ + "backend": "vercel", + "mount_path": mount_path_text, + }, + ) + _raise_command_failure( + "/usr/bin/umount", + args, + result, + context={"mount_path": mount_path_text}, + ) + + +class VercelCloudBucketMountStrategy(MountStrategyBase): + """Select Vercel's create-time-only application of the remote mount policy. + + This strategy does not imply dynamic mount mutation, credential refresh, or resumable mounts. + Those exclusions keep the provider lifecycle auditable. + """ + + type: Literal["vercel_cloud_bucket"] = "vercel_cloud_bucket" + + def validate_mount(self, mount: Mount) -> None: + _validate_s3_mount(mount) + + def supports_native_snapshot_detach(self, mount: Mount) -> bool: + _ = mount + return False + + async def activate( + self, + mount: Mount, + session: BaseSandboxSession, + dest: Path, + base_dir: Path, + ) -> list[MaterializedFile]: + _ = base_dir + vercel_session = _require_vercel_session(session) + async with vercel_session._s3_mount_operation(force_lock=True): + if not vercel_session._runtime_s3_mount_activation_allowed(): + raise MountConfigError( + message=( + "Vercel S3 mount topology is fixed when the sandbox is created; " + "dynamic manifest application is not supported" + ), + context={"backend": "vercel"}, + ) + declared_mount = _validate_s3_mount(mount) + mount_path = declared_mount._resolve_mount_path(vercel_session, dest) + s3_mount = vercel_session._runtime_trusted_s3_mount(mount_path) + try: + await _mount_s3(s3_mount, vercel_session, mount_path) + except (Exception, asyncio.CancelledError) as exc: + await vercel_session._runtime_fail_s3_mount_transition(exc) + raise + vercel_session._runtime_record_s3_mount_active(mount_path) + return [] + + async def deactivate( + self, + mount: Mount, + session: BaseSandboxSession, + dest: Path, + base_dir: Path, + ) -> None: + _ = base_dir + vercel_session = _require_vercel_session(session) + declared_mount = _validate_s3_mount(mount) + mount_path = declared_mount._resolve_mount_path(vercel_session, dest) + if not vercel_session._runtime_s3_mount_is_active(mount_path): + return + try: + await _unmount_s3(vercel_session, mount_path) + except (Exception, asyncio.CancelledError) as exc: + await vercel_session._runtime_fail_s3_mount_transition(exc) + raise + vercel_session._runtime_record_s3_mount_inactive(mount_path) + + async def teardown_for_snapshot( + self, + mount: Mount, + session: BaseSandboxSession, + path: Path, + ) -> None: + _validate_s3_mount(mount) + vercel_session = _require_vercel_session(session) + if not vercel_session._runtime_s3_mount_is_active(path): + return + try: + await _unmount_s3(vercel_session, path) + except (Exception, asyncio.CancelledError) as exc: + await vercel_session._runtime_fail_s3_mount_transition(exc) + raise + vercel_session._runtime_record_s3_mount_detached(path) + + async def restore_after_snapshot( + self, + mount: Mount, + session: BaseSandboxSession, + path: Path, + ) -> None: + _validate_s3_mount(mount) + vercel_session = _require_vercel_session(session) + if not vercel_session._runtime_s3_mount_is_detached(path): + return + s3_mount = vercel_session._runtime_trusted_s3_mount(path) + try: + await _mount_s3(s3_mount, vercel_session, path) + except (Exception, asyncio.CancelledError) as exc: + await vercel_session._runtime_fail_s3_mount_transition(exc) + raise + vercel_session._runtime_record_s3_mount_restored(path) + + def build_docker_volume_driver_config( + self, + mount: Mount, + ) -> tuple[str, dict[str, str], bool] | None: + _ = mount + return None + + +__all__ = [ + "VercelCloudBucketMountStrategy", +] diff --git a/src/agents/extensions/sandbox/vercel/sandbox.py b/src/agents/extensions/sandbox/vercel/sandbox.py index ab25bc3398..28f9bbafc8 100644 --- a/src/agents/extensions/sandbox/vercel/sandbox.py +++ b/src/agents/extensions/sandbox/vercel/sandbox.py @@ -17,6 +17,9 @@ import posixpath import tarfile import uuid +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from contextvars import ContextVar from pathlib import Path, PurePosixPath from typing import Any, Literal, cast from urllib.parse import urlsplit @@ -25,6 +28,7 @@ from pydantic import TypeAdapter, field_serializer, field_validator from vercel import sandbox as vercel_sandbox +from ....sandbox.entries import BaseEntry, Dir, S3Mount, resolve_workspace_path from ....sandbox.errors import ( ConfigurationError, ErrorCode, @@ -32,6 +36,7 @@ ExecTimeoutError, ExecTransportError, ExposedPortUnavailableError, + MountConfigError, WorkspaceArchiveReadError, WorkspaceArchiveWriteError, WorkspaceReadNotFoundError, @@ -39,7 +44,8 @@ WorkspaceWriteTypeError, ) from ....sandbox.manifest import Manifest -from ....sandbox.session import SandboxSession, SandboxSessionState +from ....sandbox.materialization import MaterializationResult +from ....sandbox.session import SandboxSession, SandboxSessionState, manifest_ops from ....sandbox.session.base_sandbox_session import BaseSandboxSession from ....sandbox.session.dependencies import Dependencies from ....sandbox.session.manager import Instrumentation @@ -67,6 +73,10 @@ _WORKSPACE_PERSISTENCE_TAR: WorkspacePersistenceMode = "tar" _WORKSPACE_PERSISTENCE_SNAPSHOT: WorkspacePersistenceMode = "snapshot" _VERCEL_SNAPSHOT_MAGIC = b"UC_VERCEL_SNAPSHOT_V1\n" +_VERCEL_S3_MOUNT_START_SESSION: ContextVar[object | None] = ContextVar( + "vercel_s3_mount_start_session", + default=None, +) DEFAULT_VERCEL_WORKSPACE_ROOT = "/vercel/sandbox" _DEFAULT_MANIFEST_ROOT = cast(str, Manifest.model_fields["root"].default) DEFAULT_VERCEL_SANDBOX_TIMEOUT_MS = 270_000 @@ -180,6 +190,156 @@ def _serialize_network_policy(value: NetworkPolicy | None) -> object | None: return cast(object | None, _NETWORK_POLICY_ADAPTER.dump_python(value, mode="json")) +def _vercel_s3_mounts(manifest: Manifest) -> list[S3Mount]: + mounts: list[S3Mount] = [] + for mount, _mount_path in manifest.mount_targets(): + if mount.mount_strategy.type != "vercel_cloud_bucket": + continue + if not isinstance(mount, S3Mount): + raise MountConfigError( + message="VercelCloudBucketMountStrategy only supports S3Mount", + context={"backend": "vercel", "mount_type": mount.type}, + ) + mounts.append(mount) + return mounts + + +def _entry_without_vercel_s3_mounts(entry: BaseEntry) -> BaseEntry | None: + if isinstance(entry, S3Mount) and entry.mount_strategy.type == "vercel_cloud_bucket": + return None + if not isinstance(entry, Dir): + return entry.model_copy(deep=True) + + children: dict[str | Path, BaseEntry] = {} + for name, child in entry.children.items(): + retained = _entry_without_vercel_s3_mounts(child) + if retained is not None: + children[name] = retained + return entry.model_copy(update={"children": children}, deep=True) + + +def _manifest_without_vercel_s3_mounts(manifest: Manifest) -> Manifest: + entries: dict[str | Path, BaseEntry] = {} + for name, entry in manifest.entries.items(): + retained = _entry_without_vercel_s3_mounts(entry) + if retained is not None: + entries[name] = retained + return manifest.model_copy(update={"entries": entries}, deep=True) + + +def _vercel_s3_mount_activation_targets(manifest: Manifest) -> list[tuple[S3Mount, Path]]: + root = posix_path_as_path(coerce_posix_path(manifest.root)) + targets: list[tuple[S3Mount, Path]] = [] + for logical_path, entry in manifest.iter_entries(): + if not isinstance(entry, S3Mount): + continue + if entry.mount_strategy.type != "vercel_cloud_bucket": + continue + targets.append((entry, resolve_workspace_path(root, logical_path))) + return targets + + +def _vercel_s3_mount_map(manifest: Manifest) -> dict[str, S3Mount]: + _vercel_s3_mounts(manifest) + targets = manifest.mount_targets() + root = posixpath.normpath(manifest.root) + root_path = PurePosixPath(root) + entry_targets = [ + ( + entry, + PurePosixPath(posixpath.normpath(posixpath.join(root, logical_path.as_posix()))), + ) + for logical_path, entry in manifest.iter_entries() + ] + mounts: dict[str, S3Mount] = {} + for index, (mount, mount_path) in enumerate(targets): + if mount.mount_strategy.type != "vercel_cloud_bucket": + continue + assert isinstance(mount, S3Mount) + path_text = posixpath.normpath(mount_path.as_posix()) + if path_text == root: + raise MountConfigError( + message="Vercel does not support mounting an S3 bucket at the workspace root", + context={"backend": "vercel", "mount_path": path_text}, + ) + path = PurePosixPath(path_text) + if root_path not in path.parents: + raise MountConfigError( + message="Vercel S3 mount paths must stay within the workspace root", + context={ + "backend": "vercel", + "mount_path": path_text, + "workspace_root": root, + }, + ) + for entry, entry_path in entry_targets: + if entry is mount or isinstance(entry, Dir) and entry_path in path.parents: + continue + if path == entry_path or path in entry_path.parents or entry_path in path.parents: + raise MountConfigError( + message="Vercel S3 mount paths must not overlap manifest entries", + context={ + "backend": "vercel", + "mount_path": path_text, + "overlapping_entry_path": entry_path.as_posix(), + }, + ) + for other_index, (_other_mount, other_path) in enumerate(targets): + if other_index == index: + continue + other = PurePosixPath(posixpath.normpath(other_path.as_posix())) + if path == other or path in other.parents or other in path.parents: + raise MountConfigError( + message="Vercel S3 mount paths must not overlap other mounts", + context={ + "backend": "vercel", + "mount_path": path_text, + "overlapping_mount_path": other.as_posix(), + }, + ) + mounts[path_text] = mount + return mounts + + +def _strip_vercel_mount_inline_credentials(value: object) -> None: + if isinstance(value, dict): + mount_strategy = value.get("mount_strategy") + if ( + value.get("type") == "s3_mount" + and isinstance(mount_strategy, dict) + and mount_strategy.get("type") == "vercel_cloud_bucket" + ): + value.pop("access_key_id", None) + value.pop("secret_access_key", None) + value.pop("session_token", None) + for nested_value in value.values(): + _strip_vercel_mount_inline_credentials(nested_value) + elif isinstance(value, list | tuple): + for nested_value in value: + _strip_vercel_mount_inline_credentials(nested_value) + + +def _manifest_without_vercel_s3_credentials(manifest: Manifest) -> Manifest: + sanitized = manifest.model_copy(deep=True) + for mount in _vercel_s3_mounts(sanitized): + mount.access_key_id = None + mount.secret_access_key = None + mount.session_token = None + return sanitized + + +def _manifest_has_vercel_s3_credentials(manifest: Manifest) -> bool: + return any( + credential is not None + for mount in _vercel_s3_mounts(manifest) + for credential in ( + mount.access_key_id, + mount.secret_access_key, + mount.session_token, + ) + ) + + class VercelSandboxClientOptions(BaseSandboxClientOptions): """Client options for the Vercel sandbox backend.""" @@ -195,6 +355,7 @@ class VercelSandboxClientOptions(BaseSandboxClientOptions): workspace_persistence: WorkspacePersistenceMode = _WORKSPACE_PERSISTENCE_TAR snapshot_expiration_ms: int | None = None network_policy: NetworkPolicy | None = None + allow_s3_credential_exposure: bool = False def __init__( self, @@ -209,6 +370,7 @@ def __init__( workspace_persistence: WorkspacePersistenceMode = _WORKSPACE_PERSISTENCE_TAR, snapshot_expiration_ms: int | None = None, network_policy: NetworkPolicy | None = None, + allow_s3_credential_exposure: bool = False, *, type: Literal["vercel"] = "vercel", ) -> None: @@ -225,6 +387,7 @@ def __init__( workspace_persistence=workspace_persistence, snapshot_expiration_ms=snapshot_expiration_ms, network_policy=network_policy, + allow_s3_credential_exposure=allow_s3_credential_exposure, ) @field_validator("network_policy", mode="before") @@ -252,6 +415,19 @@ class VercelSandboxSessionState(SandboxSessionState): workspace_persistence: WorkspacePersistenceMode = _WORKSPACE_PERSISTENCE_TAR snapshot_expiration_ms: int | None = None network_policy: NetworkPolicy | None = None + s3_mounts_non_resumable: bool = False + + @field_serializer("manifest") + def _serialize_manifest_without_inline_credentials( + self, + manifest: Manifest, + ) -> dict[str, object]: + payload = cast( + dict[str, object], + manifest.model_dump(mode="json", serialize_as_any=True), + ) + _strip_vercel_mount_inline_credentials(payload) + return payload @field_validator("network_policy", mode="before") @classmethod @@ -264,11 +440,31 @@ def _serialize_network_policy_field(self, value: NetworkPolicy | None) -> object class VercelSandboxSession(BaseSandboxSession): - """SandboxSession implementation backed by a Vercel sandbox.""" + """SandboxSession implementation backed by a Vercel sandbox. + + This provider applies the remote mount simplicity boundary by fixing the mount set at creation + and keeping its trusted configuration only in memory. Keep the lifecycle limited to create, + tar detach/remount, and close. Do not add dynamic mutation, persisted mount reconstruction, + credential refresh, or best-effort reconciliation without a trusted provider primitive that + makes those transitions unambiguous. + """ state: VercelSandboxSessionState _sandbox: Any | None _token: str | None + _s3_mounts_started: bool + _active_s3_mount_paths: set[str] + _detached_s3_mount_paths: set[str] + _trusted_s3_mounts: dict[str, S3Mount] + _trusted_s3_mount_credentials: dict[ + str, + tuple[str | None, str | None, str | None], + ] + _trusted_manifest: Manifest + _s3_mount_session_closed: bool + _s3_mount_failure: str | None + _s3_mount_operation_lock: asyncio.Lock + _s3_mount_operation_owner: asyncio.Task[Any] | None def __init__( self, @@ -276,10 +472,66 @@ def __init__( state: VercelSandboxSessionState, sandbox: Any | None = None, token: str | None = None, + allow_s3_credential_exposure: bool = False, + trusted_s3_mounts: dict[str, S3Mount] | None = None, ) -> None: + resolved_trusted_s3_mounts: dict[str, S3Mount] = {} + trusted_s3_mount_credentials: dict[ + str, + tuple[str | None, str | None, str | None], + ] = {} + for path, mount in (trusted_s3_mounts or {}).items(): + trusted_mount = mount.model_copy(deep=True) + credentials = ( + trusted_mount.access_key_id, + trusted_mount.secret_access_key, + trusted_mount.session_token, + ) + trusted_mount.access_key_id = None + trusted_mount.secret_access_key = None + trusted_mount.session_token = None + resolved_trusted_s3_mounts[path] = trusted_mount + trusted_s3_mount_credentials[path] = credentials + has_trusted_credentials = any( + credential is not None + for credentials in trusted_s3_mount_credentials.values() + for credential in credentials + ) + if has_trusted_credentials and not allow_s3_credential_exposure: + raise MountConfigError( + message=( + "Vercel S3 mounts expose inline credentials to code running in the sandbox; " + "set allow_s3_credential_exposure=True only for credentials scoped to that " + "sandbox" + ), + context={"backend": "vercel"}, + ) + declared_mount_paths = set(_vercel_s3_mount_map(state.manifest)) + if declared_mount_paths != set(resolved_trusted_s3_mounts): + raise MountConfigError( + message=( + "Vercel S3 mount topology must match trusted create-time configuration; " + "persisted session state cannot reconstruct or change it" + ), + context={ + "backend": "vercel", + "declared_mount_paths": sorted(declared_mount_paths), + "trusted_mount_paths": sorted(resolved_trusted_s3_mounts), + }, + ) self.state = state self._sandbox = sandbox self._token = token + self._s3_mounts_started = False + self._active_s3_mount_paths = set() + self._detached_s3_mount_paths = set() + self._trusted_s3_mounts = resolved_trusted_s3_mounts + self._trusted_s3_mount_credentials = trusted_s3_mount_credentials + self._trusted_manifest = state.manifest.model_copy(deep=True) + self._s3_mount_session_closed = False + self._s3_mount_failure = None + self._s3_mount_operation_lock = asyncio.Lock() + self._s3_mount_operation_owner = None @classmethod def from_state( @@ -288,8 +540,212 @@ def from_state( *, sandbox: Any | None = None, token: str | None = None, + allow_s3_credential_exposure: bool = False, + trusted_s3_mounts: dict[str, S3Mount] | None = None, ) -> VercelSandboxSession: - return cls(state=state, sandbox=sandbox, token=token) + return cls( + state=state, + sandbox=sandbox, + token=token, + allow_s3_credential_exposure=allow_s3_credential_exposure, + trusted_s3_mounts=trusted_s3_mounts, + ) + + @staticmethod + def _s3_mount_path_key(path: Path) -> str: + return posixpath.normpath(path.as_posix()) + + def _runtime_s3_mount_activation_allowed(self) -> bool: + return _VERCEL_S3_MOUNT_START_SESSION.get() is self + + def _runtime_s3_mount_is_active(self, path: Path) -> bool: + return self._s3_mount_path_key(path) in self._active_s3_mount_paths + + def _runtime_s3_mount_is_detached(self, path: Path) -> bool: + return self._s3_mount_path_key(path) in self._detached_s3_mount_paths + + def _runtime_trusted_s3_mount(self, path: Path) -> S3Mount: + key = self._s3_mount_path_key(path) + mount = self._trusted_s3_mounts.get(key) + if mount is None: + raise MountConfigError( + message="Vercel S3 mount configuration is unavailable outside sandbox creation", + context={"backend": "vercel", "mount_path": key}, + ) + return mount + + def _runtime_s3_mount_is_authenticated(self, path: Path) -> bool: + key = self._s3_mount_path_key(path) + credentials = self._trusted_s3_mount_credentials.get(key) + return credentials is not None and credentials[0] is not None + + def _runtime_s3_mount_environment(self, path: Path) -> dict[str, str]: + key = self._s3_mount_path_key(path) + credentials = self._trusted_s3_mount_credentials.get(key) + mount = self._trusted_s3_mounts.get(key) + if credentials is None or mount is None: + raise MountConfigError( + message="Vercel S3 mount configuration is unavailable outside sandbox creation", + context={"backend": "vercel", "mount_path": key}, + ) + access_key_id, secret_access_key, session_token = credentials + env: dict[str, str] = {} + if access_key_id is not None and secret_access_key is not None: + env["AWS_ACCESS_KEY_ID"] = access_key_id + env["AWS_SECRET_ACCESS_KEY"] = secret_access_key + if session_token is not None: + env["AWS_SESSION_TOKEN"] = session_token + if mount.region is not None: + env["AWS_REGION"] = mount.region + return env + + def _runtime_provider_retryability(self, error: BaseException) -> bool | None: + return _vercel_provider_retryability(error) + + async def _runtime_fail_s3_mount_transition(self, error: BaseException) -> None: + self._s3_mount_failure = type(error).__name__ + stop_task = asyncio.create_task(self._stop_attached_sandbox()) + while not stop_task.done(): + try: + await asyncio.shield(stop_task) + except asyncio.CancelledError: + # A cancelled privileged transition has unknown state, so cleanup must finish. + continue + await stop_task + + def _runtime_assert_s3_mount_topology(self) -> None: + topology_changed = ( + self.state.manifest != self._trusted_manifest + if self._trusted_s3_mounts + else bool(_vercel_s3_mounts(self.state.manifest)) + ) + if topology_changed: + raise MountConfigError( + message="Vercel S3 mount topology cannot change after sandbox creation", + context={"backend": "vercel"}, + ) + + def _runtime_assert_s3_workspace_root(self) -> None: + if self._trusted_s3_mounts and self.state.manifest.root != self._trusted_manifest.root: + raise MountConfigError( + message="Vercel S3 mount topology cannot change after sandbox creation", + context={"backend": "vercel"}, + ) + + @asynccontextmanager + async def _s3_mount_operation( + self, + *, + force_lock: bool = False, + validate_topology: bool = True, + ) -> AsyncIterator[None]: + if not self._trusted_s3_mounts or ( + self._runtime_s3_mount_activation_allowed() + and not self._active_s3_mount_paths + and not self._detached_s3_mount_paths + and not force_lock + ): + if validate_topology and self._trusted_s3_mounts: + self._runtime_assert_s3_workspace_root() + yield + return + + current_task = asyncio.current_task() + assert current_task is not None + if self._s3_mount_operation_owner is current_task: + yield + return + + async with self._s3_mount_operation_lock: + if validate_topology: + self._runtime_assert_s3_workspace_root() + self._s3_mount_operation_owner = current_task + try: + yield + finally: + self._s3_mount_operation_owner = None + + def _runtime_record_s3_mount_active(self, path: Path) -> None: + key = self._s3_mount_path_key(path) + self._detached_s3_mount_paths.discard(key) + self._active_s3_mount_paths.add(key) + + def _runtime_record_s3_mount_inactive(self, path: Path) -> None: + key = self._s3_mount_path_key(path) + self._active_s3_mount_paths.discard(key) + self._detached_s3_mount_paths.discard(key) + + def _runtime_record_s3_mount_detached(self, path: Path) -> None: + key = self._s3_mount_path_key(path) + self._active_s3_mount_paths.discard(key) + self._detached_s3_mount_paths.add(key) + + def _runtime_record_s3_mount_restored(self, path: Path) -> None: + self._runtime_record_s3_mount_active(path) + + async def _start_workspace(self) -> None: + self._runtime_assert_s3_mount_topology() + if not _vercel_s3_mounts(self.state.manifest): + await super()._start_workspace() + return + if self._s3_mounts_started: + raise MountConfigError( + message=( + "Vercel S3 mount topology is fixed when the sandbox is created; " + "starting the same mounted session again is not supported" + ), + context={"backend": "vercel"}, + ) + + activation_token = _VERCEL_S3_MOUNT_START_SESSION.set(self) + try: + await super()._start_workspace() + for mount, destination in _vercel_s3_mount_activation_targets(self._trusted_manifest): + await mount.mount_strategy.activate( + mount, + self, + destination, + self._manifest_base_dir(), + ) + except (Exception, asyncio.CancelledError) as exc: + if self._active_s3_mount_paths: + self._s3_mounts_started = True + await self._runtime_fail_s3_mount_transition(exc) + raise + finally: + _VERCEL_S3_MOUNT_START_SESSION.reset(activation_token) + self._s3_mounts_started = True + + async def _apply_manifest( + self, + *, + only_ephemeral: bool = False, + provision_accounts: bool = True, + ) -> MaterializationResult: + if self._runtime_s3_mount_activation_allowed() and self._trusted_s3_mounts: + return await manifest_ops.apply_manifest( + self, + manifest=_manifest_without_vercel_s3_mounts(self._trusted_manifest), + only_ephemeral=only_ephemeral, + provision_accounts=provision_accounts, + ) + return await super()._apply_manifest( + only_ephemeral=only_ephemeral, + provision_accounts=provision_accounts, + ) + + async def _validate_manifest_application(self, *, only_ephemeral: bool = False) -> None: + _ = only_ephemeral + if not self._runtime_s3_mount_activation_allowed() and ( + self._trusted_s3_mounts or _vercel_s3_mounts(self.state.manifest) + ): + raise MountConfigError( + message=( + "Vercel S3 mount topology is fixed when the sandbox is created; " + "dynamic manifest application is not supported" + ), + context={"backend": "vercel"}, + ) def supports_pty(self) -> bool: return False @@ -326,12 +782,14 @@ def _validate_tar_bytes( self, raw: bytes, *, + reject_rel_paths: set[Path] | None = None, allow_external_symlink_targets: bool = True, ) -> None: try: with tarfile.open(fileobj=io.BytesIO(raw), mode="r:*") as tar: validate_tarfile( tar, + reject_rel_paths=reject_rel_paths or (), allow_external_symlink_targets=allow_external_symlink_targets, ) except UnsafeTarMemberError as exc: @@ -362,6 +820,23 @@ async def _prepare_backend_workspace(self) -> None: ) async def _ensure_sandbox(self, *, source: Any | None = None) -> Any: + if self._s3_mount_session_closed: + raise WorkspaceStartError( + path=self._workspace_root_path(), + context={ + "backend": "vercel", + "reason": "mounted_session_closed", + }, + ) + if self._s3_mount_failure is not None: + raise WorkspaceStartError( + path=self._workspace_root_path(), + context={ + "backend": "vercel", + "reason": "mount_transition_failed", + "cause_type": self._s3_mount_failure, + }, + ) sandbox = self._sandbox if sandbox is not None: return sandbox @@ -410,13 +885,9 @@ async def _stop_attached_sandbox(self) -> None: sandbox = self._sandbox if sandbox is None: return - try: - await sandbox.stop() - except Exception: - pass - finally: - await self._close_sandbox_client() - self._sandbox = None + await sandbox.stop(blocking=True) + await self._close_sandbox_client() + self._sandbox = None async def _replace_sandbox_from_snapshot(self, snapshot_id: str) -> None: await self._stop_attached_sandbox() @@ -451,31 +922,73 @@ async def running(self) -> bool: return bool(sandbox.status == SandboxStatus.RUNNING) async def shutdown(self) -> None: - await self._stop_attached_sandbox() + async with self._s3_mount_operation(validate_topology=False): + if self._s3_mount_session_closed: + return + await self._shutdown_with_s3_mounts() + + async def stop(self) -> None: + if self._s3_mount_session_closed: + return + await super().stop() + + def _should_compute_snapshot_fingerprint_on_persist(self) -> bool: + return not self._trusted_s3_mounts + + async def _shutdown_with_s3_mounts(self) -> None: + first_error: Exception | None = None + if self._s3_mount_failure is None: + for mount_path_text, mount in self._trusted_s3_mounts.items(): + mount_path = Path(mount_path_text) + try: + await mount.mount_strategy.teardown_for_snapshot(mount, self, mount_path) + except Exception as exc: + if first_error is None: + first_error = exc + try: + await self._stop_attached_sandbox() + except (Exception, asyncio.CancelledError) as exc: + if self._detached_s3_mount_paths: + await self._runtime_fail_s3_mount_transition(exc) + raise + self._active_s3_mount_paths.clear() + self._detached_s3_mount_paths.clear() + if self._trusted_s3_mounts: + self._s3_mount_session_closed = True + if first_error is not None: + raise first_error async def _exec_internal( self, *command: str | Path, timeout: float | None = None, + ) -> ExecResult: + async with self._s3_mount_operation(): + return await self._exec_internal_with_s3_mounts(*command, timeout=timeout) + + async def _exec_internal_with_s3_mounts( + self, + *command: str | Path, + timeout: float | None = None, ) -> ExecResult: sandbox = await self._ensure_sandbox() normalized = [str(part) for part in command] if not normalized: return ExecResult(stdout=b"", stderr=b"", exit_code=0) - try: - finished = await asyncio.wait_for( - sandbox.run_command( - normalized[0], - normalized[1:], - cwd=self.state.manifest.root, - ), - timeout=timeout, + async def run_and_collect_output() -> ExecResult: + finished = await sandbox.run_command( + normalized[0], + normalized[1:], + cwd=self.state.manifest.root, ) stdout = (await finished.stdout()).encode("utf-8") stderr = (await finished.stderr()).encode("utf-8") return ExecResult(stdout=stdout, stderr=stderr, exit_code=finished.exit_code) - except TimeoutError as exc: + + try: + return await asyncio.wait_for(run_and_collect_output(), timeout=timeout) + except asyncio.TimeoutError as exc: raise ExecTimeoutError(command=normalized, timeout_s=timeout, cause=exc) from exc except ExecTimeoutError: raise @@ -522,6 +1035,15 @@ async def _resolve_exposed_port(self, port: int) -> ExposedPortEndpoint: ) async def read(self, path: Path, *, user: str | User | None = None) -> io.IOBase: + async with self._s3_mount_operation(): + return await self._read_with_s3_mounts(path, user=user) + + async def _read_with_s3_mounts( + self, + path: Path, + *, + user: str | User | None = None, + ) -> io.IOBase: if user is not None: self._reject_user_arg(op="read", user=user) @@ -545,6 +1067,16 @@ async def write( data: io.IOBase, *, user: str | User | None = None, + ) -> None: + async with self._s3_mount_operation(): + await self._write_with_s3_mounts(path, data, user=user) + + async def _write_with_s3_mounts( + self, + path: Path, + data: io.IOBase, + *, + user: str | User | None = None, ) -> None: if user is not None: self._reject_user_arg(op="write", user=user) @@ -570,16 +1102,26 @@ async def write( ) from exc async def persist_workspace(self) -> io.IOBase: - return await with_ephemeral_mounts_removed( - self, - self._persist_workspace_internal, - error_path=self._workspace_root_path(), - error_cls=WorkspaceArchiveReadError, - operation_error_context_key="snapshot_error_before_remount_corruption", - ) + async with self._s3_mount_operation(validate_topology=False): + self._runtime_assert_s3_mount_topology() + try: + return await with_ephemeral_mounts_removed( + self, + self._persist_workspace_internal, + error_path=self._workspace_root_path(), + error_cls=WorkspaceArchiveReadError, + operation_error_context_key="snapshot_error_before_remount_corruption", + ) + except (Exception, asyncio.CancelledError) as exc: + if self._detached_s3_mount_paths: + await self._runtime_fail_s3_mount_transition(exc) + raise async def _persist_workspace_internal(self) -> io.IOBase: - if self.state.workspace_persistence == _WORKSPACE_PERSISTENCE_SNAPSHOT: + if ( + self.state.workspace_persistence == _WORKSPACE_PERSISTENCE_SNAPSHOT + and not self._native_snapshot_requires_tar_fallback() + ): root = self._workspace_root_path() sandbox = await self._ensure_sandbox() try: @@ -604,7 +1146,7 @@ async def _persist_workspace_internal(self) -> io.IOBase: key=lambda item: item.as_posix(), ) ] - tar_command = ("tar", "cf", archive_path.as_posix(), *excludes, ".") + tar_command = ("tar", "cf", archive_path.as_posix(), "--no-wildcards", *excludes, ".") try: result = await self.exec(*tar_command, shell=False) if not result.ok(): @@ -639,6 +1181,11 @@ async def _persist_workspace_internal(self) -> io.IOBase: pass async def hydrate_workspace(self, data: io.IOBase) -> None: + async with self._s3_mount_operation(validate_topology=False): + self._runtime_assert_s3_mount_topology() + await self._hydrate_workspace_with_s3_mounts(data) + + async def _hydrate_workspace_with_s3_mounts(self, data: io.IOBase) -> None: raw = data.read() if isinstance(raw, str): raw = raw.encode("utf-8") @@ -648,13 +1195,37 @@ async def hydrate_workspace(self, data: io.IOBase) -> None: actual_type=type(raw).__name__, ) - await with_ephemeral_mounts_removed( - self, - lambda: self._hydrate_workspace_internal(bytes(raw)), - error_path=self._workspace_root_path(), - error_cls=WorkspaceArchiveWriteError, - operation_error_context_key="hydrate_error_before_remount_corruption", - ) + raw_bytes = bytes(raw) + if self._active_s3_mount_paths: + if _decode_snapshot_ref(raw_bytes) is not None: + raise MountConfigError( + message="Vercel cannot hydrate a native snapshot while S3 mounts are active", + context={"backend": "vercel"}, + ) + try: + self._validate_tar_bytes( + raw_bytes, + reject_rel_paths=self._mount_relpaths_within_workspace(), + allow_external_symlink_targets=False, + ) + except Exception as exc: + raise WorkspaceArchiveWriteError( + path=self._workspace_root_path(), + cause=exc, + ) from exc + + try: + await with_ephemeral_mounts_removed( + self, + lambda: self._hydrate_workspace_internal(raw_bytes), + error_path=self._workspace_root_path(), + error_cls=WorkspaceArchiveWriteError, + operation_error_context_key="hydrate_error_before_remount_corruption", + ) + except (Exception, asyncio.CancelledError) as exc: + if self._detached_s3_mount_paths: + await self._runtime_fail_s3_mount_transition(exc) + raise async def _hydrate_workspace_internal(self, raw: bytes) -> None: snapshot_id = ( @@ -717,6 +1288,22 @@ async def _write_files_with_retry(self, files: list[dict[str, object]]) -> None: await sandbox.write_files(files) +class _VercelSandboxSessionWrapper(SandboxSession): + async def aclose(self) -> None: + try: + await super().aclose() + except BaseException as error: + inner = cast(VercelSandboxSession, self._inner) + if inner._trusted_s3_mounts and inner._sandbox is not None: + try: + await inner.shutdown() + except BaseException as cleanup_error: + if isinstance(cleanup_error, asyncio.CancelledError): + raise cleanup_error from error + raise error from cleanup_error + raise + + class VercelSandboxClient(BaseSandboxClient[VercelSandboxClientOptions]): """Vercel-backed sandbox client.""" @@ -742,6 +1329,18 @@ def __init__( self._instrumentation = instrumentation or Instrumentation() self._dependencies = dependencies + def _wrap_session( + self, + inner: BaseSandboxSession, + *, + instrumentation: Instrumentation | None = None, + ) -> SandboxSession: + return _VercelSandboxSessionWrapper( + inner, + instrumentation=instrumentation, + dependencies=self._resolve_dependencies(), + ) + async def create( self, *, @@ -750,6 +1349,22 @@ async def create( options: VercelSandboxClientOptions, ) -> SandboxSession: resolved_manifest = _resolve_manifest_root(manifest) + if ( + _manifest_has_vercel_s3_credentials(resolved_manifest) + and not options.allow_s3_credential_exposure + ): + raise MountConfigError( + message=( + "Vercel S3 mounts expose inline credentials to code running in the sandbox; " + "set allow_s3_credential_exposure=True only for credentials scoped to that " + "sandbox" + ), + context={"backend": "vercel"}, + ) + trusted_s3_mounts = _vercel_s3_mount_map(resolved_manifest) + for mount in trusted_s3_mounts.values(): + mount.mount_strategy.validate_mount(mount) + state_manifest = _manifest_without_vercel_s3_credentials(resolved_manifest) resolved_token = self._token resolved_project_id = options.project_id or self._project_id resolved_team_id = options.team_id or self._team_id @@ -761,7 +1376,7 @@ async def create( snapshot_instance = resolve_snapshot(snapshot, str(session_id)) state = VercelSandboxSessionState( session_id=session_id, - manifest=resolved_manifest, + manifest=state_manifest, snapshot=snapshot_instance, sandbox_id="", project_id=resolved_project_id, @@ -775,8 +1390,14 @@ async def create( workspace_persistence=options.workspace_persistence, snapshot_expiration_ms=options.snapshot_expiration_ms, network_policy=options.network_policy, + s3_mounts_non_resumable=bool(trusted_s3_mounts), + ) + inner = VercelSandboxSession.from_state( + state, + token=resolved_token, + allow_s3_credential_exposure=options.allow_s3_credential_exposure, + trusted_s3_mounts=trusted_s3_mounts, ) - inner = VercelSandboxSession.from_state(state, token=resolved_token) await inner._ensure_sandbox() return self._wrap_session(inner, instrumentation=self._instrumentation) @@ -793,6 +1414,14 @@ async def delete(self, session: SandboxSession) -> SandboxSession: async def resume(self, state: SandboxSessionState) -> SandboxSession: if not isinstance(state, VercelSandboxSessionState): raise TypeError("VercelSandboxClient.resume expects a VercelSandboxSessionState") + if state.s3_mounts_non_resumable or _vercel_s3_mounts(state.manifest): + raise MountConfigError( + message=( + "Vercel sessions containing S3 mounts cannot be resumed; " + "create a new sandbox with a trusted manifest instead" + ), + context={"backend": "vercel"}, + ) resolved_token = self._token resolved_project_id = state.project_id or self._project_id @@ -838,7 +1467,7 @@ async def resume(self, state: SandboxSessionState) -> SandboxSession: # ABORTED, SNAPSHOTTING). Drop the handle and recreate below. await sandbox.client.aclose() sandbox = None - except TimeoutError: + except asyncio.TimeoutError: if sandbox is not None: await sandbox.client.aclose() sandbox = None diff --git a/src/agents/logger.py b/src/agents/logger.py index bd81a82716..18b8670680 100644 --- a/src/agents/logger.py +++ b/src/agents/logger.py @@ -1,3 +1,244 @@ import logging +from collections.abc import Callable, Mapping +from types import TracebackType + +from . import _debug logger = logging.getLogger("openai.agents") + +_DiagnosticExtra = Callable[[], Mapping[str, object]] +_DIAGNOSTIC_CONTEXT_FIELD = "openai_agents_diagnostic_context" + + +def _exception_info( + exc: BaseException, +) -> tuple[type[BaseException], BaseException, TracebackType | None]: + """Build logging exception info without evaluating exception truthiness.""" + traceback = BaseException.__getattribute__(exc, "__traceback__") + return type(exc), exc, traceback + + +def _log_record_extra(diagnostic_extra: _DiagnosticExtra | None) -> dict[str, object] | None: + if diagnostic_extra is None: + return None + try: + return {_DIAGNOSTIC_CONTEXT_FIELD: dict(diagnostic_extra())} + except Exception: + return None + + +def _log_action_error( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + redact: bool, + stacklevel: int, + diagnostic_extra: _DiagnosticExtra | None, +) -> None: + """Log an action failure without inspecting a redacted exception.""" + if redact: + target_logger.error("%s", message, stacklevel=stacklevel) + else: + target_logger.error( + "%s: %s", + message, + exc, + exc_info=_exception_info(exc), + extra=_log_record_extra(diagnostic_extra), + stacklevel=stacklevel, + ) + + +def _log_action_at_level( + log_method: Callable[..., None], + message: str, + exc: BaseException, + *, + redact: bool, + stacklevel: int, + diagnostic_extra: _DiagnosticExtra | None, +) -> None: + """Log an action failure at a caller-selected level.""" + if redact: + log_method("%s", message, stacklevel=stacklevel) + else: + log_method( + "%s: %s", + message, + exc, + exc_info=_exception_info(exc), + extra=_log_record_extra(diagnostic_extra), + stacklevel=stacklevel, + ) + + +def log_model_action_error( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Log a model-data failure according to the model logging policy.""" + _log_action_error( + target_logger, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_action_debug( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Debug-log a model-data failure according to the model logging policy.""" + _log_action_at_level( + target_logger.debug, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_action_warning( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Warning-log a model-data failure according to the model logging policy.""" + _log_action_at_level( + target_logger.warning, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_tool_action_error( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Log a tool-data failure according to the tool logging policy.""" + _log_action_error( + target_logger, + message, + exc, + redact=_debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_tool_action_debug( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Debug-log a tool-data failure according to the tool logging policy.""" + _log_action_at_level( + target_logger.debug, + message, + exc, + redact=_debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_tool_action_warning( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Warning-log a tool-data failure according to the tool logging policy.""" + _log_action_at_level( + target_logger.warning, + message, + exc, + redact=_debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_and_tool_action_error( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Log a mixed model/tool-data failure only when both data policies allow it.""" + _log_action_error( + target_logger, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_and_tool_action_debug( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Debug-log a mixed-data failure only when both data policies allow it.""" + _log_action_at_level( + target_logger.debug, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_and_tool_action_warning( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Warning-log a mixed-data failure only when both data policies allow it.""" + _log_action_at_level( + target_logger.warning, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) diff --git a/src/agents/mcp/_logging.py b/src/agents/mcp/_logging.py new file mode 100644 index 0000000000..ffb6df97b5 --- /dev/null +++ b/src/agents/mcp/_logging.py @@ -0,0 +1,52 @@ +from typing import Protocol +from urllib.parse import urlsplit, urlunsplit + +from .. import _debug + +_URL_DERIVED_NAME_PREFIXES = ("sse: ", "streamable_http: ", "streamable-http: ") + + +class _MCPServerNameSource(Protocol): + @property + def name(self) -> str: ... + + +def get_mcp_server_log_name(name: str) -> str: + """Remove URL credentials, query parameters, and fragments from MCP log names.""" + prefix = next( + (candidate for candidate in _URL_DERIVED_NAME_PREFIXES if name.startswith(candidate)), + "", + ) + candidate = name[len(prefix) :] if prefix else name + + try: + parsed = urlsplit(candidate) + except ValueError: + if prefix or candidate.lower().startswith(("http://", "https://")): + return f"{prefix}" + return name + + if parsed.scheme not in {"http", "https"}: + return name + + try: + hostname = parsed.hostname + port = parsed.port + except ValueError: + return f"{prefix}" + + if not parsed.netloc or not hostname or any(character.isspace() for character in hostname): + return f"{prefix}" + + host = f"[{hostname}]" if ":" in hostname else hostname + if port is not None: + host = f"{host}:{port}" + sanitized = urlunsplit((parsed.scheme, host, parsed.path, "", "")) + return f"{prefix}{sanitized}" + + +def get_mcp_server_log_message(message: str, server: _MCPServerNameSource) -> str: + """Build an MCP log message without reading the server name in redacted mode.""" + if _debug.DONT_LOG_TOOL_DATA: + return message + return f"{message} '{get_mcp_server_log_name(server.name)}'" diff --git a/src/agents/mcp/manager.py b/src/agents/mcp/manager.py index b100af8bb8..b8838be3e3 100644 --- a/src/agents/mcp/manager.py +++ b/src/agents/mcp/manager.py @@ -6,7 +6,8 @@ from dataclasses import dataclass from typing import Any -from ..logger import logger +from ..logger import log_tool_action_debug, log_tool_action_error, logger +from ._logging import get_mcp_server_log_message from .server import MCPServer @@ -260,10 +261,18 @@ async def cleanup_all(self) -> None: except asyncio.CancelledError as exc: if not self.suppress_cancelled_error: raise - logger.debug("Cleanup cancelled for MCP server '%s': %s", server.name, exc) + log_tool_action_debug( + logger, + get_mcp_server_log_message("Cleanup cancelled for MCP server", server), + exc, + ) self.errors[server] = exc except Exception as exc: - logger.exception("Failed to cleanup MCP server '%s': %s", server.name, exc) + log_tool_action_error( + logger, + get_mcp_server_log_message("Failed to cleanup MCP server", server), + exc, + ) self.errors[server] = exc async def _run_with_timeout( @@ -283,9 +292,12 @@ async def _attempt_connect( self._remove_failed_server(server) self.errors.pop(server, None) except asyncio.CancelledError as exc: + # Always record so connect_all()'s failure cleanup includes this server. + # Re-raising without recording left partially-opened servers uncleaned + # (especially under `async with`, where __aexit__ never runs). + self._record_failure(server, exc, phase="connect") if not self.suppress_cancelled_error: raise - self._record_failure(server, exc, phase="connect") except Exception as exc: self._record_failure(server, exc, phase="connect") if raise_on_error: @@ -302,7 +314,11 @@ def _refresh_active_servers(self) -> None: self._active_servers = list(self._all_servers) def _record_failure(self, server: MCPServer, exc: BaseException, phase: str) -> None: - logger.exception("Failed to %s MCP server '%s': %s", phase, server.name, exc) + log_tool_action_error( + logger, + get_mcp_server_log_message(f"Failed to {phase} MCP server", server), + exc, + ) if server not in self._failed_server_set: self.failed_servers.append(server) self._failed_server_set.add(server) @@ -340,10 +356,18 @@ async def _cleanup_servers(self, servers: Iterable[MCPServer]) -> None: except asyncio.CancelledError as exc: if not self.suppress_cancelled_error: raise - logger.debug("Cleanup cancelled for MCP server '%s': %s", server.name, exc) + log_tool_action_debug( + logger, + get_mcp_server_log_message("Cleanup cancelled for MCP server", server), + exc, + ) self.errors[server] = exc except Exception as exc: - logger.exception("Failed to cleanup MCP server '%s': %s", server.name, exc) + log_tool_action_error( + logger, + get_mcp_server_log_message("Failed to cleanup MCP server", server), + exc, + ) self.errors[server] = exc async def _connect_all_parallel(self, servers: list[MCPServer]) -> None: diff --git a/src/agents/mcp/server.py b/src/agents/mcp/server.py index 2681606b5e..4f8f89c293 100644 --- a/src/agents/mcp/server.py +++ b/src/agents/mcp/server.py @@ -38,11 +38,18 @@ ) from typing_extensions import NotRequired, TypedDict +from .. import _debug from ..exceptions import UserError -from ..logger import logger +from ..logger import ( + log_tool_action_debug, + log_tool_action_error, + log_tool_action_warning, + logger, +) from ..run_context import RunContextWrapper from ..tool import ToolErrorFunction from ..util._types import MaybeAwaitable +from ._logging import get_mcp_server_log_message, get_mcp_server_log_name from .util import ( HttpClientFactory, MCPToolCustomDataExtractor, @@ -119,10 +126,11 @@ async def _handle_post_request(self, ctx: Any) -> None: try: await super()._handle_post_request(ctx) - except httpx.HTTPError: - logger.warning( + except httpx.HTTPError as exc: + log_tool_action_warning( + logger, "Ignoring initialized notification HTTP failure", - exc_info=True, + exc, ) return @@ -161,7 +169,13 @@ async def _streamablehttp_client_with_transport( async with client: async with anyio.create_task_group() as tg: try: - logger.debug("Connecting to StreamableHTTP endpoint: %s", url) + if _debug.DONT_LOG_TOOL_DATA: + logger.debug("Connecting to StreamableHTTP endpoint") + else: + logger.debug( + "Connecting to StreamableHTTP endpoint: %s", + get_mcp_server_log_name(url), + ) def start_get_stream() -> None: tg.start_soon(transport.handle_get_stream, client, read_stream_writer) @@ -691,12 +705,15 @@ async def _apply_dynamic_tool_filter( if should_include: filtered_tools.append(tool) except Exception as e: - logger.error( - "Error applying tool filter to tool '%s' on server '%s': %s", - tool.name, - self.name, - e, - ) + if _debug.DONT_LOG_TOOL_DATA: + message = "Error applying MCP tool filter" + else: + server_name = get_mcp_server_log_name(self.name) + message = ( + f"Error applying MCP tool filter to tool '{tool.name}' " + f"on server '{server_name}'" + ) + log_tool_action_error(logger, message, e) # On error, exclude the tool for safety continue @@ -818,16 +835,20 @@ async def connect(self): if isinstance(cleanup_error, RuntimeError) and "cancel scope" in str( cleanup_error ): - logger.debug( - "Ignoring cancel scope error during cleanup of MCP server '%s': %s", - self.name, + log_tool_action_debug( + logger, + get_mcp_server_log_message( + "Ignoring cancel scope error during cleanup of MCP server", self + ), cleanup_error, ) else: # Log other cleanup errors but don't raise - original error is more # important - logger.warning( - "Error during cleanup of MCP server '%s': %s", self.name, cleanup_error + log_tool_action_warning( + logger, + get_mcp_server_log_message("Error during cleanup of MCP server", self), + cleanup_error, ) async def list_tools( @@ -1005,7 +1026,11 @@ async def cleanup(self): try: await self.exit_stack.aclose() except asyncio.CancelledError as e: - logger.debug("Cleanup cancelled for MCP server '%s': %s", self.name, e) + log_tool_action_debug( + logger, + get_mcp_server_log_message("Cleanup cancelled for MCP server", self), + e, + ) raise except BaseExceptionGroup as eg: # Extract HTTP errors from ExceptionGroup raised during cleanup @@ -1031,9 +1056,11 @@ async def cleanup(self): raise UserError(error_message) from http_error else: # Normal teardown - log but don't raise - logger.warning( - "HTTP error during cleanup of MCP server '%s': %s", - self.name, + log_tool_action_warning( + logger, + get_mcp_server_log_message( + "HTTP error during cleanup of MCP server", self + ), http_error, ) elif connect_error: @@ -1041,9 +1068,11 @@ async def cleanup(self): error_message += "Could not reach the server." raise UserError(error_message) from connect_error else: - logger.warning( - "Connection error during cleanup of MCP server '%s': %s", - self.name, + log_tool_action_warning( + logger, + get_mcp_server_log_message( + "Connection error during cleanup of MCP server", self + ), connect_error, ) elif timeout_error: @@ -1051,9 +1080,11 @@ async def cleanup(self): error_message += "Connection timeout." raise UserError(error_message) from timeout_error else: - logger.warning( - "Timeout error during cleanup of MCP server '%s': %s", - self.name, + log_tool_action_warning( + logger, + get_mcp_server_log_message( + "Timeout error during cleanup of MCP server", self + ), timeout_error, ) else: @@ -1063,16 +1094,36 @@ async def cleanup(self): for exc in eg.exceptions ) if has_cancel_scope_error: - logger.debug("Ignoring cancel scope error during cleanup: %s", eg) + log_tool_action_debug( + logger, + get_mcp_server_log_message( + "Ignoring cancel scope error during cleanup of MCP server", self + ), + eg, + ) else: - logger.error("Error cleaning up server: %s", eg) + log_tool_action_error( + logger, + get_mcp_server_log_message("Error cleaning up MCP server", self), + eg, + ) except Exception as e: # Suppress RuntimeError about cancel scopes - this is a known issue with the MCP # library when background tasks fail during async generator cleanup if isinstance(e, RuntimeError) and "cancel scope" in str(e): - logger.debug("Ignoring cancel scope error during cleanup: %s", e) + log_tool_action_debug( + logger, + get_mcp_server_log_message( + "Ignoring cancel scope error during cleanup of MCP server", self + ), + e, + ) else: - logger.error("Error cleaning up server: %s", e) + log_tool_action_error( + logger, + get_mcp_server_log_message("Error cleaning up MCP server", self), + e, + ) finally: self.session = None self._get_session_id = None diff --git a/src/agents/mcp/util.py b/src/agents/mcp/util.py index 8f5f887fa5..498cb5df79 100644 --- a/src/agents/mcp/util.py +++ b/src/agents/mcp/util.py @@ -22,7 +22,7 @@ from mcp.shared.exceptions import McpError as _McpError except ImportError: # pragma: no cover – mcp is optional on Python < 3.10 _McpError = None # type: ignore[assignment, misc] -from ..logger import logger +from ..logger import log_tool_action_error, logger from ..run_context import RunContextWrapper from ..strict_schema import ensure_strict_json_schema from ..tool import ( @@ -41,6 +41,7 @@ from ..tracing import FunctionSpanData, get_current_span, mcp_tools_span from ..util._custom_data import maybe_extract_custom_data from ..util._types import MaybeAwaitable +from ._logging import get_mcp_server_log_message, get_mcp_server_log_name if TYPE_CHECKING: ToolOutputItem = ToolOutputTextDict | ToolOutputImageDict @@ -545,7 +546,10 @@ def to_function_tool( schema = ensure_strict_json_schema(copy.deepcopy(schema)) is_strict = True except Exception as e: - logger.info("Error converting MCP schema to strict mode: %s", e) + if _debug.DONT_LOG_TOOL_DATA: + logger.info("Error converting MCP schema to strict mode") + else: + logger.info("Error converting MCP schema to strict mode: %s", e) needs_approval: ( bool | Callable[[RunContextWrapper[Any], dict[str, Any], str], Awaitable[bool]] @@ -670,7 +674,7 @@ async def invoke_mcp_tool( if json_decode_error is not None: error_message = f"Invalid JSON input for tool {tool_name_for_display}" if _debug.DONT_LOG_TOOL_DATA: - logger.debug(error_message) + logger.debug("Invalid JSON input for MCP tool") raise ModelBehaviorError(error_message) else: error_message = f"{error_message}: {input_json}" @@ -683,7 +687,7 @@ async def invoke_mcp_tool( ) if _debug.DONT_LOG_TOOL_DATA: - logger.debug("Invoking MCP tool %s", tool_name_for_display) + logger.debug("Invoking MCP tool") else: logger.debug("Invoking MCP tool %s with input %s", tool_name_for_display, input_json) @@ -704,27 +708,31 @@ async def invoke_mcp_tool( # pipeline (failure_error_function) can handle it. The default handler # will surface the message as a structured error result; callers who set # failure_error_function=None will have the error raised as documented. - error_text = e.error.message if hasattr(e, "error") and e.error else str(e) - logger.warning( - "MCP tool %s on server '%s' returned an error: %s", - tool_name_for_display, - server.name, - error_text, - ) + if _debug.DONT_LOG_TOOL_DATA: + logger.warning("MCP tool returned an error.") + else: + server_log_name = get_mcp_server_log_name(server.name) + error_text = e.error.message if hasattr(e, "error") and e.error else str(e) + logger.warning( + "MCP tool %s on server '%s' returned an error: %s", + tool_name_for_display, + server_log_name, + error_text, + ) raise - logger.error( - "Error invoking MCP tool %s on server '%s': %s", - tool_name_for_display, - server.name, - e, - ) + log_message = "Error invoking MCP tool" + if not _debug.DONT_LOG_TOOL_DATA: + log_message = get_mcp_server_log_message( + f"Error invoking MCP tool {tool_name_for_display} on server", server + ) + log_tool_action_error(logger, log_message, e) raise AgentsException( f"Error invoking MCP tool {tool_name_for_display} on server '{server.name}': {e}" ) from e if _debug.DONT_LOG_TOOL_DATA: - logger.debug("MCP tool %s completed.", tool_name_for_display) + logger.debug("MCP tool completed.") else: logger.debug("MCP tool %s returned %s", tool_name_for_display, result) @@ -776,8 +784,12 @@ async def invoke_mcp_tool( "server": server.name, } else: - logger.warning( - "Current span is not a FunctionSpanData, skipping tool output: %s", current_span - ) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.warning("Current span is not a FunctionSpanData; skipping tool output") + else: + logger.warning( + "Current span is not a FunctionSpanData, skipping tool output: %s", + current_span, + ) return tool_output diff --git a/src/agents/memory/openai_conversations_session.py b/src/agents/memory/openai_conversations_session.py index 0220eccbb1..9114a7dea0 100644 --- a/src/agents/memory/openai_conversations_session.py +++ b/src/agents/memory/openai_conversations_session.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +from typing import Any from openai import AsyncOpenAI @@ -8,7 +9,7 @@ from ..items import TResponseInputItem from .session import SessionABC -from .session_settings import SessionSettings, resolve_session_limit +from .session_settings import SessionSettings, coerce_session_settings, resolve_session_limit async def start_openai_conversations_session(openai_client: AsyncOpenAI | None = None) -> str: @@ -30,11 +31,15 @@ def __init__( *, conversation_id: str | None = None, openai_client: AsyncOpenAI | None = None, - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, ): self._session_id: str | None = conversation_id self._session_id_lock = asyncio.Lock() - self.session_settings = session_settings or SessionSettings() + self.session_settings = ( + coerce_session_settings(session_settings) + if session_settings is not None + else SessionSettings() + ) _openai_client = openai_client if _openai_client is None: _openai_client = get_default_openai_client() or AsyncOpenAI() diff --git a/src/agents/memory/openai_responses_compaction_session.py b/src/agents/memory/openai_responses_compaction_session.py index 2ec40663db..8263a81036 100644 --- a/src/agents/memory/openai_responses_compaction_session.py +++ b/src/agents/memory/openai_responses_compaction_session.py @@ -7,6 +7,7 @@ from openai import AsyncOpenAI from ..items import TResponseInputItem +from ..logger import log_model_and_tool_action_warning from ..models._openai_shared import get_default_openai_client from ..run_internal.items import normalize_input_items_for_api from .openai_conversations_session import OpenAIConversationsSession @@ -273,10 +274,11 @@ async def _restore_underlying_session_items_after_failed_clear( ) -> None: try: current_items = await self._get_all_underlying_session_items() - except Exception: - logger.warning( + except Exception as inspection_error: + log_model_and_tool_action_warning( + logger, "Failed to inspect session history after compaction replacement clear failed.", - exc_info=True, + inspection_error, ) return @@ -299,15 +301,17 @@ async def _restore_underlying_session_items( await self.underlying_session.clear_session() if previous_items: await self.underlying_session.add_items(list(previous_items)) - except Exception: - logger.warning( + except Exception as restore_error: + log_model_and_tool_action_warning( + logger, "Failed to restore session history after compaction replacement failed.", - exc_info=True, + restore_error, ) return - logger.warning( - "Restored previous session history after compaction replacement failed: %s", + log_model_and_tool_action_warning( + logger, + "Restored previous session history after compaction replacement failed", replacement_error, ) diff --git a/src/agents/memory/session_settings.py b/src/agents/memory/session_settings.py index 03dfbd8d23..eb42f617f2 100644 --- a/src/agents/memory/session_settings.py +++ b/src/agents/memory/session_settings.py @@ -8,16 +8,22 @@ from pydantic.dataclasses import dataclass +from .._config_coercion import ( + _dataclass_input_values, + _declared_dataclass_type, + coerce_dataclass_config, +) + def resolve_session_limit( explicit_limit: int | None, - settings: SessionSettings | None, + settings: SessionSettings | dict[str, Any] | None, ) -> int | None: """Safely resolve the effective limit for session operations.""" if explicit_limit is not None: return explicit_limit if settings is not None: - return settings.limit + return coerce_session_settings(settings).limit return None @@ -32,16 +38,23 @@ class SessionSettings: limit: int | None = None """Maximum number of items to retrieve. If None, retrieves all items.""" - def resolve(self, override: SessionSettings | None) -> SessionSettings: + def resolve(self, override: SessionSettings | dict[str, Any] | None) -> SessionSettings: """Produce a new SessionSettings by overlaying any non-None values from the override on top of this instance.""" if override is None: return self + override_fields = ( + set(_dataclass_input_values(override, type(self))) + if isinstance(override, dict) + else None + ) + override = _coerce_session_settings(override, settings_type=type(self)) changes = { field.name: getattr(override, field.name) for field in fields(self) - if getattr(override, field.name) is not None + if (override_fields is None or field.name in override_fields) + and getattr(override, field.name) is not None } return replace(self, **changes) @@ -49,3 +62,25 @@ def resolve(self, override: SessionSettings | None) -> SessionSettings: def to_dict(self) -> dict[str, Any]: """Convert settings to a dictionary.""" return dataclasses.asdict(self) + + +def coerce_session_settings( + value: SessionSettings | dict[str, Any], +) -> SessionSettings: + """Normalize session settings while preserving existing typed instances.""" + return _coerce_session_settings(value, settings_type=SessionSettings) + + +def _coerce_session_settings( + value: SessionSettings | dict[str, Any], + *, + settings_type: type[SessionSettings], +) -> SessionSettings: + return coerce_dataclass_config(value, settings_type, parameter_name="session") + + +def _declared_session_settings_type( + owner_type: type[Any], + field_name: str, +) -> type[SessionSettings]: + return _declared_dataclass_type(owner_type, field_name, SessionSettings) diff --git a/src/agents/memory/sqlite_session.py b/src/agents/memory/sqlite_session.py index 3a69f9883a..b57f3ebf5a 100644 --- a/src/agents/memory/sqlite_session.py +++ b/src/agents/memory/sqlite_session.py @@ -7,11 +7,11 @@ from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path -from typing import ClassVar +from typing import Any, ClassVar from ..items import TResponseInputItem from .session import SessionABC -from .session_settings import SessionSettings, resolve_session_limit +from .session_settings import SessionSettings, coerce_session_settings, resolve_session_limit class SQLiteSession(SessionABC): @@ -33,7 +33,7 @@ def __init__( db_path: str | Path = ":memory:", sessions_table: str = "agent_sessions", messages_table: str = "agent_messages", - session_settings: SessionSettings | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, ): """Initialize the SQLite session. @@ -47,7 +47,11 @@ def __init__( retrieving items. If None, uses default SessionSettings(). """ self.session_id = session_id - self.session_settings = session_settings or SessionSettings() + self.session_settings = ( + coerce_session_settings(session_settings) + if session_settings is not None + else SessionSettings() + ) self.db_path = db_path self.sessions_table = sessions_table self.messages_table = messages_table diff --git a/src/agents/model_settings.py b/src/agents/model_settings.py index 1a6b7ef963..c4821ed072 100644 --- a/src/agents/model_settings.py +++ b/src/agents/model_settings.py @@ -3,7 +3,7 @@ import dataclasses from collections.abc import Mapping from dataclasses import fields, replace -from typing import Annotated, Any, Literal, TypeAlias, cast +from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeAlias, cast from openai import Omit as _Omit from openai._types import Body, Query @@ -14,6 +14,14 @@ from pydantic.dataclasses import dataclass from pydantic_core import core_schema +from ._config_coercion import _declared_dataclass_type, coerce_dataclass_config +from .retry import ( + ModelRetryBackoffInput, + ModelRetryBackoffSettings, + ModelRetrySettings, + _coerce_backoff_settings, +) + class _OmitTypeAnnotation: @classmethod @@ -193,20 +201,58 @@ class ModelSettings: control which prompt prefixes are eligible for caching. """ - def resolve(self, override: ModelSettings | None) -> ModelSettings: + if TYPE_CHECKING: + + def __init__( + self, + temperature: float | None = None, + top_p: float | None = None, + frequency_penalty: float | None = None, + presence_penalty: float | None = None, + tool_choice: ToolChoice | dict[str, Any] = None, + parallel_tool_calls: bool | None = None, + truncation: Literal["auto", "disabled"] | None = None, + max_tokens: int | None = None, + reasoning: Reasoning | dict[str, Any] | None = None, + verbosity: Literal["low", "medium", "high"] | None = None, + metadata: dict[str, str] | None = None, + store: bool | None = None, + prompt_cache_retention: Literal["in_memory", "24h"] | None = None, + include_usage: bool | None = None, + response_include: list[ResponseIncludable | str] | None = None, + top_logprobs: int | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + extra_headers: Headers | None = None, + extra_args: dict[str, Any] | None = None, + retry: ModelRetrySettings | dict[str, Any] | None = None, + context_management: list[ContextManagement] | None = None, + prompt_cache_options: PromptCacheOptions | None = None, + ) -> None: ... + + def resolve(self, override: ModelSettings | dict[str, Any] | None) -> ModelSettings: """Produce a new ModelSettings by overlaying any non-None values from the override on top of this instance.""" if override is None: return self + override_fields = set(override) if isinstance(override, dict) else None + override = _coerce_model_settings( + override, + parameter_name="ModelSettings override", + model_settings_type=type(self), + ) changes = { field.name: getattr(override, field.name) for field in fields(self) - if getattr(override, field.name) is not None + if (override_fields is None or field.name in override_fields) + and getattr(override, field.name, None) is not None } - # Handle extra_args merging specially - merge dictionaries instead of replacing - if self.extra_args is not None or override.extra_args is not None: + # Handle extra_args merging specially - merge dictionaries instead of replacing. + if (override_fields is None or "extra_args" in override_fields) and ( + self.extra_args is not None or override.extra_args is not None + ): merged_args = {} if self.extra_args: merged_args.update(self.extra_args) @@ -214,6 +260,11 @@ def resolve(self, override: ModelSettings | None) -> ModelSettings: merged_args.update(override.extra_args) changes["extra_args"] = merged_args if merged_args else None + if (override_fields is None or "retry" in override_fields) and ( + self.retry is not None or override.retry is not None + ): + changes["retry"] = _merge_retry_settings(self.retry, override.retry) + return replace(self, **changes) def to_json_dict(self) -> dict[str, Any]: @@ -225,10 +276,90 @@ def to_traceable_dict(self) -> dict[str, Any]: return {key: payload[key] for key in _TRACEABLE_MODEL_SETTING_FIELDS if key in payload} - for field_name, value in dataclass_dict.items(): - if isinstance(value, BaseModel): - json_dict[field_name] = value.model_dump(mode="json") - else: - json_dict[field_name] = value +def _coerce_model_settings( + value: ModelSettings | dict[str, Any], + *, + parameter_name: str, + model_settings_type: type[ModelSettings] = ModelSettings, + inherited_model_settings: ModelSettings | None = None, +) -> ModelSettings: + """Normalize SDK-owned model settings without changing existing typed instances.""" + del inherited_model_settings + if isinstance(value, ModelSettings): + return value + if not isinstance(value, dict): + raise TypeError( + f"{parameter_name} must be a ModelSettings instance or a dict, " + f"got {type(value).__name__}" + ) + + field_names = {model_field.name for model_field in fields(model_settings_type)} + unknown_fields = sorted(str(name) for name in value if name not in field_names) + if unknown_fields: + raise TypeError(f"Unknown model settings: {', '.join(unknown_fields)}") + + _validate_first_party_model_settings(value) + return coerce_dataclass_config(value, model_settings_type, parameter_name=parameter_name) + + +def _declared_model_settings_type( + owner_type: type[Any], + field_name: str, +) -> type[ModelSettings]: + return _declared_dataclass_type(owner_type, field_name, ModelSettings) + + +def _validate_first_party_model_settings(value: dict[str, Any]) -> None: + """Reject SDK-owned structured-setting typos while preserving OpenAI model extras.""" + + def validate_fields(payload: object, names: set[str], path: str) -> None: + if not isinstance(payload, Mapping): + return + unknown_fields = sorted(str(name) for name in payload if name not in names) + if unknown_fields: + raise TypeError(f"Unknown model settings in {path}: {', '.join(unknown_fields)}") + + validate_fields( + value.get("tool_choice"), + {model_field.name for model_field in fields(MCPToolChoice)}, + "tool_choice", + ) + retry = value.get("retry") + validate_fields( + retry, + {model_field.name for model_field in fields(ModelRetrySettings)}, + "retry", + ) + if isinstance(retry, Mapping): + validate_fields( + retry.get("backoff"), + {model_field.name for model_field in fields(ModelRetryBackoffSettings)}, + "retry.backoff", + ) + + context_management = value.get("context_management") + if isinstance(context_management, list | tuple): + for index, item in enumerate(context_management): + validate_fields( + item, + set(ContextManagement.__annotations__), + f"context_management[{index}]", + ) + + validate_fields( + value.get("prompt_cache_options"), + set(PromptCacheOptions.__annotations__), + "prompt_cache_options", + ) + + +def _merge_retry_settings( + inherited: ModelRetrySettings | None, + override: ModelRetrySettings | None, +) -> ModelRetrySettings | None: + if inherited is None: + return override + if override is None: + return inherited return json_dict diff --git a/src/agents/models/chatcmpl_stream_handler.py b/src/agents/models/chatcmpl_stream_handler.py index f365461a06..b960c6542d 100644 --- a/src/agents/models/chatcmpl_stream_handler.py +++ b/src/agents/models/chatcmpl_stream_handler.py @@ -377,6 +377,12 @@ async def buffer_tool_call_stream( if has_passthrough_output: passthrough_choices.append(choice) + elif choice.finish_reason == "content_filter": + # A content-filtered choice ends the stream with an empty delta, so it + # would otherwise be dropped here and the handler would never see the + # finish_reason it needs to synthesize the refusal. Forward a + # delta-stripped copy so buffering semantics are unchanged. + passthrough_choices.append(choice.model_copy(update={"delta": ChoiceDelta()})) if passthrough_choices or chunk.usage is not None: yield chunk.model_copy(update={"choices": passthrough_choices}) diff --git a/src/agents/models/multi_provider.py b/src/agents/models/multi_provider.py index 4737bb8c0c..ccb644edc2 100644 --- a/src/agents/models/multi_provider.py +++ b/src/agents/models/multi_provider.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal, cast +from typing import Any, Literal, cast from openai import AsyncOpenAI @@ -87,7 +87,7 @@ def __init__( openai_websocket_base_url: str | None = None, openai_prefix_mode: MultiProviderOpenAIPrefixMode = "alias", unknown_prefix_mode: MultiProviderUnknownPrefixMode = "error", - openai_agent_registration: OpenAIAgentRegistrationConfig | None = None, + openai_agent_registration: OpenAIAgentRegistrationConfig | dict[str, Any] | None = None, openai_responses_websocket_options: OpenAIResponsesWebSocketOptions | None = None, openai_buffer_streamed_tool_calls: bool = False, ) -> None: diff --git a/src/agents/models/openai_agent_registration.py b/src/agents/models/openai_agent_registration.py index 12e62d8ba0..e0578739bc 100644 --- a/src/agents/models/openai_agent_registration.py +++ b/src/agents/models/openai_agent_registration.py @@ -4,6 +4,8 @@ from dataclasses import dataclass from typing import Any +from .._config_coercion import coerce_dataclass_config + _ENV_HARNESS_ID = "OPENAI_AGENT_HARNESS_ID" OPENAI_HARNESS_ID_TRACE_METADATA_KEY = "agent_harness_id" @@ -22,10 +24,12 @@ class ResolvedOpenAIAgentRegistrationConfig: def set_default_openai_agent_registration_config( - config: OpenAIAgentRegistrationConfig | None, + config: OpenAIAgentRegistrationConfig | dict[str, Any] | None, ) -> None: global _default_agent_registration - _default_agent_registration = config + _default_agent_registration = ( + _coerce_openai_agent_registration_config(config) if config is not None else None + ) def get_default_openai_agent_registration_config() -> OpenAIAgentRegistrationConfig | None: @@ -33,8 +37,10 @@ def get_default_openai_agent_registration_config() -> OpenAIAgentRegistrationCon def resolve_openai_agent_registration_config( - config: OpenAIAgentRegistrationConfig | None, + config: OpenAIAgentRegistrationConfig | dict[str, Any] | None, ) -> ResolvedOpenAIAgentRegistrationConfig | None: + if config is not None: + config = _coerce_openai_agent_registration_config(config) default = get_default_openai_agent_registration_config() harness_id = _resolve_str( explicit=config.harness_id if config else None, @@ -46,6 +52,16 @@ def resolve_openai_agent_registration_config( return ResolvedOpenAIAgentRegistrationConfig(harness_id=harness_id) +def _coerce_openai_agent_registration_config( + config: OpenAIAgentRegistrationConfig | dict[str, Any], +) -> OpenAIAgentRegistrationConfig: + return coerce_dataclass_config( + config, + OpenAIAgentRegistrationConfig, + parameter_name="OpenAI agent registration", + ) + + def resolve_openai_harness_id_for_model_provider(model_provider: Any) -> str | None: """Return the configured harness ID for OpenAI-backed model providers.""" harness_id = _harness_id_from_model_provider(model_provider) diff --git a/src/agents/models/openai_chatcompletions.py b/src/agents/models/openai_chatcompletions.py index c385865224..3e3461145c 100644 --- a/src/agents/models/openai_chatcompletions.py +++ b/src/agents/models/openai_chatcompletions.py @@ -25,7 +25,8 @@ from ..exceptions import ModelBehaviorError, UserError from ..handoffs import Handoff from ..items import ModelResponse, TResponseInputItem, TResponseStreamEvent -from ..logger import logger +from ..logger import log_model_action_debug, logger +from ..retry import ModelRetryAdvice, ModelRetryAdviceRequest from ..tool import Tool from ..tracing import generation_span from ..tracing.span_data import GenerationSpanData @@ -147,7 +148,9 @@ def _consume_background_cleanup_task_result(task: asyncio.Task[Any]) -> None: except asyncio.CancelledError: pass except Exception as exc: - logger.debug("Background stream cleanup failed after cancellation: %s", exc) + log_model_action_debug( + logger, "Background stream cleanup failed after cancellation", exc + ) def _validate_official_openai_input_content_types( self, request_input: str | list[TResponseInputItem] @@ -385,8 +388,10 @@ async def stream_response( await self._maybe_aclose_async_iterator(stream) except Exception as exc: if yielded_terminal_event: - logger.debug( - "Ignoring stream cleanup error after terminal event: %s", exc + log_model_action_debug( + logger, + "Ignoring stream cleanup error after terminal event", + exc, ) else: raise diff --git a/src/agents/models/openai_provider.py b/src/agents/models/openai_provider.py index cec5513a29..e534cd1e7d 100644 --- a/src/agents/models/openai_provider.py +++ b/src/agents/models/openai_provider.py @@ -3,6 +3,7 @@ import asyncio import os import weakref +from typing import Any import httpx from openai import AsyncOpenAI, DefaultAsyncHttpxClient @@ -54,7 +55,7 @@ def __init__( use_responses: bool | None = None, use_responses_websocket: bool | None = None, strict_feature_validation: bool = False, - agent_registration: OpenAIAgentRegistrationConfig | None = None, + agent_registration: OpenAIAgentRegistrationConfig | dict[str, Any] | None = None, responses_websocket_options: OpenAIResponsesWebSocketOptions | None = None, buffer_streamed_tool_calls: bool = False, ) -> None: diff --git a/src/agents/models/openai_responses.py b/src/agents/models/openai_responses.py index dc23de3fd9..527eb91c14 100644 --- a/src/agents/models/openai_responses.py +++ b/src/agents/models/openai_responses.py @@ -50,7 +50,7 @@ from ..exceptions import ModelBehaviorError, UserError from ..handoffs import Handoff from ..items import ItemHelpers, ModelResponse, TResponseInputItem -from ..logger import logger +from ..logger import log_model_action_debug, log_model_action_error, logger from ..model_settings import MCPToolChoice from ..tool import ( ApplyPatchTool, @@ -299,7 +299,9 @@ async def _cleanup_after_exhaustion(self) -> None: await self._cleanup_once() except Exception as exc: if self._yielded_terminal_event: - logger.debug("Ignoring stream cleanup error after terminal event: %s", exc) + log_model_action_debug( + logger, "Ignoring stream cleanup error after terminal event", exc + ) return raise @@ -397,7 +399,9 @@ def _consume_background_cleanup_task_result(task: asyncio.Task[Any]) -> None: except asyncio.CancelledError: pass except Exception as exc: - logger.debug("Background stream cleanup failed after cancellation: %s", exc) + log_model_action_debug( + logger, "Background stream cleanup failed after cancellation", exc + ) async def get_response( self, @@ -451,12 +455,16 @@ async def get_response( SpanError( message="Error getting response", data={ - "error": str(e) if tracing.include_data() else e.__class__.__name__, + "error": str(e) + if tracing.include_data() + else "Error details are redacted.", }, ) ) - request_id = getattr(e, "request_id", None) - logger.error("Error getting response: %s. (request_id: %s)", e, request_id) + message = "Error getting response" + if not _debug.DONT_LOG_MODEL_DATA: + message = f"{message} (request_id: {getattr(e, 'request_id', None)})" + log_model_action_error(logger, message, e) raise return ModelResponse( @@ -541,8 +549,10 @@ async def stream_response( await self._maybe_aclose_async_iterator(stream) except Exception as exc: if yielded_terminal_event: - logger.debug( - "Ignoring stream cleanup error after terminal event: %s", exc + log_model_action_debug( + logger, + "Ignoring stream cleanup error after terminal event", + exc, ) else: raise @@ -562,11 +572,13 @@ async def stream_response( SpanError( message="Error streaming response", data={ - "error": str(e) if tracing.include_data() else e.__class__.__name__, + "error": str(e) + if tracing.include_data() + else "Error details are redacted.", }, ) ) - logger.error("Error streaming response: %s", e) + log_model_action_error(logger, "Error streaming response", e) raise @overload diff --git a/src/agents/realtime/agent.py b/src/agents/realtime/agent.py index 38c77619cc..fe2f9e7200 100644 --- a/src/agents/realtime/agent.py +++ b/src/agents/realtime/agent.py @@ -8,6 +8,7 @@ from agents.prompts import Prompt +from .. import _debug from ..agent import AgentBase from ..guardrail import OutputGuardrail from ..handoffs import Handoff @@ -125,6 +126,11 @@ async def get_system_prompt(self, run_context: RunContextWrapper[TContext]) -> s else: return cast(str, self.instructions(run_context, self)) elif self.instructions is not None: - logger.error("Instructions must be a string or a function, got %s", self.instructions) + if _debug.DONT_LOG_MODEL_DATA: + logger.error("Instructions must be a string or a function") + else: + logger.error( + "Instructions must be a string or a function, got %s", self.instructions + ) return None diff --git a/src/agents/realtime/openai_realtime.py b/src/agents/realtime/openai_realtime.py index 3a60940873..b0a4a63da6 100644 --- a/src/agents/realtime/openai_realtime.py +++ b/src/agents/realtime/openai_realtime.py @@ -199,7 +199,9 @@ def _server_event_validation_summary(error: BaseException) -> str: if isinstance(error, pydantic.ValidationError): return f"{error.error_count()} validation error(s)" - return error.__class__.__name__ + if not _debug.DONT_LOG_MODEL_DATA: + return type(error).__name__ + return "validation failed" def _server_event_identity(event: Any) -> tuple[Any, Any]: @@ -720,6 +722,8 @@ async def send_event(self, event: RealtimeModelSendEvent) -> None: ) else: await self._send_raw_message(converted) + elif _debug.DONT_LOG_MODEL_DATA: + logger.error("Failed to convert raw message type=%s", event.message.get("type")) else: logger.error("Failed to convert raw message: %s", event) elif isinstance(event, RealtimeModelSendUserInput): diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index ba79cc0d07..abfa0b41f7 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -5,11 +5,13 @@ import inspect import json from collections.abc import AsyncIterator, Sequence +from functools import partial from typing import Any, cast from pydantic import BaseModel from typing_extensions import assert_never +from .. import _debug from .._tool_identity import ( FunctionToolLookupKey, get_function_tool_lookup_key_for_tool, @@ -19,7 +21,12 @@ from ..exceptions import ToolInputGuardrailTripwireTriggered, UserError from ..handoffs import Handoff from ..items import ToolApprovalItem -from ..logger import logger +from ..logger import ( + log_model_action_error, + log_model_and_tool_action_warning, + log_tool_action_error, + logger, +) from ..run_config import ToolErrorFormatterArgs from ..run_context import RunContextWrapper, TContext from ..tool import DEFAULT_APPROVAL_REJECTION_MESSAGE, FunctionTool, Tool, invoke_function_tool @@ -86,6 +93,18 @@ class _RealtimeSessionClosedSentinel: _BACKGROUND_TASK_CANCEL_GRACE_SECONDS = 1.0 +def _guardrail_diagnostic_extra(guardrail: Any) -> dict[str, object]: + try: + return {"guardrail_name": guardrail.get_name()} + except Exception: + try: + guardrail_type = type(guardrail.guardrail_function) + type_name = f"{guardrail_type.__module__}.{guardrail_type.__qualname__}" + except Exception: + type_name = "unknown" + return {"guardrail_type": type_name} + + def _serialize_tool_output(output: Any) -> str: """Serialize structured tool outputs to JSON when possible.""" if isinstance(output, str): @@ -476,8 +495,8 @@ async def on_event(self, event: RealtimeModelEvent) -> None: if new_content: incoming_item = incoming_item.model_copy(update={"content": new_content}) - except Exception: - logger.error("Error merging transcripts", exc_info=True) + except Exception as exc: + log_model_action_error(logger, "Error merging transcripts", exc) pass self._history = self._get_new_history(self._history, incoming_item) @@ -802,18 +821,21 @@ async def _resolve_approval_rejection_message(self, *, tool: FunctionTool, call_ ) message = await maybe_message if inspect.isawaitable(maybe_message) else maybe_message except Exception as exc: - logger.error("Tool error formatter failed for %s: %s", tool.name, exc) + log_tool_action_error(logger, "Tool error formatter failed", exc) return REJECTION_MESSAGE if message is None: return REJECTION_MESSAGE if not isinstance(message, str): - logger.error( - "Tool error formatter returned non-string for %s: %s", - tool.name, - type(message).__name__, - ) + if _debug.DONT_LOG_TOOL_DATA: + logger.error("Tool error formatter returned a non-string value") + else: + logger.error( + "Tool error formatter returned non-string for %s: %s", + tool.name, + type(message).__name__, + ) return REJECTION_MESSAGE return message @@ -1298,13 +1320,12 @@ async def _run_output_guardrails(self, text: str, response_id: str) -> bool: if result.output.tripwire_triggered: triggered_results.append(result) except Exception as exc: - logger.warning( - "Output guardrail %r raised %s: %s; skipping it.", - guardrail.get_name(), - type(exc).__name__, + log_model_and_tool_action_warning( + logger, + "Output guardrail raised an exception; skipping it", exc, + diagnostic_extra=partial(_guardrail_diagnostic_extra, guardrail), ) - logger.debug("Output guardrail failure details.", exc_info=True) continue if triggered_results: @@ -1418,11 +1439,14 @@ def _on_tool_call_task_done(self, task: asyncio.Task[Any]) -> None: return if isinstance(exception, _PendingToolOutputSendError): - logger.warning( - "Realtime tool output send failed for call %s; cached output will be retried", - exception.call_id, - exc_info=exception, - ) + if _debug.DONT_LOG_TOOL_DATA: + logger.warning("Realtime tool output send failed; cached output will be retried") + else: + logger.warning( + "Realtime tool output send failed for call %s; cached output will be retried", + exception.call_id, + exc_info=exception, + ) self._put_event_nowait( RealtimeError( info=self._event_info, @@ -1435,7 +1459,7 @@ def _on_tool_call_task_done(self, task: asyncio.Task[Any]) -> None: ) return - logger.exception("Realtime tool call task failed", exc_info=exception) + log_tool_action_error(logger, "Realtime tool call task failed", exception) if self._stored_exception is None: self._stored_exception = exception diff --git a/src/agents/responses_websocket_session.py b/src/agents/responses_websocket_session.py index 3d0f18137d..b1ac69d938 100644 --- a/src/agents/responses_websocket_session.py +++ b/src/agents/responses_websocket_session.py @@ -3,7 +3,7 @@ from collections.abc import AsyncIterator, Mapping from contextlib import asynccontextmanager from dataclasses import dataclass -from typing import Any +from typing import TYPE_CHECKING, Any from .agent import Agent from .items import TResponseInputItem @@ -16,7 +16,7 @@ from .models.openai_responses import OpenAIResponsesWebSocketOptions from .result import RunResult, RunResultStreaming from .run import Runner -from .run_config import RunConfig +from .run_config import RunConfig, _coerce_run_config from .run_state import RunState @@ -27,7 +27,16 @@ class ResponsesWebSocketSession: provider: OpenAIProvider run_config: RunConfig + if TYPE_CHECKING: + + def __init__( + self, + provider: OpenAIProvider, + run_config: RunConfig | dict[str, Any], + ) -> None: ... + def __post_init__(self) -> None: + object.__setattr__(self, "run_config", _coerce_run_config(self.run_config)) self._validate_provider_alignment() def _validate_provider_alignment(self) -> MultiProvider: diff --git a/src/agents/result.py b/src/agents/result.py index f63a8e7f68..7bccccec91 100644 --- a/src/agents/result.py +++ b/src/agents/result.py @@ -28,7 +28,7 @@ ToolApprovalItem, TResponseInputItem, ) -from .logger import logger +from .logger import log_tool_action_warning, logger from .run_context import RunContextWrapper from .run_internal.items import ( NestedHistoryOwnedItemRef, @@ -661,8 +661,10 @@ async def _cleanup_once() -> None: try: await sandbox_cleanup() except Exception as error: - logger.warning( - "Failed to clean up sandbox resources after streamed run: %s", error + log_tool_action_warning( + logger, + "Failed to clean up sandbox resources after streamed run", + error, ) task = asyncio.create_task(_cleanup_once()) diff --git a/src/agents/run.py b/src/agents/run.py index ad3aa02d43..47a00cd23f 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -3,7 +3,7 @@ import asyncio import contextlib import warnings -from typing import cast +from typing import Any, cast from typing_extensions import Unpack @@ -27,7 +27,7 @@ TResponseInputItem, ) from .lifecycle import RunHooks -from .logger import logger +from .logger import log_model_and_tool_action_warning, log_tool_action_warning, logger from .memory import Session from .result import RunResult, RunResultStreaming from .run_config import ( @@ -42,6 +42,7 @@ ToolErrorFormatterArgs, ToolExecutionConfig, ToolNotFoundBehavior, + _coerce_run_config, ) from .run_context import RunContextWrapper, TContext from .run_error_handlers import RunErrorHandlers @@ -208,7 +209,7 @@ async def run( context: TContext | None = None, max_turns: int | None = DEFAULT_MAX_TURNS, hooks: RunHooks[TContext] | None = None, - run_config: RunConfig | None = None, + run_config: RunConfig | dict[str, Any] | None = None, error_handlers: RunErrorHandlers[TContext] | None = None, previous_response_id: str | None = None, auto_previous_response_id: bool = False, @@ -292,7 +293,7 @@ def run_sync( context: TContext | None = None, max_turns: int | None = DEFAULT_MAX_TURNS, hooks: RunHooks[TContext] | None = None, - run_config: RunConfig | None = None, + run_config: RunConfig | dict[str, Any] | None = None, error_handlers: RunErrorHandlers[TContext] | None = None, previous_response_id: str | None = None, auto_previous_response_id: bool = False, @@ -373,7 +374,7 @@ def run_streamed( context: TContext | None = None, max_turns: int | None = DEFAULT_MAX_TURNS, hooks: RunHooks[TContext] | None = None, - run_config: RunConfig | None = None, + run_config: RunConfig | dict[str, Any] | None = None, previous_response_id: str | None = None, auto_previous_response_id: bool = False, conversation_id: str | None = None, @@ -467,8 +468,7 @@ async def run( conversation_id = kwargs.get("conversation_id") session = kwargs.get("session") - if run_config is None: - run_config = RunConfig() + run_config = RunConfig() if run_config is None else _coerce_run_config(run_config) is_resumed_state = isinstance(input, RunState) run_state: RunState[TContext] | None = None @@ -1595,10 +1595,14 @@ def _finalize_result(result: RunResult) -> RunResult: terminal_metadata=terminal_metadata_for_exception(run_exception), ) except Exception as error: - logger.warning("Failed to enqueue sandbox memory after run: %s", error) + log_model_and_tool_action_warning( + logger, "Failed to enqueue sandbox memory after run", error + ) sandbox_resume_state = await sandbox_runtime.cleanup() except Exception as error: - logger.warning("Failed to clean up sandbox resources after run: %s", error) + log_tool_action_warning( + logger, "Failed to clean up sandbox resources after run", error + ) else: if completed_result is not None: completed_result._sandbox_resume_state = sandbox_resume_state @@ -1608,7 +1612,7 @@ def _finalize_result(result: RunResult) -> RunResult: try: await dispose_resolved_computers(run_context=context_wrapper) except Exception as error: - logger.warning("Failed to dispose computers after run: %s", error) + log_tool_action_warning(logger, "Failed to dispose computers after run", error) if current_span: current_span.finish(reset_current=True) if current_task_span: @@ -1717,8 +1721,7 @@ def run_streamed( conversation_id = kwargs.get("conversation_id") session = kwargs.get("session") - if run_config is None: - run_config = RunConfig() + run_config = RunConfig() if run_config is None else _coerce_run_config(run_config) # Handle RunState input is_resumed_state = isinstance(input, RunState) diff --git a/src/agents/run_config.py b/src/agents/run_config.py index 08ee4cff9e..393e6dd039 100644 --- a/src/agents/run_config.py +++ b/src/agents/run_config.py @@ -5,14 +5,24 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Generic, Literal +from pydantic import TypeAdapter from typing_extensions import NotRequired, TypedDict +from ._config_coercion import ( + _declared_dataclass_type, + coerce_dataclass_config, + coerce_pydantic_config, +) from .guardrail import InputGuardrail, OutputGuardrail from .handoffs import HandoffHistoryMapper, HandoffInputFilter from .items import TResponseInputItem from .lifecycle import RunHooks from .memory import Session, SessionInputCallback, SessionSettings -from .model_settings import ModelSettings +from .memory.session_settings import ( + _coerce_session_settings, + _declared_session_settings_type, +) +from .model_settings import ModelSettings, _coerce_model_settings, _declared_model_settings_type from .models.interface import Model, ModelProvider from .models.multi_provider import MultiProvider from .run_context import TContext @@ -207,6 +217,86 @@ class SandboxRunConfig: Use `SandboxArchiveLimits()` to enable SDK defaults. """ + if TYPE_CHECKING: + + def __init__( + self, + client: BaseSandboxClient[Any] | None = None, + options: Any | None = None, + session: BaseSandboxSession | None = None, + session_state: SandboxSessionState | None = None, + manifest: Manifest | dict[str, Any] | None = None, + snapshot: SnapshotSpec | SnapshotBase | dict[str, Any] | None = None, + concurrency_limits: SandboxConcurrencyLimits | dict[str, Any] = ..., + archive_limits: SandboxArchiveLimits | dict[str, Any] | None = None, + ) -> None: ... + + def __post_init__(self) -> None: + if isinstance(self.manifest, dict): + from .sandbox.manifest import _coerce_manifest + + self.manifest = _coerce_manifest(self.manifest, parameter_name="sandbox.manifest") + if isinstance(self.snapshot, dict): + from .sandbox.snapshot import SnapshotBase, SnapshotSpecUnion + + if "id" in self.snapshot: + self.snapshot = SnapshotBase.parse(self.snapshot) + else: + self.snapshot = TypeAdapter(SnapshotSpecUnion).validate_python(self.snapshot) + if isinstance(self.options, dict) and self.client is not None: + from .sandbox.session.sandbox_client import BaseSandboxClientOptions + + options_type = BaseSandboxClientOptions._options_class_for_type(self.client.backend_id) + if options_type is not None: + options = self.options + explicit_type = options.get("type") + if explicit_type is not None and explicit_type != self.client.backend_id: + raise ValueError( + f"sandbox.options type `{explicit_type}` does not match selected " + f"sandbox client backend `{self.client.backend_id}`" + ) + if "type" not in options: + options = { + **options, + "type": options_type.model_fields["type"].default, + } + self.options = coerce_pydantic_config( + options, + options_type, + parameter_name="sandbox.options", + ) + elif self.client.backend_id == "blaxel": + from .extensions.sandbox.blaxel.sandbox import ( + BlaxelSandboxClient, + BlaxelSandboxClientOptions, + ) + + if isinstance(self.client, BlaxelSandboxClient): + self.options = coerce_dataclass_config( + self.options, + BlaxelSandboxClientOptions, + parameter_name="sandbox.options", + ) + self.concurrency_limits = coerce_dataclass_config( + self.concurrency_limits, + _declared_dataclass_type( + type(self), + "concurrency_limits", + SandboxConcurrencyLimits, + ), + parameter_name="sandbox.concurrency_limits", + ) + if self.archive_limits is not None: + self.archive_limits = coerce_dataclass_config( + self.archive_limits, + _declared_dataclass_type( + type(self), + "archive_limits", + SandboxArchiveLimits, + ), + parameter_name="sandbox.archive_limits", + ) + @dataclass class RunConfig: @@ -222,7 +312,7 @@ class RunConfig: model_settings: ModelSettings | None = None """Configure global model settings. Any non-null values will override the agent-specific model - settings. + settings. Accepts a ``ModelSettings`` instance or a dictionary containing its fields. """ handoff_input_filter: HandoffInputFilter | None = None @@ -339,6 +429,64 @@ class RunConfig: the run continue. """ + if TYPE_CHECKING: + + def __init__( + self, + model: str | Model | None = None, + model_provider: ModelProvider = ..., + model_settings: ModelSettings | dict[str, Any] | None = None, + handoff_input_filter: HandoffInputFilter | None = None, + nest_handoff_history: bool = False, + handoff_history_mapper: HandoffHistoryMapper | None = None, + input_guardrails: list[InputGuardrail[Any]] | None = None, + output_guardrails: list[OutputGuardrail[Any]] | None = None, + tracing_disabled: bool = False, + tracing: TracingConfig | None = None, + trace_include_sensitive_data: bool = ..., + workflow_name: str = "Agent workflow", + trace_id: str | None = None, + group_id: str | None = None, + trace_metadata: dict[str, Any] | None = None, + session_input_callback: SessionInputCallback | None = None, + call_model_input_filter: CallModelInputFilter | None = None, + tool_error_formatter: ToolErrorFormatter | None = None, + session_settings: SessionSettings | dict[str, Any] | None = None, + reasoning_item_id_policy: ReasoningItemIdPolicy | None = None, + sandbox: SandboxRunConfig | dict[str, Any] | None = None, + tool_execution: ToolExecutionConfig | dict[str, Any] | None = None, + tool_not_found_behavior: ToolNotFoundBehavior = "raise_error", + ) -> None: ... + + def __post_init__(self) -> None: + if self.model_settings is not None: + self.model_settings = _coerce_model_settings( + self.model_settings, + parameter_name="RunConfig model_settings", + model_settings_type=_declared_model_settings_type(type(self), "model_settings"), + ) + if self.session_settings is not None: + self.session_settings = _coerce_session_settings( + self.session_settings, + settings_type=_declared_session_settings_type(type(self), "session_settings"), + ) + if self.sandbox is not None: + self.sandbox = coerce_dataclass_config( + self.sandbox, + _declared_dataclass_type(type(self), "sandbox", SandboxRunConfig), + parameter_name="run_config.sandbox", + ) + if self.tool_execution is not None: + self.tool_execution = coerce_dataclass_config( + self.tool_execution, + _declared_dataclass_type( + type(self), + "tool_execution", + ToolExecutionConfig, + ), + parameter_name="run_config.tool_execution", + ) + class RunOptions(TypedDict, Generic[TContext]): """Arguments for ``AgentRunner`` methods.""" @@ -352,7 +500,7 @@ class RunOptions(TypedDict, Generic[TContext]): hooks: NotRequired[RunHooks[TContext] | None] """Lifecycle hooks for the run.""" - run_config: NotRequired[RunConfig | None] + run_config: NotRequired[RunConfig | dict[str, Any] | None] """Run configuration.""" previous_response_id: NotRequired[str | None] @@ -371,6 +519,11 @@ class RunOptions(TypedDict, Generic[TContext]): """Error handlers keyed by error kind.""" +def _coerce_run_config(value: RunConfig | dict[str, Any]) -> RunConfig: + """Normalize run configuration dictionaries at public runner boundaries.""" + return coerce_dataclass_config(value, RunConfig, parameter_name="run_config") + + __all__ = [ "DEFAULT_MAX_TURNS", "CallModelData", diff --git a/src/agents/run_internal/items.py b/src/agents/run_internal/items.py index 9d558a54e5..255d5f2f00 100644 --- a/src/agents/run_internal/items.py +++ b/src/agents/run_internal/items.py @@ -74,6 +74,7 @@ "deduplicate_input_items", "deduplicate_input_items_preferring_latest", "strip_internal_input_item_metadata", + "function_tool_error_output", "function_rejection_item", "shell_rejection_item", "apply_patch_rejection_item", @@ -714,22 +715,49 @@ def deduplicate_input_items_preferring_latest( return list(reversed(deduplicate_input_items(list(reversed(items))))) +def function_tool_error_output( + tool_call: Any, + output: Any, + *, + output_json_schema: dict[str, Any] | None, +) -> Any: + """Encode SDK-generated programmatic tool errors as provider-compatible JSON objects.""" + if output_json_schema is None or not isinstance(output, str): + return output + + if isinstance(tool_call, dict): + caller = tool_call.get("caller") + else: + caller = getattr(tool_call, "caller", None) + caller_type = caller.get("type") if isinstance(caller, dict) else getattr(caller, "type", None) + if caller_type != "program": + return output + + return json.dumps({"error": output}, ensure_ascii=False, separators=(",", ":")) + + def function_rejection_item( agent: Any, tool_call: Any, *, rejection_message: str = REJECTION_MESSAGE, + output_json_schema: dict[str, Any] | None = None, scope_id: str | None = None, tool_origin: Any = None, ) -> ToolCallOutputItem: """Build a ToolCallOutputItem representing a rejected function tool call.""" if isinstance(tool_call, ResponseFunctionToolCall): drop_agent_tool_run_result(tool_call, scope_id=scope_id) + provider_output = function_tool_error_output( + tool_call, + rejection_message, + output_json_schema=output_json_schema, + ) return ToolCallOutputItem( output=rejection_message, raw_item=ItemHelpers.tool_call_output_item( tool_call, - rejection_message, + provider_output, ), agent=agent, tool_origin=tool_origin, diff --git a/src/agents/run_internal/model_retry.py b/src/agents/run_internal/model_retry.py index aa41d07e6b..4e37139329 100644 --- a/src/agents/run_internal/model_retry.py +++ b/src/agents/run_internal/model_retry.py @@ -10,7 +10,7 @@ from openai import APIConnectionError, APITimeoutError, BadRequestError from ..items import ModelResponse, TResponseStreamEvent -from ..logger import logger +from ..logger import log_model_action_debug, logger from ..models._retry_runtime import ( get_error_code as _get_error_code, get_request_id as _get_request_id, @@ -246,7 +246,7 @@ async def _close_async_iterator_quietly(iterator: Any | None) -> None: try: await _close_async_iterator(iterator) except Exception as exc: - logger.debug("Ignoring retry stream cleanup error: %s", exc) + log_model_action_debug(logger, "Ignoring retry stream cleanup error", exc) def _get_stream_event_type(event: TResponseStreamEvent) -> str | None: diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 9113b2deda..88227c06c9 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -9,6 +9,7 @@ import dataclasses as _dc import json from collections.abc import Awaitable, Callable, Mapping +from functools import partial from typing import Any, TypeVar, cast from openai.types.responses import ( @@ -55,7 +56,13 @@ coerce_tool_search_output_raw_item, ) from ..lifecycle import RunHooks -from ..logger import logger +from ..logger import ( + log_model_action_error, + log_model_action_warning, + log_model_and_tool_action_debug, + log_tool_action_warning, + logger, +) from ..memory import Session from ..models._response_terminal import ( response_error_event_failure_error, @@ -270,7 +277,11 @@ async def cleanup_models_after_run(tool_use_tracker: AgentToolUseTracker) -> Non try: await model._cleanup_on_run_end(tool_use_tracker) except Exception as error: - logger.warning("Failed to clean up model resources after run: %s", error) + log_model_action_warning(logger, "Failed to clean up model resources after run", error) + + +def _agent_diagnostic_extra(agent: Agent[Any]) -> dict[str, object]: + return {"agent_name": agent.name} def _should_attach_generic_agent_error(exc: Exception) -> bool: @@ -384,8 +395,12 @@ async def _run_output_guardrails_for_stream( try: return cast(list[Any], await streamed_result._output_guardrails_task) - except Exception: - logger.error("Unexpected error in output guardrails", exc_info=True) + except OutputGuardrailTripwireTriggered: + raise + except asyncio.CancelledError: + raise + except Exception as exc: + log_model_action_error(logger, "Unexpected error in output guardrails", exc) raise @@ -1288,13 +1303,16 @@ async def _save_stream_items_without_count( if first_trigger is not None: raise InputGuardrailTripwireTriggered(first_trigger) except Exception as e: - logger.debug( - "Error in streamed_result finalize for agent %s - %s", current_agent.name, e + log_model_and_tool_action_debug( + logger, + "Error finalizing streamed result", + e, + diagnostic_extra=partial(_agent_diagnostic_extra, current_agent), ) try: await dispose_resolved_computers(run_context=context_wrapper) except Exception as error: - logger.warning("Failed to dispose computers after streamed run: %s", error) + log_tool_action_warning(logger, "Failed to dispose computers after streamed run", error) if current_span: current_span.finish(reset_current=True) if current_task_span: @@ -1951,9 +1969,9 @@ async def get_new_response( model_settings = model_settings_with_prompt_cache_key(model_settings, prompt_cache_key) async def rewind_model_request() -> None: - items_to_rewind = session_items_to_rewind if session_items_to_rewind is not None else [] - await rewind_session_items(session, items_to_rewind, server_conversation_tracker) if server_conversation_tracker is not None: + items_to_rewind = session_items_to_rewind if session_items_to_rewind is not None else [] + await rewind_session_items(session, items_to_rewind, server_conversation_tracker) server_conversation_tracker.rewind_input(filtered.input) with model_run_context(tool_use_tracker): diff --git a/src/agents/run_internal/session_persistence.py b/src/agents/run_internal/session_persistence.py index 7e68ebc300..f4500bef15 100644 --- a/src/agents/run_internal/session_persistence.py +++ b/src/agents/run_internal/session_persistence.py @@ -13,9 +13,14 @@ from collections.abc import Sequence from typing import Any, cast +from .. import _debug from ..exceptions import UserError from ..items import HandoffOutputItem, ItemHelpers, RunItem, ToolCallOutputItem, TResponseInputItem -from ..logger import logger +from ..logger import ( + log_model_and_tool_action_debug, + log_model_and_tool_action_warning, + logger, +) from ..memory import ( OpenAIResponsesCompactionArgs, Session, @@ -535,8 +540,9 @@ async def rewind_session_items( len(target_serializations), ) - for i, target in enumerate(target_serializations): - logger.debug("Rewind target %d (first 300 chars): %s", i, target[:300]) + if not (_debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA): + for i, target in enumerate(target_serializations): + logger.debug("Rewind target %d (first 300 chars): %s", i, target[:300]) snapshot_serializations = target_serializations.copy() rewound = await _rewind_session_tail_suffix( @@ -547,7 +553,7 @@ async def rewind_session_items( mismatch_warning=( "Skipping session rewind because the current tail does not match the retry-owned suffix" ), - pop_failure_warning="Failed to rewind session item: %s", + pop_failure_warning="Failed to rewind session item", ) if not rewound: return @@ -564,7 +570,7 @@ async def rewind_session_items( try: latest_items = await session.get_items(limit=1) except Exception as exc: - logger.debug("Failed to peek session items while rewinding: %s", exc) + log_model_and_tool_action_debug(logger, "Failed to peek session items while rewinding", exc) return if not latest_items: @@ -577,7 +583,9 @@ async def rewind_session_items( try: session_items = await session.get_items() except Exception as exc: - logger.debug("Failed to inspect session tail while stripping stray items: %s", exc) + log_model_and_tool_action_debug( + logger, "Failed to inspect session tail while stripping stray items", exc + ) return stray_serializations = _collect_retry_owned_tail_serializations( @@ -602,7 +610,7 @@ async def rewind_session_items( "Skipping stray session cleanup because the current tail no longer matches " "retry-owned conversation items" ), - pop_failure_warning="Failed to strip stray session item: %s", + pop_failure_warning="Failed to strip stray session item", ) @@ -626,7 +634,9 @@ async def wait_for_session_cleanup( try: tail_items = await session.get_items(limit=window) except Exception as exc: - logger.debug("Failed to verify session cleanup (attempt %d): %s", attempt + 1, exc) + log_model_and_tool_action_debug( + logger, f"Failed to verify session cleanup (attempt {attempt + 1})", exc + ) await asyncio.sleep(0.1 * (attempt + 1)) continue @@ -757,7 +767,7 @@ async def _rewind_session_tail_suffix( try: tail_items = await session.get_items(limit=len(expected_serializations)) except Exception as exc: - logger.warning(pop_failure_warning, exc) + log_model_and_tool_action_warning(logger, pop_failure_warning, exc) return False if len(tail_items) != len(expected_serializations): @@ -784,7 +794,7 @@ async def _rewind_session_tail_suffix( result = await result except Exception as exc: await _restore_popped_session_items(session, popped_items) - logger.warning(pop_failure_warning, exc) + log_model_and_tool_action_warning(logger, pop_failure_warning, exc) return False if result is None: @@ -820,7 +830,9 @@ async def _restore_popped_session_items( if inspect.isawaitable(result): await result except Exception as exc: - logger.warning("Failed to restore session items after a rewind mismatch: %s", exc) + log_model_and_tool_action_warning( + logger, "Failed to restore session items after a rewind mismatch", exc + ) def _collect_retry_owned_tail_serializations( diff --git a/src/agents/run_internal/tool_actions.py b/src/agents/run_internal/tool_actions.py index ee3d685997..0421c15c43 100644 --- a/src/agents/run_internal/tool_actions.py +++ b/src/agents/run_internal/tool_actions.py @@ -48,6 +48,7 @@ extract_apply_patch_call_id, format_shell_error, get_trace_tool_error, + log_tool_action_error, normalize_apply_patch_result, normalize_max_output_length, normalize_shell_output, @@ -152,7 +153,7 @@ async def _run_action(span: Any | None) -> RunItem: }, ) ) - logger.error("Failed to execute computer action: %s", exc, exc_info=True) + log_tool_action_error("Failed to execute computer action", exc) raise image_url = f"data:image/png;base64,{output}" if output else "" @@ -557,7 +558,7 @@ async def _run_call(span: Any | None) -> RunItem: if requested_max_output_length is not None: max_output_length = requested_max_output_length output_text = output_text[:max_output_length] - logger.error("Shell executor failed: %s", exc, exc_info=True) + log_tool_action_error("Shell executor failed", exc) await asyncio.gather( hooks.on_tool_end(context_wrapper, agent, call.shell_tool, output_text), @@ -713,7 +714,7 @@ async def _run_call(span: Any | None) -> RunItem: }, ) ) - logger.error("Custom tool failed: %s", exc, exc_info=True) + log_tool_action_error("Custom tool failed", exc) raw_item = cls._raw_tool_output_item( call_id, @@ -920,7 +921,7 @@ async def _run_call(span: Any | None) -> RunItem: }, ) ) - logger.error("Apply patch editor failed: %s", exc, exc_info=True) + log_tool_action_error("Apply patch editor failed", exc) raw_item: dict[str, Any] = { "type": "apply_patch_call_output", diff --git a/src/agents/run_internal/tool_execution.py b/src/agents/run_internal/tool_execution.py index 91dc1754fd..c38bdfe4e5 100644 --- a/src/agents/run_internal/tool_execution.py +++ b/src/agents/run_internal/tool_execution.py @@ -21,6 +21,7 @@ from openai.types.responses.response_input_param import McpApprovalResponse from openai.types.responses.response_output_item import McpApprovalRequest +from .. import _debug from .._tool_identity import ( FunctionToolLookupKey, NamedToolLookupKey, @@ -57,7 +58,7 @@ ToolApprovalItem, ToolCallOutputItem, ) -from ..logger import logger +from ..logger import log_tool_action_error as _log_tool_action_error, logger from ..model_settings import ModelSettings from ..run_config import RunConfig, ToolErrorFormatterArgs from ..run_context import RunContextWrapper @@ -103,6 +104,7 @@ extract_mcp_request_id, extract_mcp_request_id_from_run, function_rejection_item, + function_tool_error_output, ) from .run_steps import ToolRunFunction from .tool_use_tracker import AgentToolUseTracker @@ -1039,6 +1041,31 @@ def format_shell_error(error: Exception | BaseException | Any) -> str: return repr(error) +def _tool_name_diagnostic_extra(tool_name: str) -> dict[str, object]: + return {"tool_name": tool_name} + + +def log_tool_action_error( + message: str, + exc: Exception | BaseException, + *, + diagnostic_extra: Callable[[], Mapping[str, object]] | None = None, +) -> None: + """Log a tool-action failure without leaking tool data. + + Tool exceptions can embed tool call arguments or output, so the exception is + redacted by default (matching ``_debug.DONT_LOG_TOOL_DATA``). The full exception + and traceback are logged only when tool-data logging is explicitly enabled. + """ + _log_tool_action_error( + logger, + message, + exc, + stacklevel=4, + diagnostic_extra=diagnostic_extra, + ) + + async def with_tool_function_span( *, config: RunConfig, @@ -1186,18 +1213,25 @@ async def resolve_approval_rejection_message( ) message = await maybe_message if inspect.isawaitable(maybe_message) else maybe_message except Exception as exc: - logger.error("Tool error formatter failed for %s: %s", tool_name, exc) + log_tool_action_error( + "Tool error formatter failed", + exc, + diagnostic_extra=functools.partial(_tool_name_diagnostic_extra, tool_name), + ) return REJECTION_MESSAGE if message is None: return REJECTION_MESSAGE if not isinstance(message, str): - logger.error( - "Tool error formatter returned non-string for %s: %s", - tool_name, - type(message).__name__, - ) + if _debug.DONT_LOG_TOOL_DATA: + logger.error("Tool error formatter returned a non-string value") + else: + logger.error( + "Tool error formatter returned non-string for %s: %s", + tool_name, + type(message).__name__, + ) return REJECTION_MESSAGE return message @@ -1715,6 +1749,7 @@ async def _maybe_execute_tool_approval( self.public_agent, tool_call, rejection_message=rejected_message, + output_json_schema=func_tool.output_json_schema, scope_id=self.tool_state_scope_id, tool_origin=get_function_tool_origin(func_tool), ), @@ -1764,6 +1799,7 @@ async def _maybe_execute_tool_approval( self.public_agent, tool_call, rejection_message=rejection_message, + output_json_schema=func_tool.output_json_schema, scope_id=self.tool_state_scope_id, tool_origin=get_function_tool_origin(func_tool), ), @@ -1877,9 +1913,18 @@ async def _invoke_tool_and_run_post_invoke( bypass_output_schema = bypass_output_schema or (output_guardrail_result.is_rejection) if bypass_output_schema: self.schema_bypassed_tool_runs.add(id(task_state.tool_run)) + provider_result = ( + function_tool_error_output( + tool_call, + final_result, + output_json_schema=func_tool.output_json_schema, + ) + if bypass_output_schema + else final_result + ) raw_output_item = ItemHelpers.tool_call_output_item( tool_call, - final_result, + provider_result, output_json_schema=None if bypass_output_schema else func_tool.output_json_schema, output_type_adapter=None if bypass_output_schema else func_tool._output_type_adapter, ) @@ -1998,11 +2043,20 @@ def _build_function_tool_results(self) -> list[FunctionToolResult]: run_item: RunItem | None if not nested_interruptions: + provider_result = ( + function_tool_error_output( + tool_run.tool_call, + result, + output_json_schema=tool_run.function_tool.output_json_schema, + ) + if bypass_output_schema + else result + ) run_item = ToolCallOutputItem( output=result, raw_item=ItemHelpers.tool_call_output_item( tool_run.tool_call, - result, + provider_result, output_json_schema=( None if bypass_output_schema diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 3202dee3a8..d6d87f447a 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -29,6 +29,7 @@ ) from openai.types.responses.response_reasoning_item import ResponseReasoningItem +from .. import _debug from .._mcp_tool_metadata import collect_mcp_list_tools_metadata from .._tool_identity import ( build_function_tool_lookup_map, @@ -72,7 +73,7 @@ coerce_tool_search_output_raw_item, ) from ..lifecycle import RunHooks -from ..logger import logger +from ..logger import log_tool_action_error, logger from ..run_config import RunConfig, ToolErrorFormatterArgs from ..run_context import AgentHookContext, RunContextWrapper, TContext from ..run_error_handlers import RunErrorHandlers @@ -252,18 +253,21 @@ async def _resolve_tool_not_found_message( ) message = await maybe_message if inspect.isawaitable(maybe_message) else maybe_message except Exception as exc: - logger.error("Tool error formatter failed for missing tool %s: %s", tool_name, exc) + log_tool_action_error(logger, "Tool error formatter failed for missing tool", exc) return default_message if message is None: return default_message if not isinstance(message, str): - logger.error( - "Tool error formatter returned non-string for missing tool %s: %s", - tool_name, - type(message).__name__, - ) + if _debug.DONT_LOG_TOOL_DATA: + logger.error("Tool error formatter returned a non-string value for a missing tool") + else: + logger.error( + "Tool error formatter returned non-string for missing tool %s: %s", + tool_name, + type(message).__name__, + ) return default_message return message @@ -1047,6 +1051,7 @@ async def _record_function_rejection( public_agent, tool_call, rejection_message=rejection_message, + output_json_schema=function_tool.output_json_schema, scope_id=tool_state_scope_id, tool_origin=get_function_tool_origin(function_tool), ) diff --git a/src/agents/run_state.py b/src/agents/run_state.py index ccefd91235..dc96ddf4ba 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -81,7 +81,7 @@ coerce_tool_search_call_raw_item, coerce_tool_search_output_raw_item, ) -from .logger import logger +from .logger import log_model_and_tool_action_warning, logger from .run_context import RunContextWrapper from .run_internal.items import ( NestedHistoryOwnedItemRef, @@ -3731,7 +3731,9 @@ def _resolve_agent_info( except UserError: raise except Exception as e: - logger.warning("Failed to deserialize item of type %s: %s", item_type, e) + log_model_and_tool_action_warning( + logger, f"Failed to deserialize item of type {item_type}", e + ) continue return result diff --git a/src/agents/sandbox/config.py b/src/agents/sandbox/config.py index 206ed459f1..1e9dc4acd2 100644 --- a/src/agents/sandbox/config.py +++ b/src/agents/sandbox/config.py @@ -1,11 +1,15 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Final +from typing import TYPE_CHECKING, Any, Final from openai.types.shared import Reasoning -from ..model_settings import ModelSettings +from ..model_settings import ( + ModelSettings, + _coerce_model_settings, + _declared_model_settings_type, +) from ..models.interface import Model DEFAULT_PYTHON_SANDBOX_IMAGE: Final = "python:3.14-slim" @@ -47,7 +51,10 @@ class MemoryGenerateConfig: phase_one_model_settings: ModelSettings | None = field( default_factory=_default_memory_phase_one_model_settings ) - """Model settings used for phase-1 single-rollout extraction.""" + """Model settings used for phase-1 single-rollout extraction. + + Accepts a ``ModelSettings`` instance or a dictionary containing its fields. + """ phase_two_model: str | Model = "gpt-5.5" """Model used for phase-2 memory consolidation.""" @@ -55,7 +62,10 @@ class MemoryGenerateConfig: phase_two_model_settings: ModelSettings | None = field( default_factory=_default_memory_phase_two_model_settings ) - """Model settings used for phase-2 memory consolidation.""" + """Model settings used for phase-2 memory consolidation. + + Accepts a ``ModelSettings`` instance or a dictionary containing its fields. + """ extra_prompt: str | None = None """Optional developer-specific guidance appended to memory extraction and consolidation @@ -70,7 +80,36 @@ class MemoryGenerateConfig: evidence you actually want it to summarize. """ + if TYPE_CHECKING: + + def __init__( + self, + max_raw_memories_for_consolidation: int = 256, + phase_one_model: str | Model = "gpt-5.4-mini", + phase_one_model_settings: ModelSettings | dict[str, Any] | None = ..., + phase_two_model: str | Model = "gpt-5.5", + phase_two_model_settings: ModelSettings | dict[str, Any] | None = ..., + extra_prompt: str | None = None, + ) -> None: ... + def __post_init__(self) -> None: + if self.phase_one_model_settings is not None: + self.phase_one_model_settings = _coerce_model_settings( + self.phase_one_model_settings, + parameter_name="MemoryGenerateConfig.phase_one_model_settings", + model_settings_type=_declared_model_settings_type( + type(self), "phase_one_model_settings" + ), + ) + if self.phase_two_model_settings is not None: + self.phase_two_model_settings = _coerce_model_settings( + self.phase_two_model_settings, + parameter_name="MemoryGenerateConfig.phase_two_model_settings", + model_settings_type=_declared_model_settings_type( + type(self), "phase_two_model_settings" + ), + ) + if self.max_raw_memories_for_consolidation <= 0: raise ValueError( "MemoryGenerateConfig.max_raw_memories_for_consolidation must be greater than 0." diff --git a/src/agents/sandbox/errors.py b/src/agents/sandbox/errors.py index 8e7848a39a..252b2a6f28 100644 --- a/src/agents/sandbox/errors.py +++ b/src/agents/sandbox/errors.py @@ -816,6 +816,7 @@ def __init__( stderr: str | None, context: Mapping[str, object] | None = None, cause: BaseException | None = None, + retryable: bool | None = False, ) -> None: super().__init__( message="mount command failed", @@ -823,7 +824,7 @@ def __init__( op="materialize", context={"command": command, "stderr": stderr, **_as_context(context)}, cause=cause, - retryable=False, + retryable=retryable, ) diff --git a/src/agents/sandbox/manifest.py b/src/agents/sandbox/manifest.py index d4cc014870..9421694ecb 100644 --- a/src/agents/sandbox/manifest.py +++ b/src/agents/sandbox/manifest.py @@ -2,11 +2,12 @@ import asyncio from collections.abc import Iterator, Mapping from pathlib import Path, PurePath, PurePosixPath -from typing import Literal +from typing import Any, Literal from pydantic import BaseModel, Field, field_serializer, field_validator from typing_extensions import assert_never +from .._config_coercion import coerce_pydantic_config from .entries import BaseEntry, Dir, Mount, resolve_workspace_path from .errors import InvalidManifestPathError from .manifest_render import render_manifest_description @@ -256,3 +257,15 @@ def describe(self, depth: int | None = 1) -> str: coerce_rel_path=self._coerce_rel_path, depth=depth, ) + + +def _coerce_manifest(value: Manifest | dict[str, Any], *, parameter_name: str) -> Manifest: + """Normalize manifest dictionaries without granting untrusted host filesystem access.""" + if isinstance(value, dict) and "extra_path_grants" in value: + extra_path_grants = value["extra_path_grants"] + if not isinstance(extra_path_grants, list | tuple) or extra_path_grants: + raise TypeError( + f"{parameter_name}.extra_path_grants must be configured on a trusted " + "Manifest instance, not in a dictionary" + ) + return coerce_pydantic_config(value, Manifest, parameter_name=parameter_name) diff --git a/src/agents/sandbox/memory/manager.py b/src/agents/sandbox/memory/manager.py index 28025466dc..9919d8035b 100644 --- a/src/agents/sandbox/memory/manager.py +++ b/src/agents/sandbox/memory/manager.py @@ -10,6 +10,7 @@ from ...exceptions import UserError from ...items import TResponseInputItem +from ...logger import log_model_and_tool_action_error from ...run_config import RunConfig, SandboxRunConfig from ..capabilities.memory import Memory from ..config import MemoryGenerateConfig @@ -150,8 +151,8 @@ async def _worker(self) -> None: if queue_item is _STOP: return await self._process_rollout_file(str(queue_item)) - except Exception: - logger.exception("Sandbox memory worker failed") + except Exception as exc: + log_model_and_tool_action_error(logger, "Sandbox memory worker failed", exc) finally: self._queue.task_done() @@ -227,8 +228,8 @@ async def _run_phase_two(self) -> None: selection=selection, run_config=self._memory_run_config(), ) - except Exception: - logger.exception("Sandbox memory phase 2 failed") + except Exception as exc: + log_model_and_tool_action_error(logger, "Sandbox memory phase 2 failed", exc) return await self._storage.write_phase_two_selection(selected_items=selection.selected) self._pending_phase_two_rollout_ids = [ diff --git a/src/agents/sandbox/runtime.py b/src/agents/sandbox/runtime.py index d273a54411..0378323a63 100644 --- a/src/agents/sandbox/runtime.py +++ b/src/agents/sandbox/runtime.py @@ -9,6 +9,7 @@ from ..agent import Agent from ..exceptions import UserError from ..items import TResponseInputItem +from ..logger import log_model_and_tool_action_warning from ..result import RunResult, RunResultStreaming from ..run_config import RunConfig from ..run_context import RunContextWrapper, TContext @@ -106,8 +107,10 @@ async def _cleanup_and_store() -> None: input_override=_stream_memory_input_override(result), ) except Exception as error: - logger.warning( - "Failed to enqueue sandbox memory after streamed run: %s", error + log_model_and_tool_action_warning( + logger, + "Failed to enqueue sandbox memory after streamed run", + error, ) payload = await self.cleanup() result._sandbox_resume_state = payload diff --git a/src/agents/sandbox/runtime_session_manager.py b/src/agents/sandbox/runtime_session_manager.py index ec8d8fb268..1d98f7337b 100644 --- a/src/agents/sandbox/runtime_session_manager.py +++ b/src/agents/sandbox/runtime_session_manager.py @@ -300,6 +300,8 @@ async def _create_resources( session=sandbox_config.session, running=running, ) + if manifest_update.processed_manifest is not None: + await sandbox_config.session._validate_manifest_application() if manifest_update.entries_to_apply: await sandbox_config.session._apply_entry_batch( manifest_update.entries_to_apply, diff --git a/src/agents/sandbox/sandbox_agent.py b/src/agents/sandbox/sandbox_agent.py index 6021415428..82ccbba1f9 100644 --- a/src/agents/sandbox/sandbox_agent.py +++ b/src/agents/sandbox/sandbox_agent.py @@ -2,14 +2,29 @@ from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Literal +from .._config_coercion import coerce_pydantic_config from ..agent import Agent from ..run_context import RunContextWrapper, TContext from .capabilities import Capability from .capabilities.capabilities import Capabilities -from .manifest import Manifest +from .manifest import Manifest, _coerce_manifest from .types import User +if TYPE_CHECKING: + from ..agent import MCPConfig, StopAtTools, ToolsToFinalOutputFunction + from ..agent_output import AgentOutputSchemaBase + from ..guardrail import InputGuardrail, OutputGuardrail + from ..handoffs import Handoff + from ..lifecycle import AgentHooks + from ..mcp import MCPServer + from ..model_settings import ModelSettings + from ..models.interface import Model + from ..prompts import DynamicPromptFunction, Prompt + from ..tool import Tool + from ..util._types import MaybeAwaitable + @dataclass class SandboxAgent(Agent[TContext]): @@ -39,8 +54,58 @@ class SandboxAgent(Agent[TContext]): _sandbox_concurrency_guard: object | None = field(default=None, init=False, repr=False) + if TYPE_CHECKING: + + def __init__( + self, + name: str, + handoff_description: str | None = None, + tools: list[Tool] = ..., + mcp_servers: list[MCPServer] = ..., + mcp_config: MCPConfig = ..., + instructions: ( + str + | Callable[ + [RunContextWrapper[TContext], Agent[TContext]], + MaybeAwaitable[str], + ] + | None + ) = None, + prompt: Prompt | DynamicPromptFunction | None = None, + handoffs: list[Agent[Any] | Handoff[TContext, Any]] = ..., + model: str | Model | None = None, + model_settings: ModelSettings | dict[str, Any] = ..., + input_guardrails: list[InputGuardrail[TContext]] = ..., + output_guardrails: list[OutputGuardrail[TContext]] = ..., + output_type: type[Any] | AgentOutputSchemaBase | None = None, + hooks: AgentHooks[TContext] | None = None, + tool_use_behavior: ( + Literal["run_llm_again", "stop_on_first_tool"] + | StopAtTools + | ToolsToFinalOutputFunction + ) = "run_llm_again", + reset_tool_choice: bool = True, + default_manifest: Manifest | dict[str, Any] | None = None, + base_instructions: ( + str + | Callable[ + [RunContextWrapper[TContext], Agent[TContext]], + Awaitable[str | None] | str | None, + ] + | None + ) = None, + capabilities: Sequence[Capability] = ..., + run_as: User | dict[str, Any] | str | None = None, + ) -> None: ... + def __post_init__(self) -> None: super().__post_init__() + if isinstance(self.default_manifest, dict): + self.default_manifest = _coerce_manifest( + self.default_manifest, parameter_name="sandbox.default_manifest" + ) + if isinstance(self.run_as, dict): + self.run_as = coerce_pydantic_config(self.run_as, User, parameter_name="sandbox.run_as") if ( self.base_instructions is not None and not isinstance(self.base_instructions, str) diff --git a/src/agents/sandbox/sandboxes/unix_local.py b/src/agents/sandbox/sandboxes/unix_local.py index a6aff578b9..a2bfeac2ba 100644 --- a/src/agents/sandbox/sandboxes/unix_local.py +++ b/src/agents/sandbox/sandboxes/unix_local.py @@ -24,9 +24,11 @@ from collections.abc import Mapping, Sequence from contextlib import suppress from dataclasses import dataclass, field +from functools import partial from pathlib import Path from typing import Literal, cast +from ...logger import log_tool_action_warning from ..errors import ( ExecNonZeroError, ExecTimeoutError, @@ -75,6 +77,10 @@ logger = logging.getLogger(__name__) +def _mount_path_diagnostic_extra(mount_path: Path) -> dict[str, object]: + return {"mount_path": str(mount_path)} + + def _close_fd_quietly(fd: int) -> None: with suppress(OSError): os.close(fd) @@ -1129,12 +1135,13 @@ async def delete(self, session: SandboxSession) -> SandboxSession: for mount_entry, mount_path in inner.state.manifest.ephemeral_mount_targets(): try: await mount_entry.unmount(inner, mount_path, Path("/")) - except Exception: + except Exception as exc: unmount_failed = True - logger.warning( - "Failed to unmount UnixLocal workspace mount before deleting root: %s", - mount_path, - exc_info=True, + log_tool_action_warning( + logger, + "Failed to unmount UnixLocal workspace mount before deleting root", + exc, + diagnostic_extra=partial(_mount_path_diagnostic_extra, mount_path), ) if unmount_failed: return session diff --git a/src/agents/sandbox/session/base_sandbox_session.py b/src/agents/sandbox/session/base_sandbox_session.py index 4f172d3951..ab22940734 100644 --- a/src/agents/sandbox/session/base_sandbox_session.py +++ b/src/agents/sandbox/session/base_sandbox_session.py @@ -1199,7 +1199,11 @@ async def _apply_manifest( provision_accounts=provision_accounts, ) + async def _validate_manifest_application(self, *, only_ephemeral: bool = False) -> None: + _ = only_ephemeral + async def apply_manifest(self, *, only_ephemeral: bool = False) -> MaterializationResult: + await self._validate_manifest_application(only_ephemeral=only_ephemeral) return await self._apply_manifest( only_ephemeral=only_ephemeral, provision_accounts=not only_ephemeral, @@ -1246,6 +1250,11 @@ def _workspace_fingerprint_skip_relpaths(self) -> set[Path]: return snapshot_lifecycle.workspace_fingerprint_skip_relpaths(self) + def _should_compute_snapshot_fingerprint_on_persist(self) -> bool: + """Return whether persistence should fingerprint the workspace before archiving it.""" + + return True + async def _compute_and_cache_snapshot_fingerprint(self) -> dict[str, str]: """Compute the current workspace fingerprint in-container and atomically cache it.""" diff --git a/src/agents/sandbox/session/manager.py b/src/agents/sandbox/session/manager.py index 125765e65b..1248ce19b9 100644 --- a/src/agents/sandbox/session/manager.py +++ b/src/agents/sandbox/session/manager.py @@ -4,6 +4,7 @@ import logging from collections.abc import Sequence +from ...logger import log_tool_action_error from ..errors import OpName from .events import EventPayloadPolicy, SandboxSessionEvent, SandboxSessionFinishEvent from .sinks import ChainedSink, EventSink @@ -104,8 +105,8 @@ async def _run() -> None: if sink.mode == "sync": try: await _run() - except Exception: - self._handle_sink_error(sink, event) + except Exception as exc: + self._handle_sink_error(sink, event, exc) elif sink.mode == "async": if sink.on_error == "raise": await _run() @@ -114,8 +115,8 @@ async def _run() -> None: async def _task() -> None: try: await _run() - except Exception: - self._handle_sink_error(sink, event) + except Exception as exc: + self._handle_sink_error(sink, event, exc) task = asyncio.create_task(_task()) # Track background deliveries so the task is kept alive and can be discarded once done. @@ -126,8 +127,8 @@ async def _task() -> None: async def _task() -> None: try: await _run() - except Exception: - self._handle_sink_error(sink, event, force_no_raise=True) + except Exception as exc: + self._handle_sink_error(sink, event, exc, force_no_raise=True) task = asyncio.create_task(_task()) # Same bookkeeping as async mode, but failures are always swallowed after logging. @@ -146,16 +147,26 @@ async def _deliver_chained(self, sink: EventSink, event: SandboxSessionEvent) -> """ try: await sink.handle(event) - except Exception: + except Exception as exc: force_no_raise = sink.mode == "best_effort" - self._handle_sink_error(sink, event, force_no_raise=force_no_raise) + self._handle_sink_error(sink, event, exc, force_no_raise=force_no_raise) def _handle_sink_error( - self, sink: EventSink, event: SandboxSessionEvent, *, force_no_raise: bool = False + self, + sink: EventSink, + event: SandboxSessionEvent, + exc: Exception, + *, + force_no_raise: bool = False, ) -> None: if force_no_raise or sink.on_error in ("log", "ignore"): if sink.on_error == "log": - logger.exception("sandbox event sink failed (ignored): %s", type(sink).__name__) + log_tool_action_error( + logger, + "Sandbox event sink failed (ignored)", + exc, + diagnostic_extra=lambda: {"sink_type": type(sink).__name__}, + ) return raise RuntimeError( "sandbox event sink failed: " diff --git a/src/agents/sandbox/session/manifest_ops.py b/src/agents/sandbox/session/manifest_ops.py index 04eab029d4..aaccdf9cd8 100644 --- a/src/agents/sandbox/session/manifest_ops.py +++ b/src/agents/sandbox/session/manifest_ops.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING from ..entries import BaseEntry +from ..manifest import Manifest from ..materialization import MaterializationResult, MaterializedFile from .manifest_application import ManifestApplier @@ -16,12 +17,13 @@ async def apply_manifest( session: BaseSandboxSession, *, + manifest: Manifest | None = None, only_ephemeral: bool = False, provision_accounts: bool = True, ) -> MaterializationResult: applier = _build_manifest_applier(session, include_entry_concurrency=True) return await applier.apply_manifest( - session.state.manifest, + manifest if manifest is not None else session.state.manifest, only_ephemeral=only_ephemeral, provision_accounts=provision_accounts, base_dir=session._manifest_base_dir(), diff --git a/src/agents/sandbox/session/runtime_helpers.py b/src/agents/sandbox/session/runtime_helpers.py index 8ab58a1fcb..24fc89127b 100644 --- a/src/agents/sandbox/session/runtime_helpers.py +++ b/src/agents/sandbox/session/runtime_helpers.py @@ -214,7 +214,13 @@ exit 127 } -tar_cmd="tar" +if tar --help 2>&1 | grep -q -- '--no-wildcards'; then + tar_cmd="tar --no-wildcards" + escape_tar_patterns=0 +else + tar_cmd="tar" + escape_tar_patterns=1 +fi for rel in "$@"; do case "$rel" in ""|"."|"/"|*"/.."|*"/../"*|".."|../*|*/../*|/*) @@ -222,6 +228,9 @@ exit 65 ;; esac + if [ "$escape_tar_patterns" -eq 1 ]; then + rel=$(printf '%s\\n' "$rel" | sed 's/[][\\\\*?]/\\\\&/g') + fi quoted_rel=$(quote_sh "$rel") quoted_dot_rel=$(quote_sh "./$rel") tar_cmd="$tar_cmd --exclude=$quoted_rel --exclude=$quoted_dot_rel" diff --git a/src/agents/sandbox/session/sandbox_session.py b/src/agents/sandbox/session/sandbox_session.py index 66f51c2e24..6aba057642 100644 --- a/src/agents/sandbox/session/sandbox_session.py +++ b/src/agents/sandbox/session/sandbox_session.py @@ -14,6 +14,7 @@ from ...tracing import Span, custom_span, get_current_trace from ..errors import OpName, SandboxError from ..files import FileEntry +from ..materialization import MaterializationResult from ..types import ExecResult, ExposedPortEndpoint, User from .base_sandbox_session import BaseSandboxSession from .dependencies import Dependencies @@ -524,6 +525,12 @@ async def stop(self) -> None: async def shutdown(self) -> None: await self._inner.shutdown() + async def _validate_manifest_application(self, *, only_ephemeral: bool = False) -> None: + await self._inner._validate_manifest_application(only_ephemeral=only_ephemeral) + + async def apply_manifest(self, *, only_ephemeral: bool = False) -> MaterializationResult: + return await super().apply_manifest(only_ephemeral=only_ephemeral) + @instrumented_op( "exec", data=_exec_start_data, diff --git a/src/agents/sandbox/session/snapshot_lifecycle.py b/src/agents/sandbox/session/snapshot_lifecycle.py index 1145f8a247..3c8db9f8ef 100644 --- a/src/agents/sandbox/session/snapshot_lifecycle.py +++ b/src/agents/sandbox/session/snapshot_lifecycle.py @@ -23,10 +23,11 @@ async def persist_snapshot(session: BaseSandboxSession) -> None: return fingerprint_record: dict[str, str] | None = None - try: - fingerprint_record = await session._compute_and_cache_snapshot_fingerprint() - except Exception: - fingerprint_record = None + if session._should_compute_snapshot_fingerprint_on_persist(): + try: + fingerprint_record = await session._compute_and_cache_snapshot_fingerprint() + except Exception: + fingerprint_record = None workspace_archive = await session.persist_workspace() try: diff --git a/src/agents/sandbox/types.py b/src/agents/sandbox/types.py index d19df2fd77..efdc507c8c 100644 --- a/src/agents/sandbox/types.py +++ b/src/agents/sandbox/types.py @@ -56,7 +56,10 @@ def from_mode(cls, mode: int) -> "Permissions": @classmethod def from_str(cls, perms: str) -> "Permissions": - if len(perms) == 11 and perms[-1] in {"@", "+"}: + # coreutils/BSD ls append a single trailing marker to the mode field to flag + # alternate access methods: "+" (ACL), "@" (macOS extended attributes), and + # "." (SELinux security context). Strip it before parsing the 10 mode chars. + if len(perms) == 11 and perms[-1] in {"@", "+", "."}: perms = perms[:-1] if len(perms) != 10: raise ValueError(f"invalid permissions string length: {perms!r}") diff --git a/src/agents/sandbox/util/tar_utils.py b/src/agents/sandbox/util/tar_utils.py index 5a7167177d..6ada6378a1 100644 --- a/src/agents/sandbox/util/tar_utils.py +++ b/src/agents/sandbox/util/tar_utils.py @@ -181,6 +181,19 @@ def _is_within(path: Path, prefix: Path) -> bool: return path.parts[: len(prefix.parts)] == prefix.parts +def _tar_member_rel_variants(member_name: str, root_name: str | None) -> list[Path]: + raw_parts = [p for p in Path(member_name).parts if p not in ("", ".")] + if raw_parts[:1] == ["/"]: + raw_parts = raw_parts[1:] + if not raw_parts: + return [Path()] + + variants = [Path(*raw_parts)] + if root_name and raw_parts[0] == root_name: + variants.append(Path(*raw_parts[1:])) + return variants + + def should_skip_tar_member( member_name: str, *, @@ -194,16 +207,7 @@ def should_skip_tar_member( directory name depending on how the tar was produced. """ - raw_parts = [p for p in Path(member_name).parts if p not in ("", ".")] - if raw_parts[:1] == ["/"]: - raw_parts = raw_parts[1:] - if not raw_parts: - rel_variants = [Path()] - else: - rel_variants = [Path(*raw_parts)] - if root_name and raw_parts and raw_parts[0] == root_name: - rel_variants.append(Path(*raw_parts[1:])) - + rel_variants = _tar_member_rel_variants(member_name, root_name) prefixes = [_normalize_rel(p) for p in skip_rel_paths] return any(_is_within(rel, prefix) for rel in rel_variants for prefix in prefixes) @@ -234,6 +238,7 @@ def _ensure_no_symlink_parents(*, root: Path, dest: Path, check_leaf: bool = Tru def validate_tarfile( tar: tarfile.TarFile, *, + reject_rel_paths: Iterable[str | Path] = (), reject_symlink_rel_paths: Iterable[str | Path] = (), skip_rel_paths: Iterable[str | Path] = (), root_name: str | None = None, @@ -250,6 +255,7 @@ def validate_tarfile( been restored. """ + rejected_rel_paths = {_normalize_rel(path) for path in reject_rel_paths} rejected_symlink_rel_paths = {_normalize_rel(path) for path in reject_symlink_rel_paths} members_by_rel_path: dict[Path, tarfile.TarInfo] = {} symlink_rel_paths: set[Path] = set() @@ -265,6 +271,17 @@ def validate_tarfile( rel_path = safe_tar_member_rel_path(member, allow_symlinks=allow_symlinks) if rel_path is None: continue + rel_variants = _tar_member_rel_variants(member.name, root_name) + for rejected_path in rejected_rel_paths: + if any( + _is_within(variant, rejected_path) + or (not member.isdir() and _is_within(rejected_path, variant)) + for variant in rel_variants + ): + raise UnsafeTarMemberError( + member=member.name, + reason=f"archive member overlaps protected path: {rejected_path.as_posix()}", + ) previous = members_by_rel_path.get(rel_path) if previous is not None and not (previous.isdir() and member.isdir()): @@ -308,6 +325,7 @@ def validate_tarfile( def validate_tar_bytes( raw: bytes, *, + reject_rel_paths: Iterable[str | Path] = (), reject_symlink_rel_paths: Iterable[str | Path] = (), skip_rel_paths: Iterable[str | Path] = (), root_name: str | None = None, @@ -319,6 +337,7 @@ def validate_tar_bytes( with tarfile.open(fileobj=io.BytesIO(raw), mode="r:*") as tar: validate_tarfile( tar, + reject_rel_paths=reject_rel_paths, reject_symlink_rel_paths=reject_symlink_rel_paths, skip_rel_paths=skip_rel_paths, root_name=root_name, diff --git a/src/agents/strict_schema.py b/src/agents/strict_schema.py index 89f302f5d8..463cede791 100644 --- a/src/agents/strict_schema.py +++ b/src/agents/strict_schema.py @@ -90,7 +90,10 @@ def _ensure_strict_json_schema( elif ( typ == "object" and "additionalProperties" in json_schema - and json_schema["additionalProperties"] + # Compare with ``is not False`` rather than truthiness: OpenAPI/MCP schemas often use + # ``additionalProperties: {}`` (an empty schema meaning "allow anything"). That value is + # falsy in Python, so a truthiness check would silently leave a non-strict schema in place. + and json_schema["additionalProperties"] is not False ): raise UserError( "additionalProperties should not be set for object types. This could be because " diff --git a/src/agents/tool.py b/src/agents/tool.py index def9ea2e66..eb6c0a3645 100644 --- a/src/agents/tool.py +++ b/src/agents/tool.py @@ -43,6 +43,7 @@ from typing_extensions import NotRequired, ParamSpec, TypedDict from . import _debug +from ._config_coercion import coerce_pydantic_config from ._tool_identity import ( get_explicit_function_tool_namespace, tool_qualified_name, @@ -53,7 +54,7 @@ from .editor import ApplyPatchEditor, ApplyPatchOperation from .exceptions import ModelBehaviorError, ToolTimeoutError, UserError from .function_schema import DocstringStyle, function_schema -from .logger import logger +from .logger import log_tool_action_warning, logger from .run_context import RunContextWrapper from .strict_schema import ensure_strict_json_schema from .tool_context import ToolContext @@ -735,6 +736,22 @@ class WebSearchTool: indexed-only behavior where supported. """ + if TYPE_CHECKING: + + def __init__( + self, + user_location: UserLocation | None = None, + filters: WebSearchToolFilters | dict[str, Any] | None = None, + search_context_size: Literal["low", "medium", "high"] = "medium", + external_web_access: bool | None = None, + ) -> None: ... + + def __post_init__(self) -> None: + if isinstance(self.filters, dict): + self.filters = coerce_pydantic_config( + self.filters, WebSearchToolFilters, parameter_name="web search filters" + ) + @property def name(self): return "web_search" @@ -868,7 +885,7 @@ async def dispose_resolved_computers(*, run_context: RunContextWrapper[Any]) -> if inspect.isawaitable(result): await result except Exception as exc: - logger.warning("Failed to dispose computer for run context: %s", exc) + log_tool_action_warning(logger, "Failed to dispose computer for run context", exc) @dataclass diff --git a/src/agents/tool_context.py b/src/agents/tool_context.py index 75947630cf..b9c753c79a 100644 --- a/src/agents/tool_context.py +++ b/src/agents/tool_context.py @@ -68,7 +68,7 @@ def __init__( *, tool_namespace: str | None = None, agent: AgentBase[Any] | None = None, - run_config: RunConfig | None = None, + run_config: RunConfig | dict[str, Any] | None = None, turn_input: list[TResponseInputItem] | None = None, _approvals: dict[str, _ApprovalRecord] | None = None, tool_input: Any | None = None, @@ -102,7 +102,12 @@ def __init__( else get_tool_call_namespace(tool_call) ) self.agent = agent - self.run_config = run_config + if run_config is not None: + from .run_config import _coerce_run_config + + self.run_config = _coerce_run_config(run_config) + else: + self.run_config = None # Internal adapter hook used to attach SDK-only custom data to the emitted output item. self._custom_data: dict[str, Any] | None = None @@ -122,7 +127,7 @@ def from_agent_context( tool_name: str | None = None, tool_arguments: str | None = None, tool_namespace: str | None = None, - run_config: RunConfig | None = None, + run_config: RunConfig | dict[str, Any] | None = None, ) -> ToolContext: """ Create a ToolContext from a RunContextWrapper. diff --git a/src/agents/tracing/processors.py b/src/agents/tracing/processors.py index 776939a180..6c68a2673f 100644 --- a/src/agents/tracing/processors.py +++ b/src/agents/tracing/processors.py @@ -13,7 +13,12 @@ import httpx -from ..logger import logger +from .. import _debug +from ..logger import ( + log_model_and_tool_action_error, + log_model_and_tool_action_warning, + logger, +) from .processor_interface import TracingExporter, TracingProcessor from .spans import Span from .traces import Trace @@ -24,6 +29,12 @@ class ConsoleSpanExporter(TracingExporter): def export(self, items: list[Trace | Span[Any]]) -> None: for item in items: + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + if isinstance(item, Trace): + print("[Exporter] Export trace. Trace data is redacted.") + else: + print("[Exporter] Export span. Span data is redacted.") + continue if isinstance(item, Trace): print(f"[Exporter] Export trace_id={item.trace_id}, name={item.name}") else: @@ -173,11 +184,17 @@ def _export_with_deadline(self, items: list[Trace | Span[Any]], deadline: float # If the response is a client error (4xx), we won't retry if 400 <= response.status_code < 500: - logger.error( - "[non-fatal] Tracing client error %s: %s", - response.status_code, - response.text, - ) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.error( + "[non-fatal] Tracing client error %s. Response data is redacted.", + response.status_code, + ) + else: + logger.error( + "[non-fatal] Tracing client error %s: %s", + response.status_code, + response.text, + ) break # For 5xx or other unexpected codes, treat it as transient and retry @@ -186,7 +203,9 @@ def _export_with_deadline(self, items: list[Trace | Span[Any]], deadline: float ) except httpx.RequestError as exc: # Network or other I/O error, we'll retry - logger.warning("[non-fatal] Tracing: request failed: %s", exc) + log_model_and_tool_action_warning( + logger, "[non-fatal] Tracing request failed", exc + ) # If we reach here, we need to retry or give up if attempt >= self.max_retries: @@ -690,10 +709,13 @@ def _export_batches(self, force: bool = False, deadline: float | None = None): else: self._exporter.export(items_to_export) except Exception as exc: - logger.error( - "[non-fatal] Tracing: exporter raised %s; dropping batch of %d items", + log_model_and_tool_action_error( + logger, + ( + "[non-fatal] Tracing exporter failed; " + f"dropping batch of {len(items_to_export)} items" + ), exc, - len(items_to_export), ) diff --git a/src/agents/tracing/provider.py b/src/agents/tracing/provider.py index a5f439dceb..57817642f8 100644 --- a/src/agents/tracing/provider.py +++ b/src/agents/tracing/provider.py @@ -6,11 +6,14 @@ import time import uuid from abc import ABC, abstractmethod +from collections.abc import Callable from datetime import datetime, timezone +from functools import partial from inspect import Parameter, signature from typing import Any, cast -from ..logger import logger +from .. import _debug +from ..logger import log_model_and_tool_action_error, logger from .config import TracingConfig from .processor_interface import TracingProcessor from .scope import Scope @@ -18,7 +21,7 @@ from .traces import NoOpTrace, Trace, TraceImpl -def _safe_debug(message: str) -> None: +def _safe_debug(message: str | Callable[[], str]) -> None: """Best-effort debug logging that tolerates closed streams during shutdown.""" def _has_closed_stream_handler(log: logging.Logger) -> bool: @@ -37,12 +40,24 @@ def _has_closed_stream_handler(log: logging.Logger) -> bool: # Avoid emitting debug logs when any handler already owns a closed stream. if _has_closed_stream_handler(logger): return - logger.debug(message) + logger.debug(message() if callable(message) else message) except Exception: # Avoid noisy shutdown errors when the underlying stream is already closed. return +def _processor_diagnostic_extra(processor: TracingProcessor) -> dict[str, object]: + processor_type = type(processor) + processor_identity = ( + f"{processor_type.__module__}.{processor_type.__qualname__}@{id(processor):x}" + ) + return {"trace_processor": processor_identity} + + +def _processor_shutdown_message(processor: TracingProcessor) -> str: + return f"Shutting down trace processor {processor}" + + def _remaining_timeout(deadline: float | None) -> float | None: if deadline is None: return None @@ -107,7 +122,12 @@ def on_trace_start(self, trace: Trace) -> None: try: processor.on_trace_start(trace) except Exception as e: - logger.error("Error in trace processor %s during on_trace_start: %s", processor, e) + log_model_and_tool_action_error( + logger, + "Error in trace processor during on_trace_start", + e, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), + ) def on_trace_end(self, trace: Trace) -> None: """ @@ -117,7 +137,12 @@ def on_trace_end(self, trace: Trace) -> None: try: processor.on_trace_end(trace) except Exception as e: - logger.error("Error in trace processor %s during on_trace_end: %s", processor, e) + log_model_and_tool_action_error( + logger, + "Error in trace processor during on_trace_end", + e, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), + ) def on_span_start(self, span: Span[Any]) -> None: """ @@ -127,7 +152,12 @@ def on_span_start(self, span: Span[Any]) -> None: try: processor.on_span_start(span) except Exception as e: - logger.error("Error in trace processor %s during on_span_start: %s", processor, e) + log_model_and_tool_action_error( + logger, + "Error in trace processor during on_span_start", + e, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), + ) def on_span_end(self, span: Span[Any]) -> None: """ @@ -137,7 +167,12 @@ def on_span_end(self, span: Span[Any]) -> None: try: processor.on_span_end(span) except Exception as e: - logger.error("Error in trace processor %s during on_span_end: %s", processor, e) + log_model_and_tool_action_error( + logger, + "Error in trace processor during on_span_end", + e, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), + ) def shutdown(self, timeout: float | None = None) -> None: """ @@ -145,7 +180,10 @@ def shutdown(self, timeout: float | None = None) -> None: """ deadline = None if timeout is None else time.monotonic() + timeout for processor in self._processors: - _safe_debug(f"Shutting down trace processor {processor}") + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + _safe_debug("Shutting down trace processor") + else: + _safe_debug(partial(_processor_shutdown_message, processor)) try: processor_timeout = _remaining_timeout(deadline) if processor_timeout is not None and processor_timeout <= 0: @@ -158,7 +196,12 @@ def shutdown(self, timeout: float | None = None) -> None: else: processor.shutdown() except Exception as e: - logger.error("Error shutting down trace processor %s: %s", processor, e) + log_model_and_tool_action_error( + logger, + "Error shutting down trace processor", + e, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), + ) def force_flush(self): """ @@ -168,7 +211,12 @@ def force_flush(self): try: processor.force_flush() except Exception as e: - logger.error("Error flushing trace processor %s: %s", processor, e) + log_model_and_tool_action_error( + logger, + "Error flushing trace processor", + e, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), + ) class TraceProvider(ABC): @@ -337,12 +385,18 @@ def create_trace( """ self._refresh_disabled_flag() if self._disabled or disabled: - logger.debug("Tracing is disabled. Not creating trace %s", name) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Tracing is disabled. Not creating trace") + else: + logger.debug("Tracing is disabled. Not creating trace %s", name) return NoOpTrace() trace_id = trace_id or self.gen_trace_id() - logger.debug("Creating trace %s with id %s", name, trace_id) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Creating trace with id %s", trace_id) + else: + logger.debug("Creating trace %s with id %s", name, trace_id) return TraceImpl( name=name, @@ -367,7 +421,10 @@ def create_span( tracing_api_key: str | None = None trace_metadata: dict[str, Any] | None = None if self._disabled or disabled: - logger.debug("Tracing is disabled. Not creating span %s", span_data) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Tracing is disabled. Not creating span") + else: + logger.debug("Tracing is disabled. Not creating span %s", span_data) return NoOpSpan(span_data) if _is_noop_id(span_id): logger.debug("Span id is no-op, returning NoOpSpan") @@ -383,9 +440,14 @@ def create_span( ) return NoOpSpan(span_data) elif _is_noop_trace(current_trace) or _is_noop_span(current_span): - logger.debug( - "Parent %s or %s is no-op, returning NoOpSpan", current_span, current_trace - ) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Current trace parent is no-op, returning NoOpSpan") + else: + logger.debug( + "Parent %s or %s is no-op, returning NoOpSpan", + current_span, + current_trace, + ) return NoOpSpan(span_data) parent_id = current_span.span_id if current_span else None @@ -396,7 +458,10 @@ def create_span( elif isinstance(parent, Trace): if _is_noop_trace(parent): - logger.debug("Parent %s is no-op, returning NoOpSpan", parent) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Parent trace is no-op, returning NoOpSpan") + else: + logger.debug("Parent %s is no-op, returning NoOpSpan", parent) return NoOpSpan(span_data) trace_id = parent.trace_id parent_id = None @@ -405,14 +470,20 @@ def create_span( trace_metadata = getattr(parent, "metadata", None) elif isinstance(parent, Span): if _is_noop_span(parent): - logger.debug("Parent %s is no-op, returning NoOpSpan", parent) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Parent span is no-op, returning NoOpSpan") + else: + logger.debug("Parent %s is no-op, returning NoOpSpan", parent) return NoOpSpan(span_data) parent_id = parent.span_id trace_id = parent.trace_id tracing_api_key = parent.tracing_api_key trace_metadata = parent.trace_metadata - logger.debug("Creating span %s with id %s", span_data, span_id) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Creating span with id %s", span_id) + else: + logger.debug("Creating span %s with id %s", span_data, span_id) return SpanImpl( trace_id=trace_id, @@ -433,7 +504,7 @@ def force_flush(self) -> None: try: self._multi_processor.force_flush() except Exception as e: - logger.error("Error flushing trace provider: %s", e) + log_model_and_tool_action_error(logger, "Error flushing trace provider", e) def shutdown(self, timeout: float | None = None) -> None: self._refresh_disabled_flag() @@ -444,4 +515,4 @@ def shutdown(self, timeout: float | None = None) -> None: _safe_debug("Shutting down trace provider") self._multi_processor.shutdown(timeout=timeout) except Exception as e: - logger.error("Error shutting down trace provider: %s", e) + log_model_and_tool_action_error(logger, "Error shutting down trace provider", e) diff --git a/src/agents/util/_error_tracing.py b/src/agents/util/_error_tracing.py index 0bd2d99e90..7f714482a5 100644 --- a/src/agents/util/_error_tracing.py +++ b/src/agents/util/_error_tracing.py @@ -1,5 +1,6 @@ from typing import Any +from .. import _debug from ..logger import logger from ..tracing import Span, SpanError, get_current_span @@ -24,5 +25,7 @@ def attach_error_to_current_span(error: SpanError) -> None: span = get_current_span() if span: attach_error_to_span(span, error) + elif _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.warning("No active span; trace error was not attached") else: logger.warning("No span to add error %s to", error) diff --git a/src/agents/voice/input.py b/src/agents/voice/input.py index 6097ee7bbf..c39172c79a 100644 --- a/src/agents/voice/input.py +++ b/src/agents/voice/input.py @@ -81,11 +81,11 @@ class StreamedAudioInput: def __init__(self): self.queue: asyncio.Queue[npt.NDArray[np.int16 | np.float32] | None] = asyncio.Queue() - async def add_audio(self, audio: npt.NDArray[np.int16 | np.float32] | None): + async def add_audio(self, audio: npt.NDArray[np.int16 | np.float32] | None) -> None: """Adds more audio data to the stream. Args: audio: The audio data to add. Must be a numpy array of int16 or float32 or None. - If None passed, it indicates the end of the stream. + If None passed, it indicates the end of the stream. """ await self.queue.put(audio) diff --git a/src/agents/voice/models/openai_model_provider.py b/src/agents/voice/models/openai_model_provider.py index b992f9b4ad..2736afbf57 100644 --- a/src/agents/voice/models/openai_model_provider.py +++ b/src/agents/voice/models/openai_model_provider.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Any + import httpx from openai import AsyncOpenAI, DefaultAsyncHttpxClient @@ -41,7 +43,7 @@ def __init__( openai_client: AsyncOpenAI | None = None, organization: str | None = None, project: str | None = None, - agent_registration: OpenAIAgentRegistrationConfig | None = None, + agent_registration: OpenAIAgentRegistrationConfig | dict[str, Any] | None = None, ) -> None: """Create a new OpenAI voice model provider. diff --git a/src/agents/voice/models/openai_stt.py b/src/agents/voice/models/openai_stt.py index 83f0931fc4..89ee27dc10 100644 --- a/src/agents/voice/models/openai_stt.py +++ b/src/agents/voice/models/openai_stt.py @@ -41,13 +41,15 @@ class WebsocketDoneSentinel: def _audio_to_base64(audio_data: list[npt.NDArray[np.int16 | np.float32]]) -> str: - concatenated_audio = np.concatenate(audio_data) - if concatenated_audio.dtype == np.float32: - # convert to int16 - concatenated_audio = np.clip(concatenated_audio, -1.0, 1.0) - concatenated_audio = (concatenated_audio * 32767).astype(np.int16) - audio_bytes = concatenated_audio.tobytes() - return base64.b64encode(audio_bytes).decode("utf-8") + return _audio_buffer_to_base64(np.concatenate(audio_data)) + + +def _audio_buffer_to_base64(buffer: npt.NDArray[np.int16 | np.float32]) -> str: + if buffer.dtype == np.float32: + # Convert to int16. + buffer = np.clip(buffer, -1.0, 1.0) + buffer = (buffer * 32767).astype(np.int16) + return base64.b64encode(buffer.tobytes()).decode("utf-8") async def _wait_for_event( @@ -267,7 +269,7 @@ async def _stream_audio( json.dumps( { "type": "input_audio_buffer.append", - "audio": base64.b64encode(buffer.tobytes()).decode("utf-8"), + "audio": _audio_buffer_to_base64(buffer), } ) ) diff --git a/src/agents/voice/pipeline.py b/src/agents/voice/pipeline.py index 745f0faafb..da2bdceafd 100644 --- a/src/agents/voice/pipeline.py +++ b/src/agents/voice/pipeline.py @@ -1,9 +1,15 @@ from __future__ import annotations import asyncio +from typing import Any +from .._config_coercion import coerce_dataclass_config from ..exceptions import UserError -from ..logger import logger +from ..logger import ( + log_model_and_tool_action_error, + log_model_and_tool_action_warning, + logger, +) from ..tracing import TraceCtxManager from .input import AudioInput, StreamedAudioInput from .model import STTModel, TTSModel @@ -25,7 +31,7 @@ def __init__( workflow: VoiceWorkflowBase, stt_model: STTModel | str | None = None, tts_model: TTSModel | str | None = None, - config: VoicePipelineConfig | None = None, + config: VoicePipelineConfig | dict[str, Any] | None = None, ): """Create a new voice pipeline. @@ -43,7 +49,11 @@ def __init__( self.tts_model = tts_model if isinstance(tts_model, TTSModel) else None self._stt_model_name = stt_model if isinstance(stt_model, str) else None self._tts_model_name = tts_model if isinstance(tts_model, str) else None - self.config = config or VoicePipelineConfig() + self.config = ( + coerce_dataclass_config(config, VoicePipelineConfig, parameter_name="voice.pipeline") + if config is not None + else VoicePipelineConfig() + ) async def run(self, audio_input: AudioInput | StreamedAudioInput) -> StreamedAudioResult: """Run the voice pipeline. @@ -103,7 +113,7 @@ async def stream_events(): await output._turn_done() await output._done() except Exception as e: - logger.error("Error processing single turn: %s", e) + log_model_and_tool_action_error(logger, "Error processing single voice turn", e) await output._add_error(e) raise e @@ -129,7 +139,9 @@ async def process_turns(): async for intro_text in self.workflow.on_start(): await output._add_text(intro_text) except Exception as e: - logger.warning("on_start() failed: %s", e) + log_model_and_tool_action_warning( + logger, "Voice workflow on_start failed", e + ) transcription_session = await self._get_stt_model().create_session( audio_input, @@ -144,7 +156,7 @@ async def process_turns(): await output._add_text(text_event) await output._turn_done() except Exception as e: - logger.error("Error processing turns: %s", e) + log_model_and_tool_action_error(logger, "Error processing voice turns", e) await output._add_error(e) raise e finally: diff --git a/src/agents/voice/pipeline_config.py b/src/agents/voice/pipeline_config.py index eed2ab6940..35c55d093a 100644 --- a/src/agents/voice/pipeline_config.py +++ b/src/agents/voice/pipeline_config.py @@ -1,8 +1,9 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any +from typing import TYPE_CHECKING, Any +from .._config_coercion import _declared_dataclass_type, coerce_dataclass_config from ..tracing import TracingConfig from ..tracing.util import gen_group_id from .model import STTModelSettings, TTSModelSettings, VoiceModelProvider @@ -48,3 +49,31 @@ class VoicePipelineConfig: tts_settings: TTSModelSettings = field(default_factory=TTSModelSettings) """The settings to use for the TTS model.""" + + if TYPE_CHECKING: + + def __init__( + self, + model_provider: VoiceModelProvider = ..., + tracing_disabled: bool = False, + tracing: TracingConfig | None = None, + trace_include_sensitive_data: bool = True, + trace_include_sensitive_audio_data: bool = True, + workflow_name: str = "Voice Agent", + group_id: str = ..., + trace_metadata: dict[str, Any] | None = None, + stt_settings: STTModelSettings | dict[str, Any] = ..., + tts_settings: TTSModelSettings | dict[str, Any] = ..., + ) -> None: ... + + def __post_init__(self) -> None: + self.stt_settings = coerce_dataclass_config( + self.stt_settings, + _declared_dataclass_type(type(self), "stt_settings", STTModelSettings), + parameter_name="voice.stt", + ) + self.tts_settings = coerce_dataclass_config( + self.tts_settings, + _declared_dataclass_type(type(self), "tts_settings", TTSModelSettings), + parameter_name="voice.tts", + ) diff --git a/src/agents/voice/result.py b/src/agents/voice/result.py index 2f4b24433b..709c829779 100644 --- a/src/agents/voice/result.py +++ b/src/agents/voice/result.py @@ -7,7 +7,7 @@ from typing import Any from ..exceptions import UserError -from ..logger import logger +from ..logger import log_model_action_error, log_model_and_tool_action_error, logger from ..tracing import Span, SpeechGroupSpanData, speech_group_span, speech_span from ..tracing.util import time_iso from ..util._error_tracing import get_trace_error @@ -192,7 +192,7 @@ async def _stream_audio( }, } ) - logger.error("Error streaming audio: %s", e) + log_model_action_error(logger, "Error streaming voice audio", e) # Signal completion for whole session because of error await local_queue.put(VoiceStreamEventLifecycle(event="session_ended")) @@ -308,7 +308,9 @@ async def stream(self) -> AsyncIterator[VoiceStreamEvent]: break if isinstance(event, VoiceStreamEventError): self._stored_exception = event.error - logger.error("Error processing output: %s", event.error) + log_model_and_tool_action_error( + logger, "Error processing voice output", event.error + ) break if event is None: break diff --git a/tests/extensions/experiemental/codex/test_codex_tool.py b/tests/extensions/experiemental/codex/test_codex_tool.py index 9bf650816b..3aeb7db73d 100644 --- a/tests/extensions/experiemental/codex/test_codex_tool.py +++ b/tests/extensions/experiemental/codex/test_codex_tool.py @@ -14,6 +14,7 @@ from openai.types.responses import ResponseFunctionToolCall from pydantic import BaseModel, ConfigDict +import agents._debug as _debug from agents import Agent, function_tool from agents.exceptions import ModelBehaviorError, UserError from agents.extensions.experimental.codex import ( @@ -1802,7 +1803,13 @@ async def test_replaced_codex_tool_preserves_codex_collision_markers() -> None: @pytest.mark.asyncio -async def test_codex_tool_consume_events_with_on_stream_error() -> None: +async def test_codex_tool_consume_events_with_on_stream_error( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + secret = "SECRET_CODEX_STREAM_PAYLOAD" events = [ { "type": "item.started", @@ -1865,7 +1872,7 @@ async def event_stream(): def on_stream(payload: CodexToolStreamEvent) -> None: callbacks.append(payload.event.type) if payload.event.type == "item.started": - raise RuntimeError("boom") + raise RuntimeError(secret) context = ToolContext( context=None, @@ -1874,20 +1881,22 @@ def on_stream(payload: CodexToolStreamEvent) -> None: tool_arguments="{}", ) - with trace("codex-test"): - response, usage, thread_id = await codex_tool_module._consume_events( - event_stream(), - {"inputs": [{"type": "text", "text": "hello"}]}, - context, - SimpleNamespace(id="thread-1"), - on_stream, - 64, - ) + with caplog.at_level("ERROR", logger="openai.agents"): + with trace("codex-test"): + response, usage, thread_id = await codex_tool_module._consume_events( + event_stream(), + {"inputs": [{"type": "text", "text": "hello"}]}, + context, + SimpleNamespace(id="thread-1"), + on_stream, + 64, + ) assert response == "done" assert usage == Usage(input_tokens=1, cached_input_tokens=0, output_tokens=1) assert thread_id == "thread-1" assert "item.started" in callbacks + assert secret not in caplog.text @pytest.mark.asyncio diff --git a/tests/extensions/memory/test_advanced_sqlite_session.py b/tests/extensions/memory/test_advanced_sqlite_session.py index 28d8f3f6a9..1e1973ba39 100644 --- a/tests/extensions/memory/test_advanced_sqlite_session.py +++ b/tests/extensions/memory/test_advanced_sqlite_session.py @@ -3,17 +3,19 @@ import asyncio import contextlib import json +import logging import tempfile import threading from pathlib import Path from typing import Any, cast -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest pytest.importorskip("sqlalchemy") # Skip tests if SQLAlchemy is not installed from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails +import agents._debug as _debug from agents import Agent, Runner, TResponseInputItem, function_tool from agents.extensions.memory import AdvancedSQLiteSession from agents.result import RunResult @@ -154,6 +156,32 @@ async def test_advanced_session_basic_functionality(agent: Agent): session.close() +@pytest.mark.parametrize("redacted", [True, False]) +async def test_create_branch_logging_respects_model_data_policy(monkeypatch, redacted: bool): + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + mock_logger = Mock() + session = AdvancedSQLiteSession( + session_id="advanced_branch_logging", + create_tables=True, + logger=mock_logger, + ) + secret = "SECRET_BRANCH_TURN_CONTENT" + + try: + await session.add_items( + [ + {"role": "user", "content": secret}, + {"role": "assistant", "content": "response"}, + ] + ) + await session.create_branch_from_turn(1, "branch") + + logged = str(mock_logger.debug.call_args) + assert (secret not in logged) is redacted + finally: + session.close() + + async def test_advanced_session_respects_custom_table_names(): """AdvancedSQLiteSession should consistently use configured table names.""" session = AdvancedSQLiteSession( @@ -1438,6 +1466,72 @@ async def test_error_handling_in_usage_tracking(usage_data: Usage): await session.store_run_usage(run_result) +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], +) +async def test_usage_tracking_failure_identity_follows_model_data_policy( + usage_data: Usage, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + model_redacted: bool, + tool_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + session_id = "SECRET_USAGE_SESSION_ID" + test_logger = logging.getLogger("advanced-sqlite-usage-failure") + session = AdvancedSQLiteSession( + session_id=session_id, + create_tables=True, + logger=test_logger, + ) + secret = "SECRET_USAGE_FAILURE" + run_result = create_mock_run_result(usage_data) + + original_record_factory = logging.getLogRecordFactory() + + def application_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord: + record = original_record_factory(*args, **kwargs) + record.session_id = "APPLICATION_SESSION_ID" + return record + + logging.setLogRecordFactory(application_record_factory) + try: + with ( + patch.object( + session, + "_update_turn_usage_internal", + side_effect=RuntimeError(secret), + ), + caplog.at_level(logging.ERROR, logger=test_logger.name), + ): + await session.store_run_usage(run_result) + finally: + logging.setLogRecordFactory(original_record_factory) + + record = next( + record + for record in caplog.records + if "Failed to store session usage" in record.getMessage() + ) + assert record.__dict__["session_id"] == "APPLICATION_SESSION_ID" + if model_redacted: + assert record.msg == "%s" + assert record.args == ("Failed to store session usage",) + assert record.exc_info is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert secret not in caplog.text + assert session_id not in caplog.text + else: + assert record.__dict__["openai_agents_diagnostic_context"] == {"session_id": session_id} + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert secret in caplog.text + + session.close() + + async def test_advanced_tool_name_extraction(): """Test advanced tool name extraction for different tool types.""" session_id = "advanced_tool_names_test" @@ -1619,17 +1713,18 @@ async def test_session_settings_default(): session.close() -async def test_session_settings_constructor(): +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["class", "dictionary"]) +async def test_session_settings_constructor(use_dictionary: bool): """Test passing session_settings via constructor.""" from agents.memory import SessionSettings session = AdvancedSQLiteSession( session_id="constructor_settings_test", create_tables=True, - session_settings=SessionSettings(limit=5), + session_settings={"limit": 5} if use_dictionary else SessionSettings(limit=5), ) - assert session.session_settings is not None + assert isinstance(session.session_settings, SessionSettings) assert session.session_settings.limit == 5 session.close() diff --git a/tests/extensions/memory/test_async_sqlite_session.py b/tests/extensions/memory/test_async_sqlite_session.py index 7269951829..6ab3d9feb4 100644 --- a/tests/extensions/memory/test_async_sqlite_session.py +++ b/tests/extensions/memory/test_async_sqlite_session.py @@ -151,14 +151,15 @@ async def test_async_sqlite_session_session_settings_default(): await session.close() -async def test_async_sqlite_session_session_settings_constructor(): +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["class", "dictionary"]) +async def test_async_sqlite_session_session_settings_constructor(use_dictionary: bool): """Test passing session_settings via constructor.""" session = AsyncSQLiteSession( "async_constructor_settings", - session_settings=SessionSettings(limit=5), + session_settings={"limit": 5} if use_dictionary else SessionSettings(limit=5), ) - assert session.session_settings is not None + assert isinstance(session.session_settings, SessionSettings) assert session.session_settings.limit == 5 await session.close() diff --git a/tests/extensions/memory/test_dapr_session.py b/tests/extensions/memory/test_dapr_session.py index 9766f35d40..dd49173a19 100644 --- a/tests/extensions/memory/test_dapr_session.py +++ b/tests/extensions/memory/test_dapr_session.py @@ -894,7 +894,8 @@ async def test_session_settings_default(fake_dapr_client: FakeDaprClient): await session.close() -async def test_session_settings_constructor(fake_dapr_client: FakeDaprClient): +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["class", "dictionary"]) +async def test_session_settings_constructor(fake_dapr_client: FakeDaprClient, use_dictionary: bool): """Test passing session_settings via constructor.""" from agents.memory import SessionSettings @@ -902,11 +903,11 @@ async def test_session_settings_constructor(fake_dapr_client: FakeDaprClient): session_id="settings_test", state_store_name="statestore", dapr_client=fake_dapr_client, # type: ignore[arg-type] - session_settings=SessionSettings(limit=5), + session_settings={"limit": 5} if use_dictionary else SessionSettings(limit=5), ) try: - assert session.session_settings is not None + assert isinstance(session.session_settings, SessionSettings) assert session.session_settings.limit == 5 finally: await session.close() diff --git a/tests/extensions/memory/test_mongodb_session.py b/tests/extensions/memory/test_mongodb_session.py index cd7954e3ae..98cfc2654d 100644 --- a/tests/extensions/memory/test_mongodb_session.py +++ b/tests/extensions/memory/test_mongodb_session.py @@ -396,15 +396,17 @@ async def test_get_items_limit_exceeds_count(session: MongoDBSession) -> None: assert len(result) == 1 -async def test_session_settings_limit_used_as_default() -> None: +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["class", "dictionary"]) +async def test_session_settings_limit_used_as_default(use_dictionary: bool) -> None: """session_settings.limit is applied when no explicit limit is given.""" MongoDBSession._init_state.clear() s = MongoDBSession( "ls-test", client=FakeAsyncMongoClient(), # type: ignore[arg-type] database="agents_test", - session_settings=SessionSettings(limit=2), + session_settings={"limit": 2} if use_dictionary else SessionSettings(limit=2), ) + assert isinstance(s.session_settings, SessionSettings) await s.add_items([{"role": "user", "content": str(i)} for i in range(5)]) result = await s.get_items() diff --git a/tests/extensions/memory/test_redis_session.py b/tests/extensions/memory/test_redis_session.py index b5011cdd4d..0cc4c07d8b 100644 --- a/tests/extensions/memory/test_redis_session.py +++ b/tests/extensions/memory/test_redis_session.py @@ -840,7 +840,8 @@ async def test_session_settings_default(): await session.close() -async def test_session_settings_constructor(): +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["class", "dictionary"]) +async def test_session_settings_constructor(use_dictionary: bool): """Test passing session_settings via constructor.""" from agents.memory import SessionSettings @@ -849,15 +850,17 @@ async def test_session_settings_constructor(): session_id="settings_test", redis_client=fake_redis, key_prefix="test:", - session_settings=SessionSettings(limit=5), + session_settings={"limit": 5} if use_dictionary else SessionSettings(limit=5), ) else: session = RedisSession.from_url( - "settings_test", url=REDIS_URL, session_settings=SessionSettings(limit=5) + "settings_test", + url=REDIS_URL, + session_settings={"limit": 5} if use_dictionary else SessionSettings(limit=5), ) try: - assert session.session_settings is not None + assert isinstance(session.session_settings, SessionSettings) assert session.session_settings.limit == 5 finally: await session.close() diff --git a/tests/extensions/memory/test_sqlalchemy_session.py b/tests/extensions/memory/test_sqlalchemy_session.py index 091f88a482..c75d7d6141 100644 --- a/tests/extensions/memory/test_sqlalchemy_session.py +++ b/tests/extensions/memory/test_sqlalchemy_session.py @@ -836,7 +836,8 @@ async def test_session_settings_default(): assert session.session_settings.limit is None -async def test_session_settings_from_url(): +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["class", "dictionary"]) +async def test_session_settings_from_url(use_dictionary: bool): """Test passing session_settings via from_url.""" from agents.memory import SessionSettings @@ -844,10 +845,10 @@ async def test_session_settings_from_url(): "from_url_settings_test", url=DB_URL, create_tables=True, - session_settings=SessionSettings(limit=5), + session_settings={"limit": 5} if use_dictionary else SessionSettings(limit=5), ) - assert session.session_settings is not None + assert isinstance(session.session_settings, SessionSettings) assert session.session_settings.limit == 5 diff --git a/tests/extensions/sandbox/test_blaxel.py b/tests/extensions/sandbox/test_blaxel.py index 2e77fb8f80..97d0168ab3 100644 --- a/tests/extensions/sandbox/test_blaxel.py +++ b/tests/extensions/sandbox/test_blaxel.py @@ -3,6 +3,7 @@ import asyncio import io import json +import logging import tarfile import time import uuid @@ -14,6 +15,8 @@ import pytest from pydantic import ValidationError +import agents._debug as _debug +from agents.run_config import SandboxRunConfig from agents.sandbox import Manifest, SandboxPathGrant from agents.sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE from agents.sandbox.errors import ( @@ -766,6 +769,26 @@ async def test_create(self, monkeypatch: pytest.MonkeyPatch) -> None: session = await client.create(options=options) assert session is not None + @pytest.mark.asyncio + async def test_create_with_dictionary_run_config_options( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from agents.extensions.sandbox.blaxel import sandbox as mod + + monkeypatch.setattr(mod, "_import_blaxel_sdk", lambda: _FakeSandboxInstance) + + client = mod.BlaxelSandboxClient(token="test-token") + config = SandboxRunConfig( + client=client, + options={"name": "dict-options", "timeouts": {"exec_timeout_s": 120}}, + ) + + assert isinstance(config.options, mod.BlaxelSandboxClientOptions) + session = await client.create(options=config.options) + + assert isinstance(session.state, mod.BlaxelSandboxSessionState) + assert session.state.timeouts.exec_timeout_s == 120 + @pytest.mark.asyncio async def test_create_with_image(self, monkeypatch: pytest.MonkeyPatch) -> None: from agents.extensions.sandbox.blaxel import sandbox as mod @@ -3518,13 +3541,45 @@ async def test_detach_drive_success(self) -> None: assert sandbox.drives.unmount_calls == ["/mnt/data"] @pytest.mark.asyncio - async def test_detach_drive_error_logged_not_raised(self) -> None: + @pytest.mark.parametrize("redacted", [True, False], ids=["redacted", "diagnostic"]) + async def test_detach_drive_error_logged_not_raised( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, + ) -> None: from agents.extensions.sandbox.blaxel.mounts import _detach_drive + mount_path = "/mnt/SECRET_DRIVE_PATH" + error = RuntimeError("SECRET_UNMOUNT_ERROR") sandbox = _FakeSandboxInstance() - sandbox.drives.unmount_error = RuntimeError("unmount failed") + sandbox.drives.unmount_error = error + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + caplog.set_level(logging.WARNING) + # Should not raise; error is logged. - await _detach_drive(sandbox, "/mnt/data") + await _detach_drive(sandbox, mount_path) + + record = next( + record + for record in caplog.records + if "Drive detach failed" in logging.Formatter().format(record) + ) + if redacted: + assert record.msg == "%s" + assert record.args == ("Drive detach failed (non-fatal)",) + assert record.exc_info is None + assert record.exc_text is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert error not in record.__dict__.values() + rendered = logging.Formatter().format(record) + assert mount_path not in rendered + assert "SECRET_UNMOUNT_ERROR" not in rendered + else: + assert record.__dict__["openai_agents_diagnostic_context"] == {"mount_path": mount_path} + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "SECRET_UNMOUNT_ERROR" in logging.Formatter().format(record) @pytest.mark.asyncio async def test_detach_drive_no_drives_api(self) -> None: diff --git a/tests/extensions/sandbox/test_cloudflare.py b/tests/extensions/sandbox/test_cloudflare.py index 84e5ae9f39..c0a32f13de 100644 --- a/tests/extensions/sandbox/test_cloudflare.py +++ b/tests/extensions/sandbox/test_cloudflare.py @@ -50,6 +50,7 @@ def __init__(self, status: int = 200, json_body: Any = None, raw_body: bytes = b self.status = status self._json_body = json_body self._raw_body = raw_body + self.read_calls = 0 async def json(self, *, content_type: str | None = None) -> Any: _ = content_type @@ -58,6 +59,7 @@ async def json(self, *, content_type: str | None = None) -> Any: return json.loads(self._raw_body) async def read(self) -> bytes: + self.read_calls += 1 if self._json_body is not None: return json.dumps(self._json_body).encode() return self._raw_body @@ -1535,31 +1537,33 @@ def delete(self, url: str, **kwargs: Any) -> Any: @pytest.mark.asyncio -async def test_cloudflare_shutdown_logs_delete_response_details( +@pytest.mark.parametrize("redacted", [True, False]) +async def test_cloudflare_shutdown_logs_respect_tool_data_policy( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, + redacted: bool, ) -> None: - """Verify that DELETE response bodies are kept when shutdown cleanup fails.""" + """Verify that DELETE response bodies follow the tool-data logging policy.""" import logging - sess = _make_session( - fake_http=_FakeHttp( - { - "DELETE /v1/sandbox/": _FakeResponse( - status=502, - json_body={ - "error": "pool error: Failed to start container", - "code": "pool_error", - }, - ) - } - ) + monkeypatch.setattr("agents._debug.DONT_LOG_TOOL_DATA", redacted) + + response = _FakeResponse( + status=502, + json_body={ + "error": "pool error: Failed to start container", + "code": "pool_error", + }, ) + sess = _make_session(fake_http=_FakeHttp({"DELETE /v1/sandbox/": response})) with caplog.at_level(logging.DEBUG, logger="agents.extensions.sandbox.cloudflare.sandbox"): await sess._shutdown_backend() - assert any( + has_detail = any( "DELETE /sandbox failed: HTTP 502: pool_error: pool error: Failed to start container" in r.message for r in caplog.records ) + assert has_detail is not redacted + assert response.read_calls == (0 if redacted else 1) diff --git a/tests/extensions/sandbox/test_e2b.py b/tests/extensions/sandbox/test_e2b.py index b5b7d0d0a5..f830546517 100644 --- a/tests/extensions/sandbox/test_e2b.py +++ b/tests/extensions/sandbox/test_e2b.py @@ -141,14 +141,15 @@ async def test_e2b_ensure_fuse_uses_root_chmod() -> None: @pytest.mark.asyncio -async def test_e2b_ensure_rclone_installs_with_root_apt() -> None: +async def test_e2b_ensure_rclone_installs_verified_release() -> None: session = _FakeMountSession( [ _exec_fail(), # rclone missing _exec_ok(), # apt-get present + _exec_ok(stdout=b"x86_64\n"), # supported release architecture _exec_ok(), # apt-get update succeeds - _exec_ok(), # package install succeeds - _exec_ok(), # upstream rclone install succeeds + _exec_ok(), # prerequisite install succeeds + _exec_ok(), # verified rclone install succeeds _exec_ok(), # rclone now present ] ) @@ -159,20 +160,20 @@ async def test_e2b_ensure_rclone_installs_with_root_apt() -> None: "sh -lc command -v rclone >/dev/null 2>&1 || test -x /usr/local/bin/rclone", "sh -lc command -v apt-get >/dev/null 2>&1", ] - assert session.exec_calls[2] == ( + assert session.exec_calls[2] == "uname -m" + assert session.exec_calls[3] == ( "sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive " "DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 update -qq" ) - assert session.exec_calls[3] == ( + assert session.exec_calls[4] == ( "sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive " "DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 install -y -qq " - "curl unzip ca-certificates" - ) - assert ( - session.exec_calls[4] - == "sudo -u root -- sh -lc curl -fsSL https://rclone.org/install.sh | bash" + "ca-certificates coreutils curl unzip" ) - assert session.exec_calls[5] == ( + assert session.exec_calls[5].startswith("sudo -u root -- sh -lc set -eu\n") + assert "sha256sum --check --strict -" in session.exec_calls[5] + assert "rclone.org/install.sh" not in session.exec_calls[5] + assert session.exec_calls[6] == ( "sh -lc command -v rclone >/dev/null 2>&1 || test -x /usr/local/bin/rclone" ) @@ -2188,11 +2189,15 @@ async def test_e2b_stop_terminates_live_pty_sessions() -> None: @pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) async def test_e2b_shutdown_logs_pause_failure_and_falls_back_to_kill( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, + redacted: bool, ) -> None: + monkeypatch.setattr("agents._debug.DONT_LOG_TOOL_DATA", redacted) sandbox = _FakeE2BSandbox() - sandbox.pause_error = RuntimeError("pause failed") + sandbox.pause_error = RuntimeError("SECRET_E2B_PAUSE_FAILURE") state = E2BSandboxSessionState( session_id=uuid.uuid4(), manifest=Manifest(root="/workspace"), @@ -2210,12 +2215,24 @@ async def test_e2b_shutdown_logs_pause_failure_and_falls_back_to_kill( assert sandbox.pause_calls == 1 assert sandbox.kill_calls == 1 assert "Failed to pause E2B sandbox on shutdown; falling back to kill." in caplog.text + assert ("SECRET_E2B_PAUSE_FAILURE" not in caplog.text) is redacted + record = caplog.records[-1] + assert ("openai_agents_diagnostic_context" in record.__dict__) is not redacted + if not redacted: + assert record.__dict__["openai_agents_diagnostic_context"] == { + "sandbox_id": sandbox.sandbox_id, + "pause_on_exit": True, + } @pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) async def test_e2b_shutdown_logs_kill_failure_after_pause_fallback( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, + redacted: bool, ) -> None: + monkeypatch.setattr("agents._debug.DONT_LOG_TOOL_DATA", redacted) sandbox = _FakeE2BSandbox() sandbox.pause_error = RuntimeError("pause failed") sandbox.kill_error = RuntimeError("kill failed") @@ -2236,10 +2253,23 @@ async def test_e2b_shutdown_logs_kill_failure_after_pause_fallback( assert sandbox.pause_calls == 1 assert sandbox.kill_calls == 1 assert "Failed to kill E2B sandbox after pause fallback failure." in caplog.text + record = caplog.records[-1] + assert ("openai_agents_diagnostic_context" in record.__dict__) is not redacted + if not redacted: + assert record.__dict__["openai_agents_diagnostic_context"] == { + "sandbox_id": sandbox.sandbox_id, + "pause_on_exit": True, + } @pytest.mark.asyncio -async def test_e2b_shutdown_logs_direct_kill_failure(caplog: pytest.LogCaptureFixture) -> None: +@pytest.mark.parametrize("redacted", [True, False]) +async def test_e2b_shutdown_logs_direct_kill_failure( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, +) -> None: + monkeypatch.setattr("agents._debug.DONT_LOG_TOOL_DATA", redacted) sandbox = _FakeE2BSandbox() sandbox.kill_error = RuntimeError("kill failed") state = E2BSandboxSessionState( @@ -2259,6 +2289,13 @@ async def test_e2b_shutdown_logs_direct_kill_failure(caplog: pytest.LogCaptureFi assert sandbox.pause_calls == 0 assert sandbox.kill_calls == 1 assert "Failed to kill E2B sandbox on shutdown." in caplog.text + record = caplog.records[-1] + assert ("openai_agents_diagnostic_context" in record.__dict__) is not redacted + if not redacted: + assert record.__dict__["openai_agents_diagnostic_context"] == { + "sandbox_id": sandbox.sandbox_id, + "pause_on_exit": False, + } @pytest.mark.asyncio diff --git a/tests/extensions/sandbox/test_modal.py b/tests/extensions/sandbox/test_modal.py index 335139bd54..f6f52a291c 100644 --- a/tests/extensions/sandbox/test_modal.py +++ b/tests/extensions/sandbox/test_modal.py @@ -1552,6 +1552,53 @@ async def _fake_exec( ] +@pytest.mark.asyncio +async def test_modal_tar_persist_preserves_dot_prefixed_skip_paths( + monkeypatch: pytest.MonkeyPatch, +) -> None: + modal_module, _create_calls, _registry_tags = _load_modal_module(monkeypatch) + state = modal_module.ModalSandboxSessionState( + manifest=Manifest(root="/workspace"), + snapshot=modal_module.resolve_snapshot(None, "snapshot"), + app_name="sandbox-tests", + sandbox_id="sb-123", + ) + session = modal_module.ModalSandboxSession.from_state(state) + # A dot-prefixed skip path must keep its leading dot in the exclude pattern. + session.register_persist_workspace_skip_path(Path(".venv")) + + commands: list[list[str]] = [] + + async def _fake_exec( + *command: object, + timeout: float | None = None, + shell: bool | list[str] = True, + user: object | None = None, + ) -> ExecResult: + _ = (timeout, shell, user) + rendered = [str(part) for part in command] + commands.append(rendered) + return ExecResult(stdout=b"fake-tar-bytes", stderr=b"", exit_code=0) + + monkeypatch.setattr(session, "exec", _fake_exec) + + archive = await session.persist_workspace() + + assert archive.read() == b"fake-tar-bytes" + assert commands == [ + [ + "tar", + "cf", + "-", + "--exclude", + "./.venv", + "-C", + "/workspace", + ".", + ] + ] + + @pytest.mark.asyncio async def test_modal_snapshot_failure_restores_ephemeral_paths( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/extensions/sandbox/test_rclone.py b/tests/extensions/sandbox/test_rclone.py new file mode 100644 index 0000000000..5f8de7a3a4 --- /dev/null +++ b/tests/extensions/sandbox/test_rclone.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import pytest + +from agents.extensions.sandbox._rclone import ( + _RCLONE_CHECKSUM_MISMATCH_EXIT, + _RCLONE_SHA256_BY_ARCH, + _RCLONE_VERSION, + _rclone_arch, + _rclone_install_command, + ensure_rclone, +) +from agents.sandbox.errors import MountConfigError +from agents.sandbox.types import ExecResult + + +def _result(*, exit_code: int = 0, stdout: bytes = b"") -> ExecResult: + return ExecResult(stdout=stdout, stderr=b"", exit_code=exit_code) + + +class _FakeSession: + def __init__(self, results: list[ExecResult]) -> None: + self.results = results + self.calls: list[tuple[tuple[str, ...], dict[str, object]]] = [] + + async def exec(self, *command: str, **kwargs: object) -> ExecResult: + self.calls.append((command, kwargs)) + return self.results.pop(0) + + +@pytest.mark.parametrize( + ("machine", "expected"), + [ + ("x86_64", "amd64"), + ("amd64", "amd64"), + ("i386", "386"), + ("i686", "386"), + ("x86", "386"), + ("aarch64", "arm64"), + ("arm64", "arm64"), + ("armv7l", "arm-v7"), + ("armv6l", "arm-v6"), + ("armv5l", "arm"), + ], +) +def test_rclone_arch_maps_upstream_linux_archives(machine: str, expected: str) -> None: + assert _rclone_arch(machine) == expected + + +def test_rclone_arch_rejects_unknown_machine() -> None: + assert _rclone_arch("mips64") is None + + +def test_rclone_install_command_pins_and_verifies_archive() -> None: + sha256 = _RCLONE_SHA256_BY_ARCH["amd64"] + + command = _rclone_install_command("amd64", sha256) + + archive = f"rclone-v{_RCLONE_VERSION}-linux-amd64.zip" + assert f"https://downloads.rclone.org/v{_RCLONE_VERSION}/{archive}" in command + assert "github.com" not in command + assert f"expected_sha256='{sha256}'" in command + assert "sha256sum --check --strict -" in command + assert "mktemp /usr/local/bin/.rclone.XXXXXX" in command + verify_execution = 'version_output="$("$target_tmp" version)"' + verify_version = f"grep -Fx 'rclone v{_RCLONE_VERSION}'" + atomic_replace = 'mv -f "$target_tmp" /usr/local/bin/rclone' + assert command.index(verify_execution) < command.index(verify_version) + assert command.index(verify_version) < command.index(atomic_replace) + assert 'mv -f "$target_tmp" /usr/local/bin/rclone' in command + assert "curl -fsSL https://rclone.org/install.sh | bash" not in command + + +@pytest.mark.asyncio +async def test_ensure_rclone_preserves_preinstalled_binary() -> None: + session = _FakeSession([_result()]) + + await ensure_rclone(session) # type: ignore[arg-type] + + assert len(session.calls) == 1 + + +@pytest.mark.asyncio +async def test_ensure_rclone_rejects_unsupported_architecture_before_install() -> None: + session = _FakeSession( + [ + _result(exit_code=1), + _result(), + _result(stdout=b"mips64\n"), + ] + ) + + with pytest.raises(MountConfigError, match="architecture is unsupported") as exc_info: + await ensure_rclone(session) # type: ignore[arg-type] + + assert exc_info.value.context["architecture"] == "mips64" + assert len(session.calls) == 3 + + +@pytest.mark.asyncio +async def test_ensure_rclone_reports_checksum_mismatch() -> None: + session = _FakeSession( + [ + _result(exit_code=1), + _result(), + _result(stdout=b"x86_64\n"), + _result(), + _result(), + _result(exit_code=_RCLONE_CHECKSUM_MISMATCH_EXIT), + ] + ) + + with pytest.raises(MountConfigError, match="checksum verification failed") as exc_info: + await ensure_rclone(session) # type: ignore[arg-type] + + assert exc_info.value.context == { + "package": "rclone", + "version": _RCLONE_VERSION, + "architecture": "amd64", + } + assert len(session.calls) == 6 diff --git a/tests/extensions/sandbox/test_runloop_mounts.py b/tests/extensions/sandbox/test_runloop_mounts.py index 3a071515dd..8ee619fda9 100644 --- a/tests/extensions/sandbox/test_runloop_mounts.py +++ b/tests/extensions/sandbox/test_runloop_mounts.py @@ -134,13 +134,15 @@ def test_runloop_session_guard_accepts_correct_type() -> None: @pytest.mark.asyncio -async def test_runloop_ensure_rclone_installs_with_root_apt() -> None: - from agents.extensions.sandbox._rclone import ensure_rclone +async def test_runloop_ensure_rclone_installs_verified_release() -> None: + from agents.extensions.sandbox._rclone import _RCLONE_VERSION, ensure_rclone session = _FakeRunloopMountSession( [ _exec_fail(), _exec_ok(), + _exec_ok(stdout=b"aarch64\n"), + _exec_ok(), _exec_ok(), _exec_ok(), _exec_ok(), @@ -153,20 +155,20 @@ async def test_runloop_ensure_rclone_installs_with_root_apt() -> None: "sh -lc command -v rclone >/dev/null 2>&1 || test -x /usr/local/bin/rclone", "sh -lc command -v apt-get >/dev/null 2>&1", ] - assert session.exec_calls[2] == ( + assert session.exec_calls[2] == "uname -m" + assert session.exec_calls[3] == ( "sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive " "DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 update -qq" ) - assert session.exec_calls[3] == ( + assert session.exec_calls[4] == ( "sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive " "DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 install -y -qq " - "curl unzip ca-certificates" - ) - assert ( - session.exec_calls[4] - == "sudo -u root -- sh -lc curl -fsSL https://rclone.org/install.sh | bash" + "ca-certificates coreutils curl unzip" ) - assert session.exec_calls[5] == ( + assert session.exec_calls[5].startswith("sudo -u root -- sh -lc set -eu\n") + assert f"rclone-v{_RCLONE_VERSION}-linux-arm64.zip" in session.exec_calls[5] + assert "sha256sum --check --strict -" in session.exec_calls[5] + assert session.exec_calls[6] == ( "sh -lc command -v rclone >/dev/null 2>&1 || test -x /usr/local/bin/rclone" ) diff --git a/tests/extensions/sandbox/test_vercel.py b/tests/extensions/sandbox/test_vercel.py index 0430c23d13..f3f634595b 100644 --- a/tests/extensions/sandbox/test_vercel.py +++ b/tests/extensions/sandbox/test_vercel.py @@ -1,8 +1,10 @@ from __future__ import annotations +import asyncio import builtins import importlib import io +import json import sys import tarfile import types @@ -14,13 +16,27 @@ from pydantic import BaseModel, PrivateAttr from agents.sandbox import Manifest, SandboxPathGrant -from agents.sandbox.entries import File, InContainerMountStrategy, Mount, MountpointMountPattern +from agents.sandbox.entries import ( + Dir, + File, + InContainerMountStrategy, + Mount, + MountpointMountPattern, + S3Mount, +) from agents.sandbox.entries.mounts.base import InContainerMountAdapter -from agents.sandbox.errors import ConfigurationError, InvalidManifestPathError -from agents.sandbox.manifest import Environment +from agents.sandbox.errors import ( + ConfigurationError, + InvalidManifestPathError, + MountCommandError, + MountConfigError, +) +from agents.sandbox.manifest import EnvEntry, Environment, StrEnvValue from agents.sandbox.materialization import MaterializedFile from agents.sandbox.session.base_sandbox_session import BaseSandboxSession from agents.sandbox.session.dependencies import Dependencies +from agents.sandbox.session.manager import Instrumentation +from agents.sandbox.session.sinks import CallbackSink from agents.sandbox.snapshot import NoopSnapshot, SnapshotBase from agents.sandbox.types import User from tests._fake_workspace_paths import resolve_fake_workspace_path @@ -120,6 +136,22 @@ async def restorable(self, *, dependencies: Dependencies | None = None) -> bool: return self.is_restorable +class _FailingPersistSnapshot(SnapshotBase): + type: Literal["test-vercel-failing-persist"] = "test-vercel-failing-persist" + + async def persist(self, data: io.IOBase, *, dependencies: Dependencies | None = None) -> None: + _ = (data, dependencies) + raise RuntimeError("snapshot persist failed") + + async def restore(self, *, dependencies: Dependencies | None = None) -> io.IOBase: + _ = dependencies + return io.BytesIO() + + async def restorable(self, *, dependencies: Dependencies | None = None) -> bool: + _ = dependencies + return False + + class _FakeCommandFinished: def __init__(self, *, stdout: str = "", stderr: str = "", exit_code: int = 0) -> None: self._stdout = stdout @@ -170,9 +202,17 @@ def __init__( self.client = _FakeClient() self.next_command_result = _FakeCommandFinished() self.run_command_calls: list[tuple[str, list[str], str | None]] = [] + self.run_command_options: list[tuple[str, dict[str, str] | None, bool]] = [] + self.command_results: dict[str, list[_FakeCommandFinished]] = {} + self.command_started: dict[str, asyncio.Event] = {} + self.command_waiters: dict[str, asyncio.Event] = {} self.refresh_calls = 0 self.read_file_calls: list[tuple[str, str | None]] = [] self.stop_calls = 0 + self.stop_blocking_calls: list[bool] = [] + self.stop_failures: list[BaseException] = [] + self.stop_started: asyncio.Event | None = None + self.stop_waiters: list[asyncio.Event] = [] self.wait_for_status_calls: list[tuple[object, float | None]] = [] self.wait_for_status_error: BaseException | None = None self.write_failures: list[BaseException] = [] @@ -250,9 +290,16 @@ async def run_command( env: dict[str, str] | None = None, sudo: bool = False, ) -> _FakeCommandFinished: - _ = (env, sudo) args = args or [] self.run_command_calls.append((cmd, list(args), cwd)) + self.run_command_options.append((cmd, env, sudo)) + if started := self.command_started.get(cmd): + started.set() + if waiter := self.command_waiters.get(cmd): + await waiter.wait() + queued_results = self.command_results.get(cmd) + if queued_results: + return queued_results.pop(0) resolved = resolve_fake_workspace_path( (cmd, *args), symlinks=self.symlinks, @@ -311,136 +358,1642 @@ async def run_command( return _FakeCommandFinished() return self.next_command_result - async def read_file(self, path: str, *, cwd: str | None = None) -> bytes | None: - self.read_file_calls.append((path, cwd)) - resolved = path if path.startswith("/") or cwd is None else f"{cwd.rstrip('/')}/{path}" - return self.files.get(resolved) + async def read_file(self, path: str, *, cwd: str | None = None) -> bytes | None: + self.read_file_calls.append((path, cwd)) + resolved = path if path.startswith("/") or cwd is None else f"{cwd.rstrip('/')}/{path}" + return self.files.get(resolved) + + async def write_files(self, files: list[dict[str, object]]) -> None: + self.write_files_calls.append(files) + if self.write_failures: + raise self.write_failures.pop(0) + for file in files: + self.files[str(file["path"])] = bytes(cast(bytes, file["content"])) + + async def stop( + self, *, blocking: bool = False, timeout: float = 30.0, poll_interval: float = 0.5 + ) -> None: + _ = (blocking, timeout, poll_interval) + self.stop_calls += 1 + self.stop_blocking_calls.append(blocking) + if self.stop_started is not None: + self.stop_started.set() + if self.stop_waiters: + await self.stop_waiters.pop(0).wait() + if self.stop_failures: + raise self.stop_failures.pop(0) + self.status = "stopped" + + async def snapshot(self, *, expiration: int | None = None) -> _FakeAsyncSnapshot: + _ = expiration + type(self).snapshot_counter += 1 + snapshot_id = f"vercel-snapshot-{type(self).snapshot_counter}" + type(self).snapshots[snapshot_id] = dict(self.files) + self.status = "stopped" + return _FakeAsyncSnapshot(snapshot_id) + + +class _RecordingMount(Mount): + type: str = "test_vercel_recording_mount" + bucket: str = "bucket" + _events: list[tuple[str, str]] = PrivateAttr(default_factory=list) + + def supported_in_container_patterns( + self, + ) -> tuple[builtins.type[MountpointMountPattern], ...]: + return (MountpointMountPattern,) + + def in_container_adapter(self) -> InContainerMountAdapter: + mount = self + + class _Adapter(InContainerMountAdapter): + def validate(self, strategy: InContainerMountStrategy) -> None: + super().validate(strategy) + + async def activate( + self, + strategy: InContainerMountStrategy, + session: BaseSandboxSession, + dest: Path, + base_dir: Path, + ) -> list[MaterializedFile]: + _ = (strategy, session, dest, base_dir) + return [] + + async def deactivate( + self, + strategy: InContainerMountStrategy, + session: BaseSandboxSession, + dest: Path, + base_dir: Path, + ) -> None: + _ = (strategy, session, dest, base_dir) + + async def teardown_for_snapshot( + self, + strategy: InContainerMountStrategy, + session: BaseSandboxSession, + path: Path, + ) -> None: + _ = strategy + mount._events.append(("unmount", path.as_posix())) + sandbox = cast(Any, session)._sandbox + if sandbox is not None: + sandbox.files.pop(f"{path.as_posix()}/mounted.txt", None) + + async def restore_after_snapshot( + self, + strategy: InContainerMountStrategy, + session: BaseSandboxSession, + path: Path, + ) -> None: + _ = strategy + mount._events.append(("mount", path.as_posix())) + sandbox = cast(Any, session)._sandbox + if sandbox is not None: + sandbox.files[f"{path.as_posix()}/mounted.txt"] = b"mounted-content" + + return _Adapter(self) + + +def _load_vercel_module(monkeypatch: pytest.MonkeyPatch) -> Any: + _FakeAsyncSandbox.reset() + + fake_vercel = types.ModuleType("vercel") + fake_vercel_sandbox = cast(Any, types.ModuleType("vercel.sandbox")) + fake_vercel_sandbox.AsyncSandbox = _FakeAsyncSandbox + fake_vercel_sandbox.NetworkPolicy = NetworkPolicy + fake_vercel_sandbox.NetworkPolicyCustom = NetworkPolicyCustom + fake_vercel_sandbox.NetworkPolicyRule = NetworkPolicyRule + fake_vercel_sandbox.NetworkPolicySubnets = NetworkPolicySubnets + fake_vercel_sandbox.Resources = Resources + fake_vercel_sandbox.SandboxAuthError = _FakeVercelSandboxAuthError + fake_vercel_sandbox.SandboxNotFoundError = _FakeVercelSandboxNotFoundError + fake_vercel_sandbox.SandboxPermissionError = _FakeVercelSandboxPermissionError + fake_vercel_sandbox.SandboxRateLimitError = _FakeVercelSandboxRateLimitError + fake_vercel_sandbox.SandboxServerError = _FakeVercelSandboxServerError + fake_vercel_sandbox.SandboxStatus = types.SimpleNamespace(RUNNING="running") + fake_vercel_sandbox.SandboxValidationError = _FakeVercelSandboxValidationError + fake_vercel_sandbox.SnapshotSource = SnapshotSource + cast(Any, fake_vercel).sandbox = fake_vercel_sandbox + + monkeypatch.setitem(sys.modules, "vercel", fake_vercel) + monkeypatch.setitem(sys.modules, "vercel.sandbox", fake_vercel_sandbox) + sys.modules.pop("agents.extensions.sandbox.vercel.mounts", None) + sys.modules.pop("agents.extensions.sandbox.vercel.sandbox", None) + sys.modules.pop("agents.extensions.sandbox.vercel", None) + + return importlib.import_module("agents.extensions.sandbox.vercel.sandbox") + + +async def _noop_sleep(*_args: object, **_kwargs: object) -> None: + return None + + +def test_vercel_package_re_exports_backend_symbols(monkeypatch: pytest.MonkeyPatch) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + + assert package_module.VercelCloudBucketMountStrategy.__name__ == ( + "VercelCloudBucketMountStrategy" + ) + assert package_module.VercelSandboxClient is vercel_module.VercelSandboxClient + assert package_module.VercelSandboxSessionState is vercel_module.VercelSandboxSessionState + + +def _vercel_s3_manifest( + package_module: Any, + *, + credentials: bool = False, + mount_path: Path | None = None, +) -> Manifest: + return Manifest( + root="/workspace", + entries={ + "remote": S3Mount( + bucket="test-bucket", + access_key_id="test-access-key" if credentials else None, + secret_access_key="test-secret-key" if credentials else None, + session_token="test-session-token" if credentials else None, + region="us-west-2", + mount_path=mount_path, + mount_strategy=package_module.VercelCloudBucketMountStrategy(), + ) + }, + ) + + +def _queue_successful_s3_mounts(sandbox: _FakeAsyncSandbox, count: int = 1) -> None: + sandbox.command_results.update( + { + "/usr/bin/test": [_FakeCommandFinished() for _ in range(count)], + "/usr/bin/rpm": [_FakeCommandFinished(stdout="1.21.0") for _ in range(count)], + "/usr/bin/find": [_FakeCommandFinished() for _ in range(count)], + "/usr/bin/mount-s3": [_FakeCommandFinished() for _ in range(count)], + } + ) + + +def test_vercel_s3_mount_validates_credentials_and_lifecycle( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + + with pytest.raises(MountConfigError, match="require both"): + S3Mount( + bucket="test-bucket", + access_key_id="test-access-key", + mount_strategy=package_module.VercelCloudBucketMountStrategy(), + ) + + with pytest.raises(MountConfigError, match="must be ephemeral"): + S3Mount( + bucket="test-bucket", + ephemeral=False, + mount_strategy=package_module.VercelCloudBucketMountStrategy(), + ) + + +@pytest.mark.asyncio +async def test_vercel_create_requires_explicit_s3_credential_exposure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + + with pytest.raises(MountConfigError, match="allow_s3_credential_exposure"): + await client.create( + manifest=_vercel_s3_manifest(package_module, credentials=True), + options=vercel_module.VercelSandboxClientOptions(), + ) + + assert _FakeAsyncSandbox.create_calls == [] + + +@pytest.mark.asyncio +async def test_vercel_create_revalidates_mutated_s3_mount( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + manifest = _vercel_s3_manifest(package_module) + mount = cast(S3Mount, manifest.entries["remote"]) + mount.ephemeral = False + + with pytest.raises(MountConfigError, match="must be ephemeral"): + await vercel_module.VercelSandboxClient().create( + manifest=manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + + assert _FakeAsyncSandbox.create_calls == [] + + +@pytest.mark.asyncio +async def test_vercel_rejects_root_and_overlapping_s3_mounts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + strategy = package_module.VercelCloudBucketMountStrategy + client = vercel_module.VercelSandboxClient() + + root_manifest = Manifest( + root="/custom-workspace", + entries={ + "remote": S3Mount( + bucket="root-bucket", + mount_path=Path("/custom-workspace"), + mount_strategy=strategy(), + ) + }, + ) + with pytest.raises(MountConfigError, match="workspace root"): + await client.create( + manifest=root_manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + + outside_manifest = Manifest( + root="/workspace", + entries={ + "remote": S3Mount( + bucket="outside-bucket", + mount_path=Path("/tmp/remote"), + mount_strategy=strategy(), + ) + }, + extra_path_grants=(SandboxPathGrant(path="/tmp/remote"),), + ) + with pytest.raises(MountConfigError, match="within the workspace root"): + await client.create( + manifest=outside_manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + + overlapping_manifest = Manifest( + root="/workspace", + entries={ + "remote": S3Mount(bucket="outer", mount_strategy=strategy()), + "remote/nested": S3Mount(bucket="inner", mount_strategy=strategy()), + }, + ) + with pytest.raises(MountConfigError, match="must not overlap"): + await client.create( + manifest=overlapping_manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + + physical_overlap_manifest = Manifest( + root="/workspace", + entries={ + "remote": S3Mount( + bucket="physical-overlap", + mount_path=Path("actual"), + mount_strategy=strategy(), + ), + "actual/config.json": File(content=b"{}"), + }, + ) + with pytest.raises(MountConfigError, match="must not overlap manifest entries"): + await client.create( + manifest=physical_overlap_manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + + assert _FakeAsyncSandbox.create_calls == [] + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_is_create_time_only_and_credentials_are_not_serialized( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=_vercel_s3_manifest(package_module, credentials=True), + options=vercel_module.VercelSandboxClientOptions(allow_s3_credential_exposure=True), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + sandbox.command_results = { + "/usr/bin/test": [_FakeCommandFinished()], + "/usr/bin/rpm": [_FakeCommandFinished(stdout="1.21.0")], + "/usr/bin/find": [_FakeCommandFinished()], + "/usr/bin/mount-s3": [_FakeCommandFinished()], + } + + await session.start() + + mount_call = next( + options for options in sandbox.run_command_options if options[0] == "/usr/bin/mount-s3" + ) + assert mount_call == ( + "/usr/bin/mount-s3", + { + "AWS_ACCESS_KEY_ID": "test-access-key", + "AWS_SECRET_ACCESS_KEY": "test-secret-key", + "AWS_SESSION_TOKEN": "test-session-token", + "AWS_REGION": "us-west-2", + }, + True, + ) + state_mount = cast(S3Mount, session.state.manifest.entries["remote"]) + assert state_mount.access_key_id is None + assert state_mount.secret_access_key is None + assert state_mount.session_token is None + + payload = client.serialize_session_state(session.state) + serialized = json.dumps(payload, sort_keys=True) + assert "test-access-key" not in serialized + assert "test-secret-key" not in serialized + assert "test-session-token" not in serialized + assert "vercel_cloud_bucket" in serialized + + remote_mount = session.state.manifest.entries.pop("remote") + session.state.manifest.entries = { + "before.txt": File(content=b"must-not-write", ephemeral=True), + "remote": remote_mount, + } + write_call_count = len(sandbox.write_files_calls) + with pytest.raises(MountConfigError, match="dynamic manifest application"): + await session.apply_manifest(only_ephemeral=True) + assert len(sandbox.write_files_calls) == write_call_count + + session.state.manifest.entries.pop("remote") + mutated_payload = client.serialize_session_state(session.state) + assert mutated_payload["s3_mounts_non_resumable"] is True + assert "vercel_cloud_bucket" not in json.dumps(mutated_payload, sort_keys=True) + + restored = client.deserialize_session_state(mutated_payload) + with pytest.raises(MountConfigError, match="cannot be resumed"): + await client.resume(restored) + assert _FakeAsyncSandbox.get_calls == [] + + +@pytest.mark.asyncio +async def test_vercel_s3_dynamic_mount_is_rejected_before_materialization( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=Manifest(entries={"before.txt": File(content=b"must-not-write", ephemeral=True)}), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + session.state.manifest.entries["remote"] = S3Mount( + bucket="test-bucket", + mount_strategy=package_module.VercelCloudBucketMountStrategy(), + ) + + with pytest.raises(MountConfigError, match="dynamic manifest application"): + await session.apply_manifest(only_ephemeral=True) + + assert sandbox.write_files_calls == [] + with pytest.raises(MountConfigError, match="topology cannot change"): + await session.persist_workspace() + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_starts_after_restorable_tar_snapshot( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + archive = io.BytesIO() + with tarfile.open(fileobj=archive, mode="w"): + pass + snapshot = _MemorySnapshot( + id="snapshot", + payload=archive.getvalue(), + is_restorable=True, + ) + client = vercel_module.VercelSandboxClient() + session = await client.create( + snapshot=snapshot, + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + + await session.start() + + assert len([call for call in sandbox.run_command_calls if call[0] == "/usr/bin/mount-s3"]) == 1 + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_snapshot_entries_materialize_before_mount_activation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + archive = io.BytesIO() + with tarfile.open(fileobj=archive, mode="w"): + pass + manifest = Manifest( + root="/workspace", + entries={ + "remote": S3Mount( + bucket="test-bucket", + mount_path=Path("actual/remote"), + mount_strategy=package_module.VercelCloudBucketMountStrategy(), + ), + "alias/remote/config.json": File(content=b"{}", ephemeral=True), + }, + ) + session = await vercel_module.VercelSandboxClient().create( + snapshot=_MemorySnapshot( + id="snapshot", + payload=archive.getvalue(), + is_restorable=True, + ), + manifest=manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + sandbox.symlinks["/vercel/sandbox/alias"] = "/vercel/sandbox/actual" + sandbox.command_results = { + "/usr/bin/rpm": [_FakeCommandFinished(stdout="1.21.0")], + "/usr/bin/test": [_FakeCommandFinished()], + "/usr/bin/find": [_FakeCommandFinished(stdout="/vercel/sandbox/actual/remote/config.json")], + } + + with pytest.raises(MountConfigError, match="require an empty mount directory"): + await session.start() + + assert [ + {"path": "/vercel/sandbox/alias/remote/config.json", "content": b"{}"} + ] in sandbox.write_files_calls + assert not any(call[0] == "/usr/bin/mount-s3" for call in sandbox.run_command_calls) + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_snapshots_trusted_create_time_configuration( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + manifest = _vercel_s3_manifest(package_module) + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + + supplied_mount = cast(S3Mount, manifest.entries["remote"]) + supplied_mount.bucket = "mutated-bucket" + supplied_mount.access_key_id = "mutated-access-key" + supplied_mount.secret_access_key = "mutated-secret-key" + + await session.start() + + mount_call = next(call for call in sandbox.run_command_calls if call[0] == "/usr/bin/mount-s3") + mount_options = next( + options for options in sandbox.run_command_options if options[0] == "/usr/bin/mount-s3" + ) + assert mount_call[1][0] == "test-bucket" + assert mount_options[1] == {"AWS_REGION": "us-west-2"} + + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_rejects_symlink_components( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=_vercel_s3_manifest(package_module, mount_path=Path("link/remote")), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + sandbox.symlinks["/vercel/sandbox/link"] = "/vercel/sandbox/durable" + + with pytest.raises(MountConfigError, match="must not resolve through symlinks"): + await session.start() + + assert not any(call[0] == "/usr/bin/mount-s3" for call in sandbox.run_command_calls) + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_entry_failure_happens_before_mount_activation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + manifest = _vercel_s3_manifest(package_module) + manifest.entries["later.txt"] = File(content=b"later") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + sandbox.write_failures = [RuntimeError("later entry failed")] + + with pytest.raises(vercel_module.WorkspaceArchiveWriteError): + await session.start() + + mount_calls = [call for call in sandbox.run_command_calls if call[0] == "/usr/bin/mount-s3"] + assert mount_calls == [] + assert sandbox.stop_calls == 0 + + await session.start() + assert len([call for call in sandbox.run_command_calls if call[0] == "/usr/bin/mount-s3"]) == 1 + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_entry_cancellation_happens_before_mount_activation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + manifest = _vercel_s3_manifest(package_module) + manifest.entries["later.txt"] = File(content=b"later") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + write_started = asyncio.Event() + hold_write = asyncio.Event() + original_write_files = sandbox.write_files + + async def blocking_write_files(files: list[dict[str, object]]) -> None: + _ = files + write_started.set() + await hold_write.wait() + + monkeypatch.setattr(sandbox, "write_files", blocking_write_files) + start_task = asyncio.create_task(session.start()) + await asyncio.wait_for(write_started.wait(), timeout=1) + start_task.cancel() + with pytest.raises(asyncio.CancelledError): + await start_task + + assert sandbox.stop_calls == 0 + assert not any(call[0] == "/usr/bin/mount-s3" for call in sandbox.run_command_calls) + + monkeypatch.setattr(sandbox, "write_files", original_write_files) + await session.start() + assert len([call for call in sandbox.run_command_calls if call[0] == "/usr/bin/mount-s3"]) == 1 + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_nested_activation_serializes_workspace_commands( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + manifest = Manifest( + entries={ + "parent": Dir( + children={ + "remote": S3Mount( + bucket="test-bucket", + mount_strategy=package_module.VercelCloudBucketMountStrategy(), + ) + } + ) + } + ) + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + mount_started = asyncio.Event() + release_mount = asyncio.Event() + sandbox.command_started["/usr/bin/mount-s3"] = mount_started + sandbox.command_waiters["/usr/bin/mount-s3"] = release_mount + + start_task = asyncio.create_task(session.start()) + await asyncio.wait_for(mount_started.wait(), timeout=1) + apply_task = asyncio.create_task(session.apply_manifest(only_ephemeral=True)) + exec_task = asyncio.create_task(session.exec("true", shell=False)) + await asyncio.sleep(0) + assert not exec_task.done() + + release_mount.set() + await start_task + with pytest.raises(MountConfigError, match="dynamic manifest application"): + await apply_task + assert (await exec_task).ok() + assert len([call for call in sandbox.run_command_calls if call[0] == "/usr/bin/mount-s3"]) == 1 + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_manifest_sanitization_preserves_typed_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + manifest = _vercel_s3_manifest(package_module, credentials=True) + manifest.environment = Environment( + value={ + "DIRECT": StrEnvValue(value="direct-value"), + "ENTRY": EnvEntry( + description="typed entry", + ephemeral=True, + value=StrEnvValue(value="entry-value"), + ), + } + ) + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=manifest, + options=vercel_module.VercelSandboxClientOptions( + allow_s3_credential_exposure=True, + ), + ) + + state_environment = session.state.manifest.environment.value + assert state_environment["DIRECT"] == StrEnvValue(value="direct-value") + assert state_environment["ENTRY"] == EnvEntry( + description="typed entry", + ephemeral=True, + value=StrEnvValue(value="entry-value"), + ) + payload = client.serialize_session_state(session.state) + serialized_environment = cast( + dict[str, object], + cast(dict[str, object], payload["manifest"])["environment"], + ) + assert serialized_environment == { + "value": { + "DIRECT": {"value": "direct-value"}, + "ENTRY": { + "description": "typed entry", + "ephemeral": True, + "value": {"value": "entry-value"}, + }, + } + } + serialized = json.dumps(payload, sort_keys=True) + assert "test-access-key" not in serialized + assert "test-secret-key" not in serialized + assert "test-session-token" not in serialized + + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_detaches_for_tar_persistence_and_unmounts_on_shutdown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + snapshot = _MemorySnapshot(id="snapshot") + client = vercel_module.VercelSandboxClient() + session = await client.create( + snapshot=snapshot, + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions( + workspace_persistence="snapshot", + ), + ) + fingerprint_called = False + + async def unexpected_fingerprint() -> dict[str, str]: + nonlocal fingerprint_called + fingerprint_called = True + return {"fingerprint": "unexpected", "version": "unexpected"} + + monkeypatch.setattr( + session._inner, + "_compute_and_cache_snapshot_fingerprint", + unexpected_fingerprint, + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + sandbox.files[f"{session.state.manifest.root}/kept.txt"] = b"kept" + sandbox.command_results = { + "/usr/bin/test": [_FakeCommandFinished(), _FakeCommandFinished()], + "/usr/bin/rpm": [ + _FakeCommandFinished(stdout="1.21.0"), + _FakeCommandFinished(stdout="1.21.0"), + ], + "/usr/bin/find": [_FakeCommandFinished(), _FakeCommandFinished()], + "/usr/bin/mount-s3": [_FakeCommandFinished(), _FakeCommandFinished()], + "/usr/bin/findmnt": [ + _FakeCommandFinished(stdout="mountpoint-s3"), + _FakeCommandFinished(stdout="mountpoint-s3"), + ], + "/usr/bin/umount": [_FakeCommandFinished(), _FakeCommandFinished()], + } + + await session.start() + await session.stop() + await session.shutdown() + + assert fingerprint_called is False + lifecycle_commands = [ + command + for command, _args, _cwd in sandbox.run_command_calls + if command + in { + "/usr/bin/findmnt", + "/usr/bin/mount-s3", + "/usr/bin/umount", + "tar", + } + ] + assert lifecycle_commands == [ + "/usr/bin/mount-s3", + "/usr/bin/findmnt", + "/usr/bin/umount", + "tar", + "/usr/bin/mount-s3", + "/usr/bin/findmnt", + "/usr/bin/umount", + ] + assert _FakeAsyncSandbox.snapshot_counter == 0 + assert sandbox.stop_calls == 1 + with tarfile.open(fileobj=io.BytesIO(snapshot.payload), mode="r") as archive: + assert [member.name for member in archive.getmembers()] == ["kept.txt"] + + +@pytest.mark.asyncio +async def test_vercel_s3_hydrate_rejects_mount_overlaps_before_detach( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + await session.start() + + archive = io.BytesIO() + with tarfile.open(fileobj=archive, mode="w") as tar: + payload = b"must-not-be-written" + member = tarfile.TarInfo("remote/hidden.txt") + member.size = len(payload) + tar.addfile(member, io.BytesIO(payload)) + + with pytest.raises( + vercel_module.WorkspaceArchiveWriteError, + match="failed to write archive", + ): + await session.hydrate_workspace(io.BytesIO(archive.getvalue())) + with pytest.raises(MountConfigError, match="native snapshot"): + await session.hydrate_workspace( + io.BytesIO(vercel_module._encode_snapshot_ref(snapshot_id="snapshot-id")) + ) + + assert not any(call[0] == "/usr/bin/findmnt" for call in sandbox.run_command_calls) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_aclose_retries_failed_transition_stop( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + snapshot=_MemorySnapshot(id="snapshot"), + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [ + _FakeCommandFinished(stdout="mountpoint-s3"), + _FakeCommandFinished(stdout="mountpoint-s3"), + ], + "/usr/bin/umount": [_FakeCommandFinished(stderr="busy", exit_code=32)], + } + ) + sandbox.stop_failures = [RuntimeError("stop failed")] + await session.start() + + with pytest.raises(vercel_module.WorkspaceArchiveReadError): + await session.aclose() + create_count = len(_FakeAsyncSandbox.create_calls) + with pytest.raises(vercel_module.WorkspaceStartError, match="failed to start session"): + await session.exec("true", shell=False) + + assert sandbox.stop_calls == 2 + assert sandbox.stop_blocking_calls == [True, True] + assert session._inner._sandbox is None + assert len(_FakeAsyncSandbox.create_calls) == create_count + + +@pytest.mark.asyncio +async def test_vercel_s3_missing_tracked_mount_stops_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + snapshot=_MemorySnapshot(id="snapshot"), + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + await session.start() + sandbox.command_results["/usr/bin/findmnt"] = [_FakeCommandFinished(exit_code=1)] + + with pytest.raises(vercel_module.WorkspaceArchiveReadError): + await session.persist_workspace() + + assert sandbox.stop_calls == 1 + assert sandbox.stop_blocking_calls == [True] + assert session._inner._sandbox is None + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_disappearing_during_unmount_stops_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + session = await vercel_module.VercelSandboxClient().create( + snapshot=_MemorySnapshot(id="snapshot"), + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + await session.start() + sandbox.command_results.update( + { + "/usr/bin/findmnt": [ + _FakeCommandFinished(stdout="mountpoint-s3"), + _FakeCommandFinished(exit_code=1), + ], + "/usr/bin/umount": [_FakeCommandFinished(stderr="missing", exit_code=32)], + } + ) + + with pytest.raises(vercel_module.WorkspaceArchiveReadError): + await session.persist_workspace() + + assert sandbox.stop_calls == 1 + assert sandbox.stop_blocking_calls == [True] + assert session._inner._sandbox is None + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_unexpected_persist_error_stops_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + snapshot=_MemorySnapshot(id="snapshot"), + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + await session.start() + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + + async def missing_archive(_path: str, *, cwd: str | None = None) -> bytes | None: + _ = cwd + return None + + monkeypatch.setattr(sandbox, "read_file", missing_archive) + + with pytest.raises(vercel_module.WorkspaceReadNotFoundError): + await session.persist_workspace() + + assert sandbox.stop_calls == 1 + assert session._inner._sandbox is None + with pytest.raises(vercel_module.WorkspaceStartError) as exc_info: + await session.exec("true", shell=False) + assert exc_info.value.context["reason"] == "mount_transition_failed" + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_aclose_shuts_down_after_snapshot_persist_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + snapshot=_FailingPersistSnapshot(id="snapshot"), + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox, count=2) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [ + _FakeCommandFinished(stdout="mountpoint-s3"), + _FakeCommandFinished(stdout="mountpoint-s3"), + ], + "/usr/bin/umount": [ + _FakeCommandFinished(), + _FakeCommandFinished(), + ], + } + ) + + with pytest.raises(RuntimeError, match="snapshot persist failed"): + async with session: + pass + + assert sandbox.stop_calls == 1 + assert sandbox.stop_blocking_calls == [True] + assert session._inner._sandbox is None + + +@pytest.mark.asyncio +async def test_vercel_s3_stop_preserves_session_after_snapshot_persist_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + session = await vercel_module.VercelSandboxClient().create( + snapshot=_FailingPersistSnapshot(id="snapshot"), + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox, count=2) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [ + _FakeCommandFinished(stdout="mountpoint-s3"), + _FakeCommandFinished(stdout="mountpoint-s3"), + ], + "/usr/bin/umount": [ + _FakeCommandFinished(), + _FakeCommandFinished(), + ], + } + ) + await session.start() + + with pytest.raises(RuntimeError, match="snapshot persist failed"): + await session.stop() + + assert sandbox.stop_calls == 0 + assert session._inner._sandbox is sandbox + assert session._inner._active_s3_mount_paths == {"/vercel/sandbox/remote"} + + await session.shutdown() + assert sandbox.stop_calls == 1 + + +@pytest.mark.asyncio +async def test_vercel_s3_aclose_shuts_down_after_pre_stop_hook_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + + async def failing_hook() -> None: + raise RuntimeError("pre-stop hook failed") + + session.register_pre_stop_hook(failing_hook) + with pytest.raises(RuntimeError, match="pre-stop hook failed"): + async with session: + pass + + assert sandbox.stop_calls == 1 + assert sandbox.stop_blocking_calls == [True] + assert session._inner._sandbox is None + + +@pytest.mark.asyncio +async def test_vercel_s3_aclose_bypasses_failing_instrumentation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + + def fail_stop_start(event: Any, _session: BaseSandboxSession) -> None: + if event.op == "stop" and event.phase == "start": + raise RuntimeError("stop sink failed") + + client = vercel_module.VercelSandboxClient( + instrumentation=Instrumentation( + sinks=[CallbackSink(fail_stop_start, mode="sync", on_error="raise")] + ) + ) + session = await client.create( + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.start() + + with pytest.raises(RuntimeError, match="sandbox event sink failed"): + await session.aclose() + + assert sandbox.stop_calls == 1 + assert sandbox.stop_blocking_calls == [True] + assert session._inner._sandbox is None + + +@pytest.mark.asyncio +async def test_vercel_s3_closed_session_does_not_recreate_sandbox( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.start() + create_count = len(_FakeAsyncSandbox.create_calls) + + await session.shutdown() + await session.shutdown() + await session.aclose() + + with pytest.raises(vercel_module.WorkspaceStartError) as exec_error: + await session.exec("true", shell=False) + assert exec_error.value.context["reason"] == "mounted_session_closed" + with pytest.raises(vercel_module.WorkspaceStartError): + await session.start() + + assert sandbox.stop_calls == 1 + assert len(_FakeAsyncSandbox.create_calls) == create_count + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_cancellation_stops_and_marks_session_unusable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + snapshot=_MemorySnapshot(id="snapshot"), + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + output_started = asyncio.Event() + hold_output = asyncio.Event() + + class _BlockingUnmountResult(_FakeCommandFinished): + async def stdout(self) -> str: + output_started.set() + await hold_output.wait() + return "" + + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_BlockingUnmountResult()], + } + ) + await session.start() + + stop_task = asyncio.create_task(session.stop()) + await asyncio.wait_for(output_started.wait(), timeout=1) + stop_task.cancel() + with pytest.raises(asyncio.CancelledError): + await stop_task + + create_count = len(_FakeAsyncSandbox.create_calls) + with pytest.raises(vercel_module.WorkspaceStartError, match="failed to start session"): + await session.exec("true", shell=False) + assert sandbox.stop_calls == 1 + assert len(_FakeAsyncSandbox.create_calls) == create_count + await session.shutdown() + + +@pytest.mark.asyncio +async def test_vercel_s3_shutdown_cancellation_finishes_stop_and_marks_session_unusable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + await session.start() + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + sandbox.stop_started = asyncio.Event() + sandbox.stop_waiters = [asyncio.Event()] + shutdown_task = asyncio.create_task(session.shutdown()) + await asyncio.wait_for(sandbox.stop_started.wait(), timeout=1) + shutdown_task.cancel() + + with pytest.raises(asyncio.CancelledError): + await shutdown_task - async def write_files(self, files: list[dict[str, object]]) -> None: - self.write_files_calls.append(files) - if self.write_failures: - raise self.write_failures.pop(0) - for file in files: - self.files[str(file["path"])] = bytes(cast(bytes, file["content"])) + assert sandbox.stop_calls == 2 + assert sandbox.stop_blocking_calls == [True, True] + assert session._inner._sandbox is None + with pytest.raises(vercel_module.WorkspaceStartError) as exc_info: + await session.exec("true", shell=False) + assert exc_info.value.context["reason"] == "mount_transition_failed" + await session.shutdown() - async def stop( - self, *, blocking: bool = False, timeout: float = 30.0, poll_interval: float = 0.5 - ) -> None: - _ = (blocking, timeout, poll_interval) - self.stop_calls += 1 - self.status = "stopped" - async def snapshot(self, *, expiration: int | None = None) -> _FakeAsyncSnapshot: - _ = expiration - type(self).snapshot_counter += 1 - snapshot_id = f"vercel-snapshot-{type(self).snapshot_counter}" - type(self).snapshots[snapshot_id] = dict(self.files) - self.status = "stopped" - return _FakeAsyncSnapshot(snapshot_id) +@pytest.mark.asyncio +async def test_vercel_s3_archive_cancellation_stops_detached_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + snapshot=_MemorySnapshot(id="snapshot"), + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + archive_started = asyncio.Event() + hold_archive = asyncio.Event() + sandbox.command_started["tar"] = archive_started + sandbox.command_waiters["tar"] = hold_archive + await session.start() + stop_task = asyncio.create_task(session.stop()) + await asyncio.wait_for(archive_started.wait(), timeout=1) + stop_task.cancel() + with pytest.raises(asyncio.CancelledError): + await stop_task -class _RecordingMount(Mount): - type: str = "test_vercel_recording_mount" - bucket: str = "bucket" - _events: list[tuple[str, str]] = PrivateAttr(default_factory=list) + create_count = len(_FakeAsyncSandbox.create_calls) + with pytest.raises(vercel_module.WorkspaceStartError, match="failed to start session"): + await session.exec("true", shell=False) + assert sandbox.stop_calls == 1 + assert len(_FakeAsyncSandbox.create_calls) == create_count + await session.shutdown() - def supported_in_container_patterns( - self, - ) -> tuple[builtins.type[MountpointMountPattern], ...]: - return (MountpointMountPattern,) - def in_container_adapter(self) -> InContainerMountAdapter: - mount = self +@pytest.mark.asyncio +async def test_vercel_s3_rejects_state_topology_changes_and_cleans_fixed_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + await session.start() - class _Adapter(InContainerMountAdapter): - def validate(self, strategy: InContainerMountStrategy) -> None: - super().validate(strategy) + state_mount = cast(S3Mount, session.state.manifest.entries["remote"]) + state_mount.mount_path = Path("/vercel/sandbox/moved") + with pytest.raises(MountConfigError, match="cannot change after sandbox creation"): + await session._inner.persist_workspace() + assert not any(call[0] == "/usr/bin/findmnt" for call in sandbox.run_command_calls) - async def activate( - self, - strategy: InContainerMountStrategy, - session: BaseSandboxSession, - dest: Path, - base_dir: Path, - ) -> list[MaterializedFile]: - _ = (strategy, session, dest, base_dir) - return [] + state_mount.mount_path = None + state_mount.ephemeral = False + with pytest.raises(MountConfigError, match="cannot change after sandbox creation"): + await session._inner.persist_workspace() + assert not any(call[0] == "/usr/bin/findmnt" for call in sandbox.run_command_calls) - async def deactivate( - self, - strategy: InContainerMountStrategy, - session: BaseSandboxSession, - dest: Path, - base_dir: Path, - ) -> None: - _ = (strategy, session, dest, base_dir) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.shutdown() + unmount_call = next(call for call in sandbox.run_command_calls if call[0] == "/usr/bin/umount") + assert unmount_call[1] == ["/vercel/sandbox/remote"] - async def teardown_for_snapshot( - self, - strategy: InContainerMountStrategy, - session: BaseSandboxSession, - path: Path, - ) -> None: - _ = strategy - mount._events.append(("unmount", path.as_posix())) - sandbox = cast(Any, session)._sandbox - if sandbox is not None: - sandbox.files.pop(f"{path.as_posix()}/mounted.txt", None) - async def restore_after_snapshot( - self, - strategy: InContainerMountStrategy, - session: BaseSandboxSession, - path: Path, - ) -> None: - _ = strategy - mount._events.append(("mount", path.as_posix())) - sandbox = cast(Any, session)._sandbox - if sandbox is not None: - sandbox.files[f"{path.as_posix()}/mounted.txt"] = b"mounted-content" +@pytest.mark.asyncio +async def test_vercel_s3_rejects_logical_path_and_root_changes_with_explicit_mount_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + manifest = _vercel_s3_manifest( + package_module, + mount_path=Path("/vercel/sandbox/actual"), + ) + manifest.root = vercel_module.DEFAULT_VERCEL_WORKSPACE_ROOT + session = await client.create( + manifest=manifest, + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + await session.start() - return _Adapter(self) + mount = session.state.manifest.entries.pop("remote") + session.state.manifest.entries["durable"] = mount + with pytest.raises(MountConfigError, match="cannot change after sandbox creation"): + await session._inner.persist_workspace() + assert not any(call[0] == "/usr/bin/findmnt" for call in sandbox.run_command_calls) + session.state.manifest.entries["remote"] = session.state.manifest.entries.pop("durable") + session.state.manifest.root = "/vercel/sandbox/link/.." + with pytest.raises(MountConfigError, match="cannot change after sandbox creation"): + await session.exec("true", shell=False) + assert not any(call[0] == "/usr/bin/findmnt" for call in sandbox.run_command_calls) -def _load_vercel_module(monkeypatch: pytest.MonkeyPatch) -> Any: - _FakeAsyncSandbox.reset() + session.state.manifest.root = vercel_module.DEFAULT_VERCEL_WORKSPACE_ROOT + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await session.shutdown() - fake_vercel = types.ModuleType("vercel") - fake_vercel_sandbox = cast(Any, types.ModuleType("vercel.sandbox")) - fake_vercel_sandbox.AsyncSandbox = _FakeAsyncSandbox - fake_vercel_sandbox.NetworkPolicy = NetworkPolicy - fake_vercel_sandbox.NetworkPolicyCustom = NetworkPolicyCustom - fake_vercel_sandbox.NetworkPolicyRule = NetworkPolicyRule - fake_vercel_sandbox.NetworkPolicySubnets = NetworkPolicySubnets - fake_vercel_sandbox.Resources = Resources - fake_vercel_sandbox.SandboxAuthError = _FakeVercelSandboxAuthError - fake_vercel_sandbox.SandboxNotFoundError = _FakeVercelSandboxNotFoundError - fake_vercel_sandbox.SandboxPermissionError = _FakeVercelSandboxPermissionError - fake_vercel_sandbox.SandboxRateLimitError = _FakeVercelSandboxRateLimitError - fake_vercel_sandbox.SandboxServerError = _FakeVercelSandboxServerError - fake_vercel_sandbox.SandboxStatus = types.SimpleNamespace(RUNNING="running") - fake_vercel_sandbox.SandboxValidationError = _FakeVercelSandboxValidationError - fake_vercel_sandbox.SnapshotSource = SnapshotSource - cast(Any, fake_vercel).sandbox = fake_vercel_sandbox - monkeypatch.setitem(sys.modules, "vercel", fake_vercel) - monkeypatch.setitem(sys.modules, "vercel.sandbox", fake_vercel_sandbox) - sys.modules.pop("agents.extensions.sandbox.vercel.sandbox", None) - sys.modules.pop("agents.extensions.sandbox.vercel", None) +@pytest.mark.asyncio +async def test_vercel_s3_mount_transition_serializes_workspace_commands( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + snapshot=_MemorySnapshot(id="snapshot"), + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox, count=2) + sandbox.command_results.update( + { + "/usr/bin/findmnt": [ + _FakeCommandFinished(stdout="mountpoint-s3"), + _FakeCommandFinished(stdout="mountpoint-s3"), + ], + "/usr/bin/umount": [_FakeCommandFinished(), _FakeCommandFinished()], + } + ) + unmount_started = asyncio.Event() + release_unmount = asyncio.Event() + sandbox.command_started["/usr/bin/umount"] = unmount_started + sandbox.command_waiters["/usr/bin/umount"] = release_unmount + await session.start() - return importlib.import_module("agents.extensions.sandbox.vercel.sandbox") + stop_task = asyncio.create_task(session.stop()) + await asyncio.wait_for(unmount_started.wait(), timeout=1) + exec_task = asyncio.create_task(session.exec("true", shell=False)) + await asyncio.sleep(0) + assert not exec_task.done() + release_unmount.set() + await stop_task + assert (await exec_task).ok() + await session.shutdown() -async def _noop_sleep(*_args: object, **_kwargs: object) -> None: - return None +@pytest.mark.asyncio +async def test_vercel_without_s3_mounts_does_not_serialize_workspace_commands( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + state = vercel_module.VercelSandboxSessionState( + session_id="00000000-0000-0000-0000-000000000200", + manifest=Manifest(), + snapshot=NoopSnapshot(id="snapshot"), + sandbox_id="sandbox-without-mounts", + ) + sandbox = _FakeAsyncSandbox(sandbox_id="sandbox-without-mounts") + slow_started = asyncio.Event() + release_slow = asyncio.Event() + sandbox.command_started["slow"] = slow_started + sandbox.command_waiters["slow"] = release_slow + session = vercel_module.VercelSandboxSession.from_state(state, sandbox=sandbox) -def test_vercel_package_re_exports_backend_symbols(monkeypatch: pytest.MonkeyPatch) -> None: + slow_task = asyncio.create_task(session.exec("slow", shell=False)) + await asyncio.wait_for(slow_started.wait(), timeout=1) + try: + fast_result = await asyncio.wait_for(session.exec("true", shell=False), timeout=1) + assert fast_result.ok() + finally: + release_slow.set() + await slow_task + + +@pytest.mark.asyncio +async def test_vercel_mount_command_timeout_includes_output_collection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + mounts_module = importlib.import_module("agents.extensions.sandbox.vercel.mounts") + state = vercel_module.VercelSandboxSessionState( + session_id="00000000-0000-0000-0000-000000000201", + manifest=Manifest(), + snapshot=NoopSnapshot(id="snapshot"), + sandbox_id="sandbox-output-timeout", + ) + sandbox = _FakeAsyncSandbox(sandbox_id="sandbox-output-timeout") + hold_output = asyncio.Event() + + class _BlockingOutputResult(_FakeCommandFinished): + async def stdout(self) -> str: + await hold_output.wait() + return "" + + sandbox.command_results["/usr/bin/test"] = [_BlockingOutputResult()] + session = vercel_module.VercelSandboxSession.from_state(state, sandbox=sandbox) + + with pytest.raises(MountCommandError) as exc_info: + await mounts_module._run_vercel_command( + session, + "/usr/bin/test", + [], + timeout=0.01, + ) + assert exc_info.value.context["stderr"] == "TimeoutError: " + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_upgrades_mountpoint_below_minimum( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + mounts_module = importlib.import_module("agents.extensions.sandbox.vercel.mounts") + state = vercel_module.VercelSandboxSessionState( + session_id="00000000-0000-0000-0000-000000000202", + manifest=Manifest(), + snapshot=NoopSnapshot(id="snapshot"), + sandbox_id="sandbox-old-mountpoint", + ) + sandbox = _FakeAsyncSandbox(sandbox_id="sandbox-old-mountpoint") + sandbox.command_results = { + "/usr/bin/rpm": [ + _FakeCommandFinished(stdout="1.20.0"), + _FakeCommandFinished(stdout="1.21.0"), + ], + "/usr/bin/test": [_FakeCommandFinished(), _FakeCommandFinished()], + "/usr/bin/dnf": [_FakeCommandFinished()], + } + session = vercel_module.VercelSandboxSession.from_state(state, sandbox=sandbox) + + await mounts_module._ensure_mountpoint(session) + + dnf_call = next(call for call in sandbox.run_command_calls if call[0] == "/usr/bin/dnf") + assert dnf_call[1][-2:] == ["fuse", "mount-s3"] + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_failure_redacts_full_activation_traceback( + monkeypatch: pytest.MonkeyPatch, +) -> None: vercel_module = _load_vercel_module(monkeypatch) package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=_vercel_s3_manifest(package_module, credentials=True), + options=vercel_module.VercelSandboxClientOptions(allow_s3_credential_exposure=True), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + sandbox.command_results = { + "/usr/bin/test": [_FakeCommandFinished()], + "/usr/bin/rpm": [_FakeCommandFinished(stdout="1.21.0")], + "/usr/bin/find": [_FakeCommandFinished()], + } + secrets = ("test-access-key", "test-secret-key", "test-session-token") + provider_error = _FakeVercelSandboxRateLimitError(f"provider rejected {secrets[1]}") + original_run_command = sandbox.run_command + + def assert_activation_traceback_is_redacted(error: BaseException) -> None: + traceback = error.__traceback__ + while traceback is not None: + frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix() + if "/src/agents/" in frame_path: + locals_repr = repr(traceback.tb_frame.f_locals) + for secret in secrets: + assert secret not in locals_repr + traceback = traceback.tb_next + + async def fail_command( + cmd: str, + args: list[str] | None = None, + *, + cwd: str | None = None, + env: dict[str, str] | None = None, + sudo: bool = False, + ) -> _FakeCommandFinished: + if cmd == "/usr/bin/mount-s3": + assert env == { + "AWS_ACCESS_KEY_ID": secrets[0], + "AWS_SECRET_ACCESS_KEY": secrets[1], + "AWS_SESSION_TOKEN": secrets[2], + "AWS_REGION": "us-west-2", + } + raise provider_error + return await original_run_command(cmd, args, cwd=cwd, env=env, sudo=sudo) - assert package_module.VercelSandboxClient is vercel_module.VercelSandboxClient - assert package_module.VercelSandboxSessionState is vercel_module.VercelSandboxSessionState + monkeypatch.setattr(sandbox, "run_command", fail_command) + + with pytest.raises(MountCommandError) as exc_info: + await session.start() + + assert exc_info.value.context["stderr"] == ( + "_FakeVercelSandboxRateLimitError: provider rejected REDACTED" + ) + assert exc_info.value.retryable is True + assert exc_info.value.__cause__ is None + assert exc_info.value.__context__ is None + assert provider_error.__traceback__ is None + assert provider_error.__cause__ is None + assert provider_error.__context__ is None + assert_activation_traceback_is_redacted(exc_info.value) + assert sandbox.stop_calls == 1 + + +@pytest.mark.asyncio +async def test_vercel_s3_mount_cancellation_redacts_full_activation_traceback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + session = await vercel_module.VercelSandboxClient().create( + manifest=_vercel_s3_manifest(package_module, credentials=True), + options=vercel_module.VercelSandboxClientOptions(allow_s3_credential_exposure=True), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + sandbox.command_results = { + "/usr/bin/test": [_FakeCommandFinished()], + "/usr/bin/rpm": [_FakeCommandFinished(stdout="1.21.0")], + "/usr/bin/find": [_FakeCommandFinished()], + } + mount_started = asyncio.Event() + sandbox.command_started["/usr/bin/mount-s3"] = mount_started + sandbox.command_waiters["/usr/bin/mount-s3"] = asyncio.Event() + secrets = ("test-access-key", "test-secret-key", "test-session-token") + + start_task = asyncio.create_task(session.start()) + await asyncio.wait_for(mount_started.wait(), timeout=1) + start_task.cancel() + with pytest.raises(asyncio.CancelledError) as exc_info: + await start_task + + traceback = exc_info.value.__traceback__ + while traceback is not None: + frame_path = Path(traceback.tb_frame.f_code.co_filename).as_posix() + if "/src/agents/" in frame_path: + locals_repr = repr(traceback.tb_frame.f_locals) + for secret in secrets: + assert secret not in locals_repr + traceback = traceback.tb_next + assert sandbox.stop_calls == 1 + + +@pytest.mark.asyncio +async def test_vercel_exec_timeout_includes_output_collection_and_releases_mount_lock( + monkeypatch: pytest.MonkeyPatch, +) -> None: + vercel_module = _load_vercel_module(monkeypatch) + package_module = importlib.import_module("agents.extensions.sandbox.vercel") + client = vercel_module.VercelSandboxClient() + session = await client.create( + manifest=_vercel_s3_manifest(package_module), + options=vercel_module.VercelSandboxClientOptions(), + ) + sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox) + _queue_successful_s3_mounts(sandbox) + await session.start() + hold_output = asyncio.Event() + + class _BlockingOutputResult(_FakeCommandFinished): + async def stdout(self) -> str: + await hold_output.wait() + return "" + + sandbox.command_results["slow"] = [_BlockingOutputResult()] + + with pytest.raises(vercel_module.ExecTimeoutError): + await session.exec("slow", timeout=0.01, shell=False) + + sandbox.command_results.update( + { + "/usr/bin/findmnt": [_FakeCommandFinished(stdout="mountpoint-s3")], + "/usr/bin/umount": [_FakeCommandFinished()], + } + ) + await asyncio.wait_for(session.shutdown(), timeout=1) def test_vercel_supports_pty_is_disabled_until_provider_methods_exist( @@ -1051,7 +2604,7 @@ async def test_vercel_resume_recreates_sandbox_after_wait_timeout( vercel_module = _load_vercel_module(monkeypatch) # Use "pending" so that the code enters the wait path (not already RUNNING). existing = _FakeAsyncSandbox(sandbox_id="sandbox-existing", status="pending") - existing.wait_for_status_error = TimeoutError() + existing.wait_for_status_error = asyncio.TimeoutError() _FakeAsyncSandbox.sandboxes[existing.sandbox_id] = existing state = vercel_module.VercelSandboxSessionState( @@ -1428,7 +2981,7 @@ async def test_vercel_snapshot_mode_resume_uses_native_snapshot_reference( @pytest.mark.asyncio -async def test_vercel_tar_persistence_tears_down_ephemeral_mounts( +async def test_vercel_tar_persistence_treats_mount_exclusions_as_literal_paths( monkeypatch: pytest.MonkeyPatch, ) -> None: vercel_module = _load_vercel_module(monkeypatch) @@ -1440,12 +2993,13 @@ async def test_vercel_tar_persistence_tears_down_ephemeral_mounts( sandbox_id="sandbox-mount-tar", files={ "/workspace/kept.txt": b"kept", - "/workspace/remote/mounted.txt": b"mounted-content", + "/workspace/cache[1]/mounted.txt": b"mounted-content", + "/workspace/cache1/durable.txt": b"durable-content", }, ) state = vercel_module.VercelSandboxSessionState( session_id="00000000-0000-0000-0000-000000000008", - manifest=Manifest(root="/workspace", entries={"remote": mount}), + manifest=Manifest(root="/workspace", entries={"cache[1]": mount}), snapshot=snapshot, sandbox_id=sandbox.sandbox_id, workspace_persistence="tar", @@ -1460,21 +3014,25 @@ async def test_vercel_tar_persistence_tears_down_ephemeral_mounts( call for call in sandbox.run_command_calls if call[0] == "tar" and call[1][0] == "cf" ] - assert mount._events == [("unmount", "/workspace/remote"), ("mount", "/workspace/remote")] + assert mount._events == [ + ("unmount", "/workspace/cache[1]"), + ("mount", "/workspace/cache[1]"), + ] assert tar_calls == [ ( "tar", [ "cf", "/tmp/openai-agents-00000000000000000000000000000008.tar", - "--exclude=./remote", + "--no-wildcards", + "--exclude=./cache[1]", ".", ], "/workspace", ) ] - assert archived_names == ["kept.txt"] - assert sandbox.files["/workspace/remote/mounted.txt"] == b"mounted-content" + assert archived_names == ["cache1/durable.txt", "kept.txt"] + assert sandbox.files["/workspace/cache[1]/mounted.txt"] == b"mounted-content" @pytest.mark.asyncio diff --git a/tests/mcp/test_mcp_server_manager.py b/tests/mcp/test_mcp_server_manager.py index becb45eaf2..4e9d38d1ba 100644 --- a/tests/mcp/test_mcp_server_manager.py +++ b/tests/mcp/test_mcp_server_manager.py @@ -1,10 +1,13 @@ import asyncio +import logging from typing import Any, cast import pytest from mcp.types import CallToolResult, GetPromptResult, ListPromptsResult, Tool as MCPTool +from agents import _debug from agents.mcp import MCPServer, MCPServerManager +from agents.mcp._logging import get_mcp_server_log_name from agents.run_context import RunContextWrapper @@ -91,6 +94,21 @@ async def get_prompt( raise NotImplementedError +class SensitiveNamedServer(FlakyServer): + def __init__(self, name: str) -> None: + super().__init__(failures=1) + self._name = name + self.name_reads = 0 + + @property + def name(self) -> str: + self.name_reads += 1 + return self._name + + async def connect(self) -> None: + raise RuntimeError("SECRET_MCP_CONNECT_ERROR") + + class CleanupAwareServer(MCPServer): def __init__(self) -> None: super().__init__() @@ -131,16 +149,102 @@ async def get_prompt( raise NotImplementedError +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("ordinary-server", "ordinary-server"), + ( + "sse: https://user:password@example.test/events?token=secret#fragment", + "sse: https://example.test/events", + ), + ( + "streamable_http: https://example.test/mcp?token=secret", + "streamable_http: https://example.test/mcp", + ), + ( + "streamable_http: https://user:password@example.test:8443/mcp?token=secret", + "streamable_http: https://example.test:8443/mcp", + ), + ("streamable_http: https://[::1]:8000/mcp", "streamable_http: https://[::1]:8000/mcp"), + ( + "streamable-http: https://example.test/mcp#secret", + "streamable-http: https://example.test/mcp", + ), + ( + "streamable_http: https://user:password@[invalid/mcp?token=secret", + "streamable_http: ", + ), + ( + "streamable_http: https://user:password/mcp?token=secret", + "streamable_http: ", + ), + ("https://user:password@example.test/mcp?token=secret", "https://example.test/mcp"), + ("https://user:password@[invalid/mcp?token=secret", ""), + ("stdio: python server.py?token=secret", "stdio: python server.py?token=secret"), + ], +) +def test_get_mcp_server_log_name(name: str, expected: str) -> None: + assert get_mcp_server_log_name(name) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +@pytest.mark.parametrize( + ("server_name", "diagnostic_sentinel", "always_hidden"), + [ + ( + "streamable_http: https://SECRET_CREDENTIAL@example.test/" + "SECRET_MCP_PATH?token=SECRET_MCP_QUERY#SECRET_MCP_FRAGMENT", + "SECRET_MCP_PATH", + ("SECRET_CREDENTIAL", "SECRET_MCP_QUERY", "SECRET_MCP_FRAGMENT"), + ), + ( + "SECRET_CUSTOM_MCP_SERVER_NAME", + "SECRET_CUSTOM_MCP_SERVER_NAME", + (), + ), + ], +) +async def test_manager_sanitizes_url_derived_server_names_in_failure_logs( + monkeypatch, + caplog, + redacted: bool, + server_name: str, + diagnostic_sentinel: str, + always_hidden: tuple[str, ...], +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + server = SensitiveNamedServer(server_name) + manager = MCPServerManager([server]) + + with caplog.at_level(logging.ERROR, logger="openai.agents"): + await manager.connect_all() + + assert (diagnostic_sentinel not in caplog.text) is redacted + assert server.name_reads == (0 if redacted else 1) + for sentinel in always_hidden: + assert sentinel not in caplog.text + assert ("SECRET_MCP_CONNECT_ERROR" not in caplog.text) is redacted + + class CancelledServer(MCPServer): + def __init__(self) -> None: + super().__init__() + self.resource_open = False + self.cleanup_calls = 0 + @property def name(self) -> str: return "cancelled" async def connect(self) -> None: + # Simulate a transport that opened resources before cancellation. + self.resource_open = True raise asyncio.CancelledError() async def cleanup(self) -> None: - return None + self.cleanup_calls += 1 + self.resource_open = False async def list_tools( self, run_context: RunContextWrapper[Any] | None = None, agent: Any | None = None @@ -496,5 +600,28 @@ async def test_manager_cleanup_runs_on_cancelled_error_during_connect() -> None: with pytest.raises(asyncio.CancelledError): await manager.connect_all() assert server.cleanup_calls == 1 + # The cancelled server must be recorded and cleaned by connect_all()'s + # failure path — callers cannot rely on a later cleanup_all() because + # `async with` never reaches __aexit__ when __aenter__ raises. + assert cancelled_server in manager.failed_servers + assert cancelled_server.cleanup_calls == 1 + assert cancelled_server.resource_open is False finally: await manager.cleanup_all() + + +@pytest.mark.asyncio +async def test_manager_async_with_cleans_cancelled_server_when_unsuppressed() -> None: + server = CleanupAwareServer() + cancelled_server = CancelledServer() + + with pytest.raises(asyncio.CancelledError): + async with MCPServerManager( + [server, cancelled_server], + suppress_cancelled_error=False, + ): + raise AssertionError("context body should not run when connect raises") + + assert server.cleanup_calls == 1 + assert cancelled_server.cleanup_calls == 1 + assert cancelled_server.resource_open is False diff --git a/tests/mcp/test_mcp_util.py b/tests/mcp/test_mcp_util.py index 6faf014f18..def44ffca9 100644 --- a/tests/mcp/test_mcp_util.py +++ b/tests/mcp/test_mcp_util.py @@ -6,7 +6,8 @@ import pytest from inline_snapshot import snapshot -from mcp.types import CallToolResult, ImageContent, TextContent, Tool as MCPTool +from mcp.shared.exceptions import McpError +from mcp.types import CallToolResult, ErrorData, ImageContent, TextContent, Tool as MCPTool from pydantic import BaseModel, TypeAdapter import agents._debug as _debug @@ -675,7 +676,7 @@ async def test_mcp_invoke_bad_json_errors(caplog: pytest.LogCaptureFixture): with pytest.raises(ModelBehaviorError): await MCPUtil.invoke_mcp_tool(server, tool, ctx, "not_json") - assert "Invalid JSON input for tool test_tool_1" in caplog.text + assert "Invalid JSON input for MCP tool" in caplog.text @pytest.mark.asyncio @@ -765,7 +766,113 @@ async def test_mcp_invocation_crash_causes_error(caplog: pytest.LogCaptureFixtur with pytest.raises(AgentsException): await MCPUtil.invoke_mcp_tool(server, tool, ctx, "") - assert "Error invoking MCP tool test_tool_1" in caplog.text + assert "Error invoking MCP tool" in caplog.text + + +class SecretCrashingFakeMCPServer(FakeMCPServer): + async def call_tool( + self, + tool_name: str, + arguments: dict[str, Any] | None, + meta: dict[str, Any] | None = None, + ): + raise Exception("crash with SECRET_CRASH_123") + + +class McpErrorFakeMCPServer(FakeMCPServer): + async def call_tool( + self, + tool_name: str, + arguments: dict[str, Any] | None, + meta: dict[str, Any] | None = None, + ): + raise McpError(ErrorData(code=-32000, message="upstream said SECRET_MCP_123")) + + +@pytest.mark.asyncio +async def test_mcp_invocation_crash_redacts_error_when_dont_log_tool_data( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + caplog.set_level(logging.DEBUG) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + + server = SecretCrashingFakeMCPServer(server_name="SECRET_CUSTOM_MCP_SERVER") + server.add_tool("SECRET_MCP_TOOL_NAME", {}) + ctx = RunContextWrapper(context=None) + tool = MCPTool(name="SECRET_MCP_TOOL_NAME", inputSchema={}) + + with pytest.raises(AgentsException): + await MCPUtil.invoke_mcp_tool(server, tool, ctx, "") + + assert "Error invoking MCP tool" in caplog.text + assert "SECRET_CUSTOM_MCP_SERVER" not in caplog.text + assert "SECRET_MCP_TOOL_NAME" not in caplog.text + assert "SECRET_CRASH_123" not in caplog.text + + +@pytest.mark.asyncio +async def test_mcp_invocation_crash_includes_error_when_tool_logging_enabled( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + caplog.set_level(logging.DEBUG) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + + server = SecretCrashingFakeMCPServer( + server_name=( + "streamable_http: https://SECRET_CREDENTIAL@example.test/" + "SECRET_MCP_PATH?token=SECRET_MCP_QUERY" + ) + ) + server.add_tool("test_tool_1", {}) + ctx = RunContextWrapper(context=None) + tool = MCPTool(name="test_tool_1", inputSchema={}) + + with pytest.raises(AgentsException): + await MCPUtil.invoke_mcp_tool(server, tool, ctx, "") + + assert "SECRET_CRASH_123" in caplog.text + assert "SECRET_MCP_PATH" in caplog.text + assert "SECRET_CREDENTIAL" not in caplog.text + assert "SECRET_MCP_QUERY" not in caplog.text + + +@pytest.mark.asyncio +async def test_mcp_tool_returned_error_redacts_message_when_dont_log_tool_data( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + caplog.set_level(logging.DEBUG) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + + server = McpErrorFakeMCPServer(server_name="SECRET_CUSTOM_MCP_SERVER") + server.add_tool("SECRET_MCP_TOOL_NAME", {}) + ctx = RunContextWrapper(context=None) + tool = MCPTool(name="SECRET_MCP_TOOL_NAME", inputSchema={}) + + with pytest.raises(McpError): + await MCPUtil.invoke_mcp_tool(server, tool, ctx, "") + + assert "MCP tool returned an error" in caplog.text + assert "SECRET_CUSTOM_MCP_SERVER" not in caplog.text + assert "SECRET_MCP_TOOL_NAME" not in caplog.text + assert "SECRET_MCP_123" not in caplog.text + + +@pytest.mark.asyncio +async def test_mcp_tool_returned_error_includes_message_when_tool_logging_enabled( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + caplog.set_level(logging.DEBUG) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + + server = McpErrorFakeMCPServer() + server.add_tool("test_tool_1", {}) + ctx = RunContextWrapper(context=None) + tool = MCPTool(name="test_tool_1", inputSchema={}) + + with pytest.raises(McpError): + await MCPUtil.invoke_mcp_tool(server, tool, ctx, "") + + assert "SECRET_MCP_123" in caplog.text @pytest.mark.asyncio diff --git a/tests/mcp/test_tool_filtering.py b/tests/mcp/test_tool_filtering.py index 0127df806c..3da58756a6 100644 --- a/tests/mcp/test_tool_filtering.py +++ b/tests/mcp/test_tool_filtering.py @@ -9,6 +9,7 @@ import pytest from mcp import Tool as MCPTool +import agents._debug as _debug from agents import Agent from agents.mcp import ToolFilterContext, create_static_tool_filter from agents.run_context import RunContextWrapper @@ -181,6 +182,38 @@ def error_prone_filter(context: ToolFilterContext, tool: MCPTool) -> bool: assert {t.name for t in tools} == {"good_tool", "another_good_tool"} +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +async def test_dynamic_filter_error_logging_preserves_identity_only_in_diagnostic_mode( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + server = FakeMCPServer( + server_name=( + "streamable_http: https://SECRET_CREDENTIAL@example.test/" + "SECRET_SERVER_PATH?token=SECRET_QUERY" + ) + ) + server.add_tool("SECRET_TOOL_NAME", {}) + + def failing_filter(context: ToolFilterContext, tool: MCPTool) -> bool: + raise ValueError("SECRET_FILTER_ERROR") + + server.tool_filter = failing_filter + + with caplog.at_level("ERROR", logger="openai.agents"): + tools = await server.list_tools(create_test_context(), create_test_agent()) + + assert tools == [] + assert "SECRET_CREDENTIAL" not in caplog.text + assert "SECRET_QUERY" not in caplog.text + assert ("SECRET_TOOL_NAME" in caplog.text) is not redacted + assert ("SECRET_SERVER_PATH" in caplog.text) is not redacted + assert ("SECRET_FILTER_ERROR" in caplog.text) is not redacted + + # === Integration Tests === diff --git a/tests/memory/test_openai_conversations_session.py b/tests/memory/test_openai_conversations_session.py index b2b62950fe..2e241b88b8 100644 --- a/tests/memory/test_openai_conversations_session.py +++ b/tests/memory/test_openai_conversations_session.py @@ -553,3 +553,14 @@ def test_session_settings_constructor(self, mock_openai_client): assert session.session_settings is not None assert session.session_settings.limit == 5 + + def test_session_settings_constructor_normalizes_dictionary(self, mock_openai_client): + from agents.memory import SessionSettings + + session = OpenAIConversationsSession( + openai_client=mock_openai_client, + session_settings={"limit": 0}, + ) + + assert isinstance(session.session_settings, SessionSettings) + assert session.session_settings.limit == 0 diff --git a/tests/memory/test_openai_responses_compaction_session.py b/tests/memory/test_openai_responses_compaction_session.py index 30744d4ece..e0e0b2f96b 100644 --- a/tests/memory/test_openai_responses_compaction_session.py +++ b/tests/memory/test_openai_responses_compaction_session.py @@ -8,6 +8,7 @@ import pytest +import agents._debug as _debug from agents import Agent, Runner from agents.items import TResponseInputItem from agents.memory import ( @@ -705,9 +706,12 @@ async def clear_session(self) -> None: assert failing_session.add_calls == 1 @pytest.mark.asyncio + @pytest.mark.parametrize("redacted", [True, False]) async def test_run_compaction_reraises_replacement_error_when_restore_fails( - self, caplog: pytest.LogCaptureFixture + self, monkeypatch, caplog: pytest.LogCaptureFixture, redacted: bool ) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) history: list[TResponseInputItem] = [ cast(TResponseInputItem, {"type": "message", "role": "user", "content": "original"}), ] @@ -729,7 +733,7 @@ async def add_items(self, items: list[TResponseInputItem]) -> None: if self.add_calls == 1: await super().add_items(items[:1]) raise RuntimeError("replacement failed") - raise RuntimeError("restore failed") + raise RuntimeError("SECRET_COMPACTION_RESTORE_FAILURE") async def clear_session(self) -> None: self.clear_calls += 1 @@ -757,6 +761,7 @@ async def clear_session(self) -> None: assert ( "Failed to restore session history after compaction replacement failed." in caplog.text ) + assert ("SECRET_COMPACTION_RESTORE_FAILURE" not in caplog.text) is redacted assert failing_session.clear_calls == 2 assert failing_session.add_calls == 2 diff --git a/tests/memory/test_session.py b/tests/memory/test_session.py index f9cc324d2e..d727991f7d 100644 --- a/tests/memory/test_session.py +++ b/tests/memory/test_session.py @@ -7,7 +7,7 @@ import pytest -from agents import Agent, RunConfig, Runner, SQLiteSession, TResponseInputItem +from agents import Agent, RunConfig, Runner, SessionSettings, SQLiteSession, TResponseInputItem from tests.fake_model import FakeModel from tests.test_responses import get_text_message @@ -694,6 +694,22 @@ async def test_session_settings_constructor(): session.close() +@pytest.mark.asyncio +async def test_session_settings_constructor_normalizes_dictionary() -> None: + session = SQLiteSession("dictionary_settings_test", session_settings={"limit": 0}) + + assert isinstance(session.session_settings, SessionSettings) + assert session.session_settings.limit == 0 + assert session.session_settings.resolve({"limit": 4}).limit == 4 + + session.close() + + +def test_session_settings_rejects_unknown_dictionary_fields() -> None: + with pytest.raises(TypeError, match="Unknown session settings: limitt"): + SQLiteSession("invalid_settings_test", session_settings={"limitt": 1}) + + @pytest.mark.asyncio async def test_get_items_uses_session_settings_limit(): """Test that get_items uses session_settings.limit as default.""" diff --git a/tests/model_settings/test_serialization.py b/tests/model_settings/test_serialization.py index 4331814ba3..d20a624b5c 100644 --- a/tests/model_settings/test_serialization.py +++ b/tests/model_settings/test_serialization.py @@ -1,6 +1,7 @@ import json from dataclasses import fields +import pytest from openai.types.shared import Reasoning from pydantic import TypeAdapter from pydantic_core import to_json @@ -29,6 +30,54 @@ def test_basic_serialization() -> None: verify_serialization(model_settings) +def test_model_settings_direct_constructor_preserves_openai_reasoning_extensions() -> None: + settings = ModelSettings( + reasoning={"context": "all_turns", "future_reasoning_option": "enabled"} + ) + + assert isinstance(settings.reasoning, Reasoning) + assert settings.reasoning.context == "all_turns" + assert settings.reasoning.model_extra == {"future_reasoning_option": "enabled"} + + +def test_model_settings_dictionary_override_preserves_omitted_values() -> None: + settings = ModelSettings( + temperature=0.5, + reasoning=Reasoning.model_validate( + {"context": "all_turns", "future_reasoning_option": "enabled"} + ), + retry=ModelRetrySettings(max_retries=2), + ) + + resolved = settings.resolve({"temperature": 0.0}) + + assert resolved.temperature == 0.0 + assert resolved.reasoning is settings.reasoning + assert resolved.retry is settings.retry + + +def test_model_settings_dictionary_override_merges_retry_settings() -> None: + settings = ModelSettings( + retry=ModelRetrySettings( + max_retries=2, + backoff=ModelRetryBackoffSettings(initial_delay=0.1, jitter=True), + ) + ) + + resolved = settings.resolve({"retry": {"max_retries": 0, "backoff": {"jitter": False}}}) + + assert resolved.retry is not None + assert resolved.retry.max_retries == 0 + assert isinstance(resolved.retry.backoff, ModelRetryBackoffSettings) + assert resolved.retry.backoff.initial_delay == 0.1 + assert resolved.retry.backoff.jitter is False + + +def test_model_settings_dictionary_override_rejects_unknown_fields() -> None: + with pytest.raises(TypeError, match="Unknown model settings: temperatur"): + ModelSettings().resolve({"temperatur": 0.5}) + + def test_mcp_tool_choice_serialization() -> None: """Tests whether ModelSettings with MCPToolChoice can be serialized to a JSON string.""" # First, lets create a ModelSettings instance diff --git a/tests/models/test_agent_registration.py b/tests/models/test_agent_registration.py index 4741db8b64..c22f69319a 100644 --- a/tests/models/test_agent_registration.py +++ b/tests/models/test_agent_registration.py @@ -17,6 +17,7 @@ from agents.models.openai_provider import OpenAIProvider from agents.run_internal.agent_runner_helpers import resolve_trace_settings from agents.tracing import agent_span, trace +from agents.voice.models.openai_model_provider import OpenAIVoiceModelProvider def test_agent_registration_config_precedence(monkeypatch: pytest.MonkeyPatch) -> None: @@ -91,6 +92,35 @@ def test_agent_registration_provider_constructor_config() -> None: assert multi_provider.openai_provider.agent_registration.harness_id == "provider-harness" +def test_agent_registration_provider_constructors_normalize_dictionaries() -> None: + config = {"harness_id": "dictionary-harness"} + openai_provider = OpenAIProvider(agent_registration=config) + multi_provider = MultiProvider(openai_agent_registration=config) + voice_provider = OpenAIVoiceModelProvider(agent_registration=config) + + assert openai_provider.agent_registration is not None + assert openai_provider.agent_registration.harness_id == "dictionary-harness" + assert multi_provider.openai_provider.agent_registration is not None + assert multi_provider.openai_provider.agent_registration.harness_id == "dictionary-harness" + assert voice_provider.agent_registration is not None + assert voice_provider.agent_registration.harness_id == "dictionary-harness" + + +def test_default_agent_registration_normalizes_dictionary() -> None: + set_default_openai_agent_registration({"harness_id": "dictionary-default"}) + try: + resolved = resolve_openai_agent_registration_config(None) + assert resolved is not None + assert resolved.harness_id == "dictionary-default" + finally: + set_default_openai_agent_registration(None) + + +def test_agent_registration_rejects_unknown_dictionary_fields() -> None: + with pytest.raises(TypeError, match="Unknown OpenAI agent registration settings: harness_idd"): + OpenAIProvider(agent_registration={"harness_idd": "invalid"}) + + def test_harness_id_resolves_private_agent_registration() -> None: class Provider: _agent_registration = OpenAIAgentRegistrationConfig(harness_id="private-harness") diff --git a/tests/models/test_any_llm_model.py b/tests/models/test_any_llm_model.py index 06f57abd7c..c87477cd60 100644 --- a/tests/models/test_any_llm_model.py +++ b/tests/models/test_any_llm_model.py @@ -214,6 +214,18 @@ class GenericChatCompletionPayload(BaseModel): usage: Any +class GenericResponsesPayload(BaseModel): + id: str + created_at: float + model: str + object: str + output: list[Any] + parallel_tool_calls: bool + tool_choice: Any + tools: list[Any] + usage: Any + + async def _empty_chat_stream() -> AsyncIterator[ChatCompletionChunk]: if False: yield ChatCompletionChunk( @@ -260,6 +272,135 @@ async def test_user_agent_header_any_llm_chat(override_ua: str | None, monkeypat assert provider.chat_calls[0]["extra_headers"]["User-Agent"] == expected_ua +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("provider_name", ["gemini", "vertexai"]) +@pytest.mark.parametrize("options_type", ["unset", "dictionary", "model"]) +async def test_any_llm_google_chat_headers_use_http_options( + monkeypatch: pytest.MonkeyPatch, provider_name: str, options_type: str +) -> None: + class HttpOptions(BaseModel): + headers: dict[str, str] + timeout: int + + provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello")) + module, _create_calls = _import_any_llm_module(monkeypatch, provider) + model = module.AnyLLMModel(model=f"{provider_name}/gemini-2.5-flash") + + extra_args: dict[str, Any] = {} + configured_options: dict[str, Any] | HttpOptions | None = None + if options_type == "dictionary": + configured_options = {"headers": {"X-Existing": "existing"}, "timeout": 1000} + extra_args["http_options"] = configured_options + elif options_type == "model": + configured_options = HttpOptions(headers={"X-Existing": "existing"}, timeout=1000) + extra_args["http_options"] = configured_options + + await model.get_response( + system_instructions=None, + input="hi", + model_settings=ModelSettings( + extra_args=extra_args, + extra_headers={"X-Test-Header": "test"}, + ), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + + call = provider.chat_calls[0] + assert "extra_headers" not in call + http_options = call["http_options"] + if isinstance(http_options, BaseModel): + http_options = http_options.model_dump() + assert http_options["headers"]["User-Agent"] == f"Agents/Python {__version__}" + assert http_options["headers"]["X-Test-Header"] == "test" + if configured_options is not None: + assert http_options["headers"]["X-Existing"] == "existing" + assert http_options["timeout"] == 1000 + if isinstance(configured_options, BaseModel): + assert configured_options.headers == {"X-Existing": "existing"} + else: + assert configured_options["headers"] == {"X-Existing": "existing"} + + +@pytest.mark.parametrize("provider_name", ["gemini", "vertexai"]) +@pytest.mark.parametrize("content_type", ["model", "dictionary"]) +def test_any_llm_google_provider_normalizes_function_result_roles( + monkeypatch: pytest.MonkeyPatch, provider_name: str, content_type: str +) -> None: + class GoogleContent(BaseModel): + role: str + parts: list[dict[str, Any]] + + tool_result: dict[str, Any] = { + "role": "function", + "parts": [{"function_response": {"name": "get_weather", "response": {"result": "sunny"}}}], + } + original_tool_result: GoogleContent | dict[str, Any] + if content_type == "model": + original_tool_result = GoogleContent.model_validate(tool_result) + else: + original_tool_result = tool_result + + class GoogleProvider(FakeAnyLLMProvider): + @staticmethod + def _convert_completion_params(*args: Any, **kwargs: Any) -> dict[str, Any]: + return { + "model": "gemini-3.6-flash", + "contents": [ + GoogleContent(role="user", parts=[{"text": "Check the weather."}]), + original_tool_result, + GoogleContent(role="model", parts=[{"text": "Done."}]), + ], + } + + provider = GoogleProvider(supports_responses=False) + module, _create_calls = _import_any_llm_module(monkeypatch, provider) + model = module.AnyLLMModel(model=f"{provider_name}/gemini-3.6-flash") + + converted = model._get_provider()._convert_completion_params(object()) + contents = converted["contents"] + + assert [ + item.role if isinstance(item, GoogleContent) else item["role"] for item in contents + ] == [ + "user", + "user", + "model", + ] + normalized_tool_result = contents[1] + if isinstance(normalized_tool_result, BaseModel): + normalized_tool_result = normalized_tool_result.model_dump() + assert normalized_tool_result["parts"] == tool_result["parts"] + assert ( + original_tool_result.role + if isinstance(original_tool_result, GoogleContent) + else original_tool_result["role"] + ) == "function" + + +def test_any_llm_non_google_provider_does_not_normalize_function_result_roles( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class NonGoogleProvider(FakeAnyLLMProvider): + @staticmethod + def _convert_completion_params(*args: Any, **kwargs: Any) -> dict[str, Any]: + return {"contents": [{"role": "function", "parts": [{"result": "ok"}]}]} + + provider = NonGoogleProvider(supports_responses=False) + module, _create_calls = _import_any_llm_module(monkeypatch, provider) + model = module.AnyLLMModel(model="openrouter/google/gemini-3.6-flash") + + converted = model._get_provider()._convert_completion_params(object()) + + assert converted["contents"][0]["role"] == "function" + + @pytest.mark.allow_call_model_methods @pytest.mark.asyncio async def test_any_llm_chat_path_is_used_when_responses_are_unsupported(monkeypatch) -> None: @@ -410,6 +551,40 @@ async def test_any_llm_responses_path_is_used_when_supported(monkeypatch) -> Non assert response.output[0].content[0].text == "Hello" +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("payload_type", ["dict", "basemodel"]) +async def test_any_llm_responses_path_defaults_missing_cache_write_tokens( + monkeypatch: pytest.MonkeyPatch, payload_type: str +) -> None: + response_payload = _response("Hello").model_dump() + response_payload["usage"]["input_tokens_details"].pop("cache_write_tokens") + response: Any = response_payload + if payload_type == "basemodel": + response = GenericResponsesPayload.model_validate(response_payload) + + provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response) + module, _create_calls = _import_any_llm_module(monkeypatch, provider) + model = module.AnyLLMModel(model="openai/gpt-5.4-mini") + + normalized = await model.get_response( + system_instructions=None, + input="hi", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + + assert normalized.output[0].content[0].text == "Hello" + assert normalized.usage.input_tokens_details.cache_write_tokens == 0 + assert "cache_write_tokens" not in response_payload["usage"]["input_tokens_details"] + + @pytest.mark.allow_call_model_methods @pytest.mark.asyncio async def test_any_llm_can_force_chat_completions_when_responses_are_supported(monkeypatch) -> None: @@ -902,3 +1077,53 @@ def test_any_llm_split_does_not_duplicate_content_or_thinking(monkeypatch) -> No # Tool calls are still split one-per-message. assert assistants[0]["tool_calls"][0]["id"] == "call_1" assert assistants[1]["tool_calls"][0]["id"] == "call_2" + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_any_llm_chat_sets_logprobs_when_top_logprobs_set(monkeypatch) -> None: + provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello")) + module, _ = _import_any_llm_module(monkeypatch, provider) + AnyLLMModel = module.AnyLLMModel + + model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini", api_key="k") + await model.get_response( + system_instructions=None, + input="hi", + model_settings=ModelSettings(top_logprobs=2), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + + # The Chat Completions API rejects top_logprobs unless logprobs is True. + assert provider.chat_calls[0]["top_logprobs"] == 2 + assert provider.chat_calls[0]["logprobs"] is True + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_any_llm_chat_omits_logprobs_when_top_logprobs_unset(monkeypatch) -> None: + provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello")) + module, _ = _import_any_llm_module(monkeypatch, provider) + AnyLLMModel = module.AnyLLMModel + + model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini", api_key="k") + await model.get_response( + system_instructions=None, + input="hi", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + + assert "logprobs" not in provider.chat_calls[0] diff --git a/tests/models/test_kwargs_functionality.py b/tests/models/test_kwargs_functionality.py index 31c166ecc3..211c81847a 100644 --- a/tests/models/test_kwargs_functionality.py +++ b/tests/models/test_kwargs_functionality.py @@ -1,3 +1,6 @@ +from typing import Any + +import httpx import litellm import pytest from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -5,6 +8,7 @@ from openai.types.chat.chat_completion_message import ChatCompletionMessage from openai.types.completion_usage import CompletionUsage +from agents import Agent from agents.extensions.models.litellm_model import LitellmModel from agents.model_settings import ModelSettings from agents.models.interface import ModelTracing @@ -60,6 +64,44 @@ async def fake_acompletion(model, messages=None, **kwargs): assert captured["temperature"] == 0.5 +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["model-settings", "dictionary"]) +async def test_litellm_normalizes_dictionary_agent_model_settings( + monkeypatch, use_dictionary: bool +): + captured: dict[str, object] = {} + + async def fake_acompletion(model, messages=None, **kwargs): + captured.update(kwargs) + message = Message(role="assistant", content="test response") + return ModelResponse(choices=[Choices(index=0, message=message)], usage=Usage(0, 0, 0)) + + monkeypatch.setattr(litellm, "acompletion", fake_acompletion) + settings: dict[str, Any] = {"temperature": 0.0, "reasoning": {"effort": "low"}} + model = LitellmModel(model="test-model") + agent = Agent( + name="test", + model=model, + model_settings=settings if use_dictionary else ModelSettings(**settings), + ) + + await model.get_response( + system_instructions=None, + input="test input", + model_settings=agent.model_settings, + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + ) + + assert captured["temperature"] == 0.0 + assert captured["reasoning_effort"] == "low" + + @pytest.mark.allow_call_model_methods @pytest.mark.asyncio async def test_openai_chatcompletions_kwargs_forwarded(monkeypatch): diff --git a/tests/models/test_litellm_extra_body.py b/tests/models/test_litellm_extra_body.py index b7940c05df..948a8cf192 100644 --- a/tests/models/test_litellm_extra_body.py +++ b/tests/models/test_litellm_extra_body.py @@ -4,6 +4,7 @@ import pytest from litellm.types.utils import Choices, Message, ModelResponse, Usage +from agents import function_tool from agents.extensions.models.litellm_model import LitellmModel from agents.model_settings import ModelSettings from agents.models.interface import ModelTracing @@ -48,6 +49,43 @@ async def fake_acompletion(model, messages=None, **kwargs): assert "foo" not in captured +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("override", [None, False]) +async def test_function_tools_skip_litellm_proxy_mcp_discovery(monkeypatch, override): + captured: dict[str, object] = {} + + async def fake_acompletion(model, messages=None, **kwargs): + captured.update(kwargs) + msg = Message(role="assistant", content="ok") + choice = Choices(index=0, message=msg) + return ModelResponse(choices=[choice], usage=Usage(0, 0, 0)) + + @function_tool + def lookup() -> str: + """Return a deterministic result.""" + return "ok" + + monkeypatch.setattr(litellm, "acompletion", fake_acompletion) + settings = ModelSettings( + extra_args={"_skip_mcp_handler": override} if override is not None else None + ) + model = LitellmModel(model="test-model") + + await model.get_response( + system_instructions=None, + input=[], + model_settings=settings, + tools=[lookup], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + ) + + assert captured["_skip_mcp_handler"] is (True if override is None else override) + + @pytest.mark.allow_call_model_methods @pytest.mark.asyncio async def test_extra_body_reasoning_effort_is_promoted(monkeypatch): diff --git a/tests/models/test_litellm_logprobs.py b/tests/models/test_litellm_logprobs.py new file mode 100644 index 0000000000..00354ab57e --- /dev/null +++ b/tests/models/test_litellm_logprobs.py @@ -0,0 +1,118 @@ +import litellm +import pytest +from litellm.types.utils import ( + ChatCompletionTokenLogprob, + ChoiceLogprobs, + Choices, + Message, + ModelResponse, + TopLogprob, + Usage, +) +from openai.types.responses import ResponseOutputMessage, ResponseOutputText + +from agents.extensions.models.litellm_model import LitellmModel +from agents.model_settings import ModelSettings +from agents.models.interface import ModelTracing + + +async def _capture_litellm_kwargs(monkeypatch, settings: ModelSettings) -> dict[str, object]: + captured: dict[str, object] = {} + + async def fake_acompletion(model, messages=None, **kwargs): + captured.update(kwargs) + msg = Message(role="assistant", content="ok") + choice = Choices(index=0, message=msg) + return ModelResponse(choices=[choice], usage=Usage(0, 0, 0)) + + monkeypatch.setattr(litellm, "acompletion", fake_acompletion) + await LitellmModel(model="test-model").get_response( + system_instructions=None, + input=[], + model_settings=settings, + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + ) + return captured + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_top_logprobs_sets_logprobs_flag(monkeypatch): + captured = await _capture_litellm_kwargs(monkeypatch, ModelSettings(top_logprobs=2)) + # The Chat Completions API rejects top_logprobs unless logprobs is True. + assert captured["top_logprobs"] == 2 + assert captured["logprobs"] is True + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_omits_logprobs_when_top_logprobs_unset(monkeypatch): + captured = await _capture_litellm_kwargs(monkeypatch, ModelSettings()) + assert "logprobs" not in captured + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_top_logprobs_with_extra_args_logprobs_does_not_collide(monkeypatch): + # Setting both top_logprobs and extra_args["logprobs"] must defer to the caller's logprobs + # rather than adding a duplicate that collides. + captured = await _capture_litellm_kwargs( + monkeypatch, ModelSettings(top_logprobs=2, extra_args={"logprobs": True}) + ) + assert captured["top_logprobs"] == 2 + assert captured["logprobs"] is True + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_get_response_preserves_returned_logprobs_in_output(monkeypatch): + """Returned token logprobs must be attached to ResponseOutputText.logprobs.""" + + async def fake_acompletion(model, messages=None, **kwargs): + message = Message(role="assistant", content="Hello") + logprobs = ChoiceLogprobs( + content=[ + ChatCompletionTokenLogprob( + token="Hello", + logprob=-0.25, + bytes=[72, 101, 108, 108, 111], + top_logprobs=[ + TopLogprob(token="Hello", logprob=-0.25, bytes=[72, 101, 108, 108, 111]), + TopLogprob(token="Hi", logprob=-1.5, bytes=[72, 105]), + ], + ) + ] + ) + choice = Choices(index=0, message=message, logprobs=logprobs) + return ModelResponse(choices=[choice], usage=Usage(0, 0, 0)) + + monkeypatch.setattr(litellm, "acompletion", fake_acompletion) + response = await LitellmModel(model="test-model").get_response( + system_instructions=None, + input=[], + model_settings=ModelSettings(top_logprobs=2), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + ) + + texts = [ + content + for item in response.output + if isinstance(item, ResponseOutputMessage) + for content in item.content + if isinstance(content, ResponseOutputText) + ] + assert texts, "expected a ResponseOutputText in the output" + output_logprobs = texts[0].logprobs + assert output_logprobs is not None + assert len(output_logprobs) == 1 + assert output_logprobs[0].token == "Hello" + assert output_logprobs[0].logprob == -0.25 + assert [tlp.token for tlp in output_logprobs[0].top_logprobs] == ["Hello", "Hi"] diff --git a/tests/models/test_openai_chatcompletions.py b/tests/models/test_openai_chatcompletions.py index 40b7bf7303..5cef4874ad 100644 --- a/tests/models/test_openai_chatcompletions.py +++ b/tests/models/test_openai_chatcompletions.py @@ -69,7 +69,7 @@ def _minimal_chat_completion(content: str = "ok") -> ChatCompletion: async def _run_chat_completions_model_with_custom_base_url( - model_settings: ModelSettings | None = None, + model_settings: ModelSettings | dict[str, Any] | None = None, ) -> dict[str, Any]: class DummyCompletions: def __init__(self) -> None: @@ -786,6 +786,54 @@ def test_chat_completions_rejects_responses_only_reasoning_settings_in_strict_mo ) +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["model-settings", "dictionary"]) +async def test_chat_completions_requests_normalize_dictionary_agent_settings( + use_dictionary: bool, +) -> None: + settings: dict[str, Any] = { + "reasoning": {"effort": "high"}, + "prompt_cache_options": {"mode": "explicit", "ttl": "30m"}, + "prompt_cache_retention": "24h", + "verbosity": "low", + "store": False, + "temperature": 0.3, + "top_p": 1.0, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "max_tokens": 64, + "parallel_tool_calls": False, + "extra_headers": {"x-model-settings-parity": "preserved"}, + "extra_query": {"model_settings_parity": "verified"}, + "extra_body": {"prompt_cache_key": "extra-body-cache-key"}, + "retry": { + "max_retries": 0, + "backoff": {"initial_delay": 0.0, "jitter": False}, + }, + } + kwargs = await _run_chat_completions_model_with_custom_base_url( + model_settings=settings if use_dictionary else ModelSettings(**settings) + ) + + assert kwargs["reasoning_effort"] == "high" + assert kwargs["prompt_cache_options"] == settings["prompt_cache_options"] + assert kwargs["prompt_cache_retention"] == "24h" + assert kwargs["verbosity"] == "low" + assert kwargs["store"] is False + assert kwargs["temperature"] == 0.3 + assert kwargs["top_p"] == 1.0 + assert kwargs["frequency_penalty"] == 0.0 + assert kwargs["presence_penalty"] == 0.0 + assert kwargs["max_tokens"] == 64 + assert "max_output_tokens" not in kwargs + assert kwargs["parallel_tool_calls"] is False + assert kwargs["extra_headers"]["x-model-settings-parity"] == "preserved" + assert kwargs["extra_query"] == {"model_settings_parity": "verified"} + assert kwargs["extra_body"] == {"prompt_cache_key": "extra-body-cache-key"} + assert "retry" not in kwargs + + @pytest.mark.allow_call_model_methods @pytest.mark.asyncio async def test_custom_base_url_prompt_cache_key_uses_model_settings_only() -> None: diff --git a/tests/models/test_openai_chatcompletions_stream.py b/tests/models/test_openai_chatcompletions_stream.py index 75919a6a11..b435f12982 100644 --- a/tests/models/test_openai_chatcompletions_stream.py +++ b/tests/models/test_openai_chatcompletions_stream.py @@ -2,6 +2,7 @@ from collections.abc import AsyncIterator from typing import Any, cast +import httpx import pytest from openai.types.chat.chat_completion import ChatCompletion, Choice as ChatCompletionChoice from openai.types.chat.chat_completion_chunk import ( @@ -99,6 +100,95 @@ async def _collect_buffered_tool_call_chunks( ] +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["model-settings", "dictionary"]) +async def test_stream_response_forwards_dictionary_agent_model_settings( + use_dictionary: bool, +) -> None: + chunk = ChatCompletionChunk( + id="chunk-id", + created=1, + model="gpt-5.4-mini", + object="chat.completion.chunk", + choices=[ + Choice( + index=0, + delta=ChoiceDelta(role="assistant", content="ok"), + finish_reason="stop", + ) + ], + ) + + class DummyCompletions: + def __init__(self) -> None: + self.kwargs: dict[str, Any] = {} + + async def create(self, **kwargs: Any) -> AsyncIterator[ChatCompletionChunk]: + self.kwargs = kwargs + return _completion_stream(chunk) + + class DummyClient: + def __init__(self, completions: DummyCompletions) -> None: + self.chat = type("_Chat", (), {"completions": completions})() + self.base_url = httpx.URL("https://api.openai.com/v1/") + + completions = DummyCompletions() + model = OpenAIChatCompletionsModel( + model="gpt-5.4-mini", openai_client=cast(Any, DummyClient(completions)) + ) + settings: dict[str, Any] = { + "reasoning": {"effort": "low"}, + "prompt_cache_options": {"mode": "explicit", "ttl": "30m"}, + "prompt_cache_retention": "24h", + "verbosity": "low", + "store": False, + "temperature": 0.0, + "top_p": 1.0, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "max_tokens": 64, + "parallel_tool_calls": False, + "include_usage": False, + } + agent = Agent( + name="test", + model=model, + model_settings=settings if use_dictionary else ModelSettings(**settings), + ) + + events = [ + event + async for event in model.stream_response( + system_instructions=None, + input="hi", + model_settings=agent.model_settings, + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + ] + + assert any(event.type == "response.completed" for event in events) + assert completions.kwargs["reasoning_effort"] == "low" + assert completions.kwargs["prompt_cache_options"] == settings["prompt_cache_options"] + assert completions.kwargs["prompt_cache_retention"] == "24h" + assert completions.kwargs["verbosity"] == "low" + assert completions.kwargs["store"] is False + assert completions.kwargs["temperature"] == 0.0 + assert completions.kwargs["top_p"] == 1.0 + assert completions.kwargs["frequency_penalty"] == 0.0 + assert completions.kwargs["presence_penalty"] == 0.0 + assert completions.kwargs["max_tokens"] == 64 + assert completions.kwargs["parallel_tool_calls"] is False + assert completions.kwargs["stream"] is True + assert completions.kwargs["stream_options"] == {"include_usage": False} + + @pytest.mark.allow_call_model_methods @pytest.mark.asyncio async def test_stream_response_yields_events_for_text_content(monkeypatch) -> None: @@ -2746,3 +2836,257 @@ async def patched_fetch_response(self, *args, **kwargs): assert isinstance(completed_event.response.output[0], ResponseFunctionToolCall) assert isinstance(completed_event.response.output[1], ResponseFunctionToolCall) assert isinstance(completed_event.response.output[2], ResponseOutputMessage) + + +async def _buffered_stream_events(monkeypatch, chunks: list[ChatCompletionChunk]) -> list[Any]: + """Run the given chunks through the Chat Completions model with tool-call + buffering enabled, returning the streamed events.""" + + async def fake_stream() -> AsyncIterator[ChatCompletionChunk]: + for chunk in chunks: + yield chunk + + async def patched_fetch_response(self, *args, **kwargs): + return _empty_response(), fake_stream() + + monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response) + model = OpenAIProvider( + use_responses=False, + buffer_streamed_tool_calls=True, + ).get_model("gpt-4") + + return [ + event + async for event in model.stream_response( + system_instructions=None, + input="", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + ] + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_buffered_stream_synthesizes_refusal_on_content_filter(monkeypatch) -> None: + """With tool-call buffering enabled, a stream that terminates with + finish_reason == "content_filter" and no emitted content must still + synthesize a ResponseOutputRefusal. + + The buffering layer only forwarded choices whose delta carried output, so the + terminal empty-delta chunk was dropped before the handler could see the + finish_reason, turning a safety block into a silently empty turn. + """ + chunk1 = ChatCompletionChunk( + id="chunk-id", + created=1, + model="fake", + object="chat.completion.chunk", + choices=[Choice(index=0, delta=ChoiceDelta(role="assistant", content=""))], + ) + chunk2 = ChatCompletionChunk( + id="chunk-id", + created=1, + model="fake", + object="chat.completion.chunk", + choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")], + usage=CompletionUsage(completion_tokens=0, prompt_tokens=7, total_tokens=7), + ) + + output_events = await _buffered_stream_events(monkeypatch, [chunk1, chunk2]) + + types = [e.type for e in output_events] + assert "response.refusal.delta" in types + assert types[-1] == "response.completed" + + refusal_deltas = [e for e in output_events if e.type == "response.refusal.delta"] + assert refusal_deltas and refusal_deltas[0].delta + + # The assistant message is announced once and every opened part is closed. + assert types.count("response.output_item.added") == 1 + assert types.count("response.content_part.added") == types.count("response.content_part.done") + + # The empty "" content delta must not open a text content part. + assert "response.output_text.delta" not in types + added_parts = [e for e in output_events if e.type == "response.content_part.added"] + assert len(added_parts) == 1 + assert isinstance(added_parts[0].part, ResponseOutputRefusal) + + completed_event = output_events[-1] + assert isinstance(completed_event, ResponseCompletedEvent) + assistant_msg = completed_event.response.output[0] + assert isinstance(assistant_msg, ResponseOutputMessage) + assert len(assistant_msg.content) == 1 + refusal_part = assistant_msg.content[0] + assert isinstance(refusal_part, ResponseOutputRefusal) + assert refusal_part.refusal + + # Streamed content_index matches the refusal's position in the completed response. + assert added_parts[0].content_index == 0 + assert refusal_deltas[0].content_index == 0 + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_buffered_stream_content_filter_does_not_clobber_text(monkeypatch) -> None: + """A content_filter finish_reason arriving after real text was streamed must + not synthesize a refusal, even with buffering enabled.""" + chunk1 = ChatCompletionChunk( + id="chunk-id", + created=1, + model="fake", + object="chat.completion.chunk", + choices=[Choice(index=0, delta=ChoiceDelta(content="answer"))], + ) + chunk2 = ChatCompletionChunk( + id="chunk-id", + created=1, + model="fake", + object="chat.completion.chunk", + choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")], + usage=CompletionUsage(completion_tokens=1, prompt_tokens=7, total_tokens=8), + ) + + output_events = await _buffered_stream_events(monkeypatch, [chunk1, chunk2]) + + assert "response.refusal.delta" not in [e.type for e in output_events] + completed_event = output_events[-1] + assert isinstance(completed_event, ResponseCompletedEvent) + assistant_msg = completed_event.response.output[0] + assert isinstance(assistant_msg, ResponseOutputMessage) + text_part = assistant_msg.content[0] + assert isinstance(text_part, ResponseOutputText) + assert text_part.text == "answer" + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_buffered_stream_content_filter_refusal_after_reasoning(monkeypatch) -> None: + """A buffered content_filter turn preceded by reasoning still places the + synthesized refusal at content_index 0 of the assistant message, which is + output_index 1 (the reasoning item is a separate output item).""" + reasoning_delta = ChoiceDelta(role="assistant", content=None) + # reasoning_content is a provider extra field the handler reads via hasattr. + reasoning_delta.reasoning_content = "thinking..." # type: ignore[attr-defined] + chunk_reasoning = ChatCompletionChunk( + id="chunk-id", + created=1, + model="fake", + object="chat.completion.chunk", + choices=[Choice(index=0, delta=reasoning_delta)], + ) + chunk_empty = ChatCompletionChunk( + id="chunk-id", + created=1, + model="fake", + object="chat.completion.chunk", + choices=[Choice(index=0, delta=ChoiceDelta(content=""))], + ) + chunk_filter = ChatCompletionChunk( + id="chunk-id", + created=1, + model="fake", + object="chat.completion.chunk", + choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")], + usage=CompletionUsage(completion_tokens=0, prompt_tokens=7, total_tokens=7), + ) + + output_events = await _buffered_stream_events( + monkeypatch, [chunk_reasoning, chunk_empty, chunk_filter] + ) + + completed_event = output_events[-1] + assert isinstance(completed_event, ResponseCompletedEvent) + completed_resp = completed_event.response + assert isinstance(completed_resp.output[0], ResponseReasoningItem) + assistant_msg = completed_resp.output[1] + assert isinstance(assistant_msg, ResponseOutputMessage) + assert len(assistant_msg.content) == 1 + assert isinstance(assistant_msg.content[0], ResponseOutputRefusal) + + added = [ + e + for e in output_events + if e.type == "response.content_part.added" and isinstance(e.part, ResponseOutputRefusal) + ] + deltas = [e for e in output_events if e.type == "response.refusal.delta"] + assert len(added) == 1 + assert added[0].content_index == 0 + assert added[0].output_index == 1 + assert deltas and all(d.content_index == 0 and d.output_index == 1 for d in deltas) + assert "response.output_text.delta" not in [e.type for e in output_events] + + +def _chunk_with(choices: list[Choice], usage: CompletionUsage | None = None): + return ChatCompletionChunk( + id="chunk-id", + created=1, + model="fake", + object="chat.completion.chunk", + choices=choices, + usage=usage, + ) + + +@pytest.mark.asyncio +async def test_buffer_tool_call_stream_forwards_content_filter_finish_reason() -> None: + """The buffering layer must forward a content-filtered terminal choice even + though its delta is empty, so the finish_reason reaches the handler instead + of being swallowed. The delta is stripped, preserving buffering semantics.""" + chunks = [ + _chunk_with([Choice(index=0, delta=ChoiceDelta(content=""))]), + _chunk_with([Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")]), + ] + + async def source() -> AsyncIterator[ChatCompletionChunk]: + for chunk in chunks: + yield chunk + + buffered = [c async for c in ChatCmplStreamHandler.buffer_tool_call_stream(source())] + + terminal_choices = [ + choice + for chunk in buffered + for choice in chunk.choices + if choice.finish_reason == "content_filter" + ] + assert len(terminal_choices) == 1 + # The forwarded copy carries no delta output. + assert not ChatCmplStreamHandler._delta_has_passthrough_output(terminal_choices[0].delta) + + +@pytest.mark.asyncio +async def test_buffer_tool_call_stream_does_not_duplicate_tool_calls_finish() -> None: + """finish_reason == "tool_calls" is still emitted only by the synthesized + buffered chunk, so the terminal choice is not forwarded twice.""" + tool_call_delta = ChoiceDeltaToolCall( + index=0, + id="tool-id", + function=ChoiceDeltaToolCallFunction(name="my_func", arguments='{"a": 1}'), + type="function", + ) + chunks = [ + _chunk_with([Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta]))]), + _chunk_with([Choice(index=0, delta=ChoiceDelta(), finish_reason="tool_calls")]), + ] + + async def source() -> AsyncIterator[ChatCompletionChunk]: + for chunk in chunks: + yield chunk + + buffered = [c async for c in ChatCmplStreamHandler.buffer_tool_call_stream(source())] + + finish_choices = [ + choice + for chunk in buffered + for choice in chunk.choices + if choice.finish_reason == "tool_calls" + ] + assert len(finish_choices) == 1 + assert finish_choices[0].delta.tool_calls diff --git a/tests/models/test_openai_responses.py b/tests/models/test_openai_responses.py index 3c2edfb91b..7eb691f7ca 100644 --- a/tests/models/test_openai_responses.py +++ b/tests/models/test_openai_responses.py @@ -44,7 +44,7 @@ async def _run_responses_model_with_custom_base_url( - model_settings: ModelSettings | None = None, + model_settings: ModelSettings | dict[str, Any] | None = None, ) -> dict[str, Any]: class DummyResponses: def __init__(self) -> None: @@ -925,6 +925,56 @@ def test_build_response_create_kwargs_includes_gpt_5_6_request_controls(): assert kwargs["previous_response_id"] == "resp-previous" +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["model-settings", "dictionary"]) +async def test_responses_requests_normalize_dictionary_agent_settings(use_dictionary: bool) -> None: + settings: dict[str, Any] = { + "reasoning": {"effort": "low", "context": "all_turns"}, + "context_management": [{"type": "compaction", "compact_threshold": 200000}], + "prompt_cache_options": {"mode": "explicit", "ttl": "30m"}, + "prompt_cache_retention": "24h", + "store": False, + "metadata": {"request": "example"}, + "temperature": 0.0, + "top_p": 1.0, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "max_tokens": 64, + "parallel_tool_calls": False, + "extra_headers": {"x-model-settings-parity": "preserved"}, + "extra_query": {"model_settings_parity": "verified"}, + "extra_body": {"prompt_cache_key": "extra-body-cache-key"}, + "retry": { + "max_retries": 0, + "backoff": {"initial_delay": 0.0, "jitter": False}, + }, + } + kwargs = await _run_responses_model_with_custom_base_url( + model_settings=settings if use_dictionary else ModelSettings(**settings) + ) + + assert isinstance(kwargs["reasoning"], Reasoning) + assert kwargs["reasoning"].effort == "low" + assert kwargs["reasoning"].context == "all_turns" + assert kwargs["context_management"] == settings["context_management"] + assert kwargs["prompt_cache_options"] == settings["prompt_cache_options"] + assert kwargs["prompt_cache_retention"] == "24h" + assert kwargs["store"] is False + assert kwargs["metadata"] == {"request": "example"} + assert kwargs["temperature"] == 0.0 + assert kwargs["top_p"] == 1.0 + assert kwargs["max_output_tokens"] == 64 + assert "max_tokens" not in kwargs + assert kwargs["parallel_tool_calls"] is False + assert kwargs["extra_headers"]["x-model-settings-parity"] == "preserved" + assert kwargs["extra_query"] == {"model_settings_parity": "verified"} + assert kwargs["extra_body"] == {"prompt_cache_key": "extra-body-cache-key"} + assert "retry" not in kwargs + assert "frequency_penalty" not in kwargs + assert "presence_penalty" not in kwargs + + @pytest.mark.allow_call_model_methods def test_build_response_create_kwargs_rejects_duplicate_prompt_cache_options_extra_args(): client = DummyWSClient() @@ -1768,13 +1818,22 @@ async def fake_open( monkeypatch.setattr(model, "_open_websocket_connection", fake_open) + configured_agent = Agent( + name="configured", + model=model, + model_settings={ + "reasoning": {"mode": "pro", "effort": "max", "context": "all_turns"}, + "context_management": [{"type": "compaction", "compact_threshold": 200000}], + "prompt_cache_options": {"mode": "explicit", "ttl": "30m"}, + "prompt_cache_retention": "24h", + "store": False, + "metadata": {"request": "example"}, + }, + ) first = await model.get_response( system_instructions=None, input="hi", - model_settings=ModelSettings( - reasoning=Reasoning(mode="pro", effort="max", context="all_turns"), - prompt_cache_options={"mode": "explicit", "ttl": "30m"}, - ), + model_settings=configured_agent.model_settings, tools=[], output_schema=None, handoffs=[], @@ -1806,6 +1865,12 @@ async def fake_open( "mode": "explicit", "ttl": "30m", } + assert ws.sent_messages[0]["context_management"] == [ + {"type": "compaction", "compact_threshold": 200000} + ] + assert ws.sent_messages[0]["prompt_cache_retention"] == "24h" + assert ws.sent_messages[0]["store"] is False + assert ws.sent_messages[0]["metadata"] == {"request": "example"} assert ws.sent_messages[1]["type"] == "response.create" assert ws.sent_messages[1]["stream"] is True assert ws.sent_messages[1]["previous_response_id"] == "resp-1" diff --git a/tests/models/test_openai_responses_converter.py b/tests/models/test_openai_responses_converter.py index e1c8069ec9..cef2c8b81b 100644 --- a/tests/models/test_openai_responses_converter.py +++ b/tests/models/test_openai_responses_converter.py @@ -27,6 +27,7 @@ import pytest from openai import omit +from openai.types.responses.web_search_tool import Filters as WebSearchToolFilters from pydantic import BaseModel from agents import ( @@ -468,6 +469,32 @@ def test_convert_tools_includes_explicit_false_external_web_access() -> None: ] +@pytest.mark.parametrize("use_dictionary", [False, True], ids=["class", "dictionary"]) +def test_web_search_filters_preserve_existing_provider_payload(use_dictionary: bool) -> None: + filters = {"allowed_domains": ["example.com"]} + tool = WebSearchTool( + filters=filters if use_dictionary else WebSearchToolFilters.model_validate(filters) + ) + + assert isinstance(tool.filters, WebSearchToolFilters) + converted = Converter.convert_tools([tool], handoffs=[], model="gpt-5.4") + assert converted.tools == [ + { + "type": "web_search", + "filters": filters, + "user_location": None, + "search_context_size": "medium", + } + ] + + +def test_web_search_filters_preserve_openai_forward_compatible_fields() -> None: + tool = WebSearchTool(filters={"future_filter": ["example.com"]}) + + assert tool.filters is not None + assert tool.filters.model_extra == {"future_filter": ["example.com"]} + + def test_convert_tools_uses_preview_computer_payload_for_preview_model() -> None: comp_tool = ComputerTool(computer=DummyComputer()) diff --git a/tests/models/test_responses_websocket_session.py b/tests/models/test_responses_websocket_session.py index c1272da156..fc2d339756 100644 --- a/tests/models/test_responses_websocket_session.py +++ b/tests/models/test_responses_websocket_session.py @@ -2,7 +2,7 @@ import pytest -from agents import Agent, responses_websocket_session +from agents import Agent, ResponsesWebSocketSession, RunConfig, responses_websocket_session from agents.models.multi_provider import MultiProvider from agents.models.openai_provider import OpenAIProvider @@ -17,6 +17,35 @@ async def test_responses_websocket_session_builds_shared_run_config(): assert ws.run_config.model_provider.openai_provider is ws.provider +def test_responses_websocket_session_normalizes_dictionary_run_config() -> None: + provider = MultiProvider(openai_api_key="test") + + session = ResponsesWebSocketSession( + provider=provider.openai_provider, + run_config={ + "model_provider": provider, + "model_settings": {"temperature": 0.0, "retry": {"max_retries": 0}}, + }, + ) + + assert isinstance(session.run_config, RunConfig) + assert session.run_config.model_provider is provider + assert session.run_config.model_settings is not None + assert session.run_config.model_settings.temperature == 0.0 + assert session.run_config.model_settings.retry is not None + assert session.run_config.model_settings.retry.max_retries == 0 + + +def test_responses_websocket_session_rejects_unknown_dictionary_run_config_fields() -> None: + provider = MultiProvider(openai_api_key="test") + + with pytest.raises(TypeError, match="Unknown run_config settings: tracin_disabled"): + ResponsesWebSocketSession( + provider=provider.openai_provider, + run_config={"model_provider": provider, "tracin_disabled": True}, + ) + + @pytest.mark.asyncio async def test_responses_websocket_session_preserves_openai_prefix_routing(monkeypatch): captured: dict[str, object] = {} diff --git a/tests/realtime/test_agent.py b/tests/realtime/test_agent.py index 7ac5cbe359..bc2a4c408c 100644 --- a/tests/realtime/test_agent.py +++ b/tests/realtime/test_agent.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import Any +from unittest.mock import patch import pytest @@ -29,6 +30,29 @@ def _instructions(ctx, agt) -> str: assert instructions == "Dynamic" +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +async def test_mutated_invalid_instructions_respect_model_data_policy( + monkeypatch, redacted: bool +) -> None: + class SensitiveInstructions: + def __str__(self) -> str: + return "SECRET_REALTIME_INSTRUCTIONS" + + __repr__ = __str__ + + agent = RealtimeAgent(name="test") + agent.instructions = SensitiveInstructions() # type: ignore[assignment] + monkeypatch.setattr("agents.realtime.agent._debug.DONT_LOG_MODEL_DATA", redacted) + + with patch("agents.realtime.agent.logger") as mock_logger: + prompt = await agent.get_system_prompt(RunContextWrapper(context=None)) + + assert prompt is None + logged = str(mock_logger.error.call_args) + assert ("SECRET_REALTIME_INSTRUCTIONS" not in logged) is redacted + + def test_post_init_rejects_invalid_field_types() -> None: with pytest.raises(TypeError, match="RealtimeAgent name must be a string"): RealtimeAgent(name=1) # type: ignore[arg-type] diff --git a/tests/realtime/test_openai_realtime.py b/tests/realtime/test_openai_realtime.py index 31ec76dddc..aeeb58081b 100644 --- a/tests/realtime/test_openai_realtime.py +++ b/tests/realtime/test_openai_realtime.py @@ -477,6 +477,31 @@ async def test_handle_invalid_event_schema_redacts_payload_from_logs(self, model error_event = mock_listener.on_event.call_args_list[1][0][0] assert error_event.type == "error" + @pytest.mark.asyncio + async def test_send_raw_message_conversion_failure_redacts_payload_from_logs( + self, model, monkeypatch + ): + """A raw client message that fails to convert must not leak its payload to the logs + when model-data logging is disabled.""" + monkeypatch.setattr( + "agents.realtime.openai_realtime._debug.DONT_LOG_MODEL_DATA", + True, + ) + raw = RealtimeModelSendRawMessage( + message={ + "type": "invalid.event.type", + "other_data": {"transcript": "secret transcript"}, + } + ) + + with patch("agents.realtime.openai_realtime.logger") as mock_logger: + await model.send_event(raw) + + mock_logger.error.assert_called_once() + logged_call = str(mock_logger.error.call_args) + assert "secret transcript" not in logged_call + assert "invalid.event.type" in logged_call + @pytest.mark.asyncio async def test_custom_voice_response_events_update_response_sequencer(self, model, monkeypatch): """Dict-shaped custom voices should not block response.create sequencing.""" diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index 3211f2358d..8d69916505 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -1,6 +1,7 @@ import asyncio import dataclasses import json +import logging import threading from typing import Any, cast from unittest.mock import AsyncMock, Mock, PropertyMock, patch @@ -8,6 +9,7 @@ import pytest from pydantic import BaseModel, ConfigDict +import agents._debug as _debug from agents.exceptions import ToolTimeoutError, UserError from agents.guardrail import GuardrailFunctionOutput, OutputGuardrail from agents.handoffs import Handoff @@ -576,6 +578,7 @@ class _FakeAudio: @pytest.mark.asyncio async def test_item_updated_merge_exception_path_logs_error(monkeypatch): + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) model = _DummyModel() agent = RealtimeAgent(name="agent") session = RealtimeSession(model, agent, None) @@ -594,8 +597,7 @@ async def test_item_updated_merge_exception_path_logs_error(monkeypatch): with patch("agents.realtime.session.logger") as mock_logger: await session.on_event(RealtimeModelItemUpdatedEvent(item=incoming)) - # error branch should be hit - assert mock_logger.error.called + mock_logger.error.assert_called_once_with("%s", "Error merging transcripts", stacklevel=3) @pytest.mark.asyncio @@ -3119,6 +3121,31 @@ async def test_reject_pending_tool_call_uses_run_level_formatter( for ev in events ) + @pytest.mark.asyncio + async def test_rejection_formatter_error_is_redacted( + self, monkeypatch, mock_model, mock_agent, mock_function_tool + ): + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + + def fail_formatter(_args): + raise ValueError("SECRET_REALTIME_TOOL_FORMATTER") + + session = RealtimeSession( + mock_model, + mock_agent, + None, + run_config={"tool_error_formatter": fail_formatter}, + ) + + with patch("agents.realtime.session.logger") as mock_logger: + message = await session._resolve_approval_rejection_message( + tool=mock_function_tool, + call_id="call_reject_error", + ) + + assert message + mock_logger.error.assert_called_once_with("%s", "Tool error formatter failed", stacklevel=3) + @pytest.mark.asyncio async def test_reject_pending_tool_call_prefers_explicit_message( self, mock_model, mock_agent, mock_function_tool @@ -3491,6 +3518,96 @@ def guardrail_func(context, agent, output): return OutputGuardrail(guardrail_function=guardrail_func, name="safe_guardrail") + @pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], + ids=["model_redacted", "tool_redacted", "diagnostic"], + ) + @pytest.mark.asyncio + async def test_output_guardrail_failure_follows_both_data_policies( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + mock_model: RealtimeModel, + model_redacted: bool, + tool_redacted: bool, + ) -> None: + error = RuntimeError("SECRET_REALTIME_GUARDRAIL_ERROR") + + async def failing_guardrail(context, agent, output): + _ = context, agent, output + raise error + + guardrail = OutputGuardrail( + guardrail_function=failing_guardrail, + name="SECRET_REALTIME_GUARDRAIL_NAME", + ) + agent = RealtimeAgent(name="agent", output_guardrails=[guardrail]) + session = RealtimeSession(mock_model, agent, None) + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + + with caplog.at_level(logging.DEBUG, logger="openai.agents"): + triggered = await session._run_output_guardrails("model text", "response-id") + + assert triggered is False + records = [ + record + for record in caplog.records + if "Output guardrail raised an exception" in record.getMessage() + ] + assert len(records) == 1 + record = records[0] + redacted = model_redacted or tool_redacted + if redacted: + assert record.msg == "%s" + assert record.args == ("Output guardrail raised an exception; skipping it",) + assert record.exc_info is None + assert record.exc_text is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert error not in record.__dict__.values() + rendered = logging.Formatter().format(record) + assert "SECRET_REALTIME_GUARDRAIL_ERROR" not in rendered + assert "SECRET_REALTIME_GUARDRAIL_NAME" not in rendered + else: + context = record.__dict__["openai_agents_diagnostic_context"] + assert context == {"guardrail_name": "SECRET_REALTIME_GUARDRAIL_NAME"} + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "SECRET_REALTIME_GUARDRAIL_ERROR" in logging.Formatter().format(record) + + @pytest.mark.asyncio + async def test_output_guardrail_failure_tolerates_missing_callable_name( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + mock_model: RealtimeModel, + ) -> None: + class _FailingGuardrailCallable: + async def __call__(self, context, agent, output): + _ = context, agent, output + raise RuntimeError("SECRET_UNNAMED_GUARDRAIL_ERROR") + + guardrail = OutputGuardrail(guardrail_function=_FailingGuardrailCallable()) + agent = RealtimeAgent(name="agent", output_guardrails=[guardrail]) + session = RealtimeSession(mock_model, agent, None) + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + + with caplog.at_level(logging.WARNING, logger="openai.agents"): + triggered = await session._run_output_guardrails("model text", "response-id") + + assert triggered is False + records = [ + record + for record in caplog.records + if "Output guardrail raised an exception" in record.getMessage() + ] + assert len(records) == 1 + context = records[0].__dict__["openai_agents_diagnostic_context"] + assert context["guardrail_type"].endswith("._FailingGuardrailCallable") + assert records[0].exc_info is not None + @pytest.mark.asyncio async def test_transcript_delta_triggers_guardrail_at_threshold( self, mock_model, mock_agent, triggered_guardrail diff --git a/tests/sandbox/test_compatibility_guards.py b/tests/sandbox/test_compatibility_guards.py index 5a11e5bf77..cd59a8303e 100644 --- a/tests/sandbox/test_compatibility_guards.py +++ b/tests/sandbox/test_compatibility_guards.py @@ -327,6 +327,7 @@ def test_core_sandbox_public_export_surface_is_stable() -> None: ( "agents.extensions.sandbox.vercel", { + "VercelCloudBucketMountStrategy", "VercelSandboxClient", "VercelSandboxClientOptions", "VercelSandboxSession", @@ -508,6 +509,7 @@ def test_optional_sandbox_dataclass_constructor_field_order_is_stable( "workspace_persistence", "snapshot_expiration_ms", "network_policy", + "allow_s3_credential_exposure", ), ), ], @@ -743,6 +745,7 @@ def test_optional_sandbox_client_options_positional_field_order_is_stable( "workspace_persistence", "snapshot_expiration_ms", "network_policy", + "s3_mounts_non_resumable", ), ), ], @@ -959,6 +962,11 @@ def test_mount_strategy_type_strings_round_trip_through_registry( "RunloopCloudBucketMountStrategy", "runloop_cloud_bucket", ), + ( + "agents.extensions.sandbox.vercel", + "VercelCloudBucketMountStrategy", + "vercel_cloud_bucket", + ), ], ) def test_optional_mount_strategy_type_strings_round_trip_through_registry( diff --git a/tests/sandbox/test_memory.py b/tests/sandbox/test_memory.py index 5eb843de2d..fa6c3d4bca 100644 --- a/tests/sandbox/test_memory.py +++ b/tests/sandbox/test_memory.py @@ -2,20 +2,24 @@ import io import json +import logging +from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Any, cast +from typing import Any, cast, get_type_hints import pytest from openai.types.responses import ResponseCustomToolCall, ResponseFunctionToolCall from openai.types.responses.response_output_message import ResponseOutputMessage from openai.types.responses.response_reasoning_item import ResponseReasoningItem +import agents._debug as _debug import agents.sandbox.capabilities.memory as memory_module import agents.sandbox.memory.manager as memory_manager_module import agents.sandbox.memory.phase_one as phase_one_module from agents import ( Agent, + ModelSettings, ReasoningItem, RunConfig, Runner, @@ -30,7 +34,7 @@ ToolApprovalItem, TResponseOutputItem, ) -from agents.result import RunResultStreaming +from agents.result import RunResult, RunResultStreaming from agents.run import _sandbox_memory_input from agents.run_context import RunContextWrapper from agents.sandbox import ( @@ -70,6 +74,17 @@ from tests.utils.hitl import make_shell_call +@dataclass +class _DeclaredProviderModelSettings(ModelSettings): + provider_field: str | None = None + + +@dataclass +class _DeclaredProviderMemoryGenerateConfig(MemoryGenerateConfig): + phase_one_model_settings: _DeclaredProviderModelSettings | None = None + phase_two_model_settings: _DeclaredProviderModelSettings | None = None + + class _DeleteTrackingUnixLocalSandboxClient(UnixLocalSandboxClient): def __init__(self) -> None: super().__init__() @@ -696,6 +711,93 @@ def test_memory_generate_config_accepts_renamed_limit_field() -> None: assert config.max_raw_memories_for_consolidation == 123 +def test_memory_generate_config_normalizes_dictionary_model_settings() -> None: + config = MemoryGenerateConfig( + phase_one_model_settings={ + "reasoning": {"effort": "low"}, + "retry": {"max_retries": 0}, + }, + phase_two_model_settings={"temperature": 0.0, "store": False}, + ) + + assert isinstance(config.phase_one_model_settings, ModelSettings) + assert config.phase_one_model_settings.reasoning is not None + assert config.phase_one_model_settings.reasoning.effort == "low" + assert config.phase_one_model_settings.retry is not None + assert config.phase_one_model_settings.retry.max_retries == 0 + assert isinstance(config.phase_two_model_settings, ModelSettings) + assert config.phase_two_model_settings.temperature == 0.0 + assert config.phase_two_model_settings.store is False + + +def test_memory_generate_config_subclass_uses_declared_model_settings_types() -> None: + config = cast(Any, _DeclaredProviderMemoryGenerateConfig)( + phase_one_model_settings={"provider_field": "phase-one"}, + phase_two_model_settings={"provider_field": "phase-two"}, + ) + + assert isinstance(config.phase_one_model_settings, _DeclaredProviderModelSettings) + assert config.phase_one_model_settings.provider_field == "phase-one" + assert isinstance(config.phase_two_model_settings, _DeclaredProviderModelSettings) + assert config.phase_two_model_settings.provider_field == "phase-two" + + +def test_memory_generate_config_model_settings_field_types_describe_normalized_values() -> None: + type_hints = get_type_hints(MemoryGenerateConfig) + + assert type_hints["phase_one_model_settings"] == ModelSettings | None + assert type_hints["phase_two_model_settings"] == ModelSettings | None + + +def test_memory_generate_config_preserves_typed_model_settings() -> None: + phase_one_settings = ModelSettings(reasoning={"effort": "low"}) + phase_two_settings = ModelSettings(temperature=0.2) + config = MemoryGenerateConfig( + phase_one_model_settings=phase_one_settings, + phase_two_model_settings=phase_two_settings, + ) + + assert config.phase_one_model_settings is phase_one_settings + assert config.phase_two_model_settings is phase_two_settings + + +@pytest.mark.parametrize( + "field_name", + ["phase_one_model_settings", "phase_two_model_settings"], +) +def test_memory_generate_config_preserves_forward_compatible_reasoning_settings( + field_name: str, +) -> None: + settings: dict[str, Any] = {field_name: {"reasoning": {"future_reasoning_option": "enabled"}}} + + config = MemoryGenerateConfig(**settings) + model_settings = getattr(config, field_name) + + assert model_settings is not None + assert model_settings.reasoning is not None + assert model_settings.reasoning.model_extra == {"future_reasoning_option": "enabled"} + + +@pytest.mark.parametrize( + "field_name", + ["phase_one_model_settings", "phase_two_model_settings"], +) +def test_memory_generate_config_rejects_invalid_model_settings(field_name: str) -> None: + settings: dict[str, Any] = {field_name: "invalid"} + with pytest.raises( + TypeError, + match=f"MemoryGenerateConfig.{field_name} must be a ModelSettings instance or a dict", + ): + MemoryGenerateConfig(**settings) + + +def test_memory_generate_config_preserves_disabled_model_settings() -> None: + config = MemoryGenerateConfig(phase_one_model_settings=None, phase_two_model_settings=None) + + assert config.phase_one_model_settings is None + assert config.phase_two_model_settings is None + + def test_memory_generate_config_rejects_too_many_raw_memories() -> None: with pytest.raises( ValueError, @@ -1395,15 +1497,31 @@ async def test_sandbox_memory_unregisters_manager_on_session_close() -> None: await client.delete(session) +@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"]) +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], + ids=["model_redacted", "tool_redacted", "diagnostic"], +) @pytest.mark.asyncio -async def test_sandbox_memory_enqueue_failure_still_cleans_up_owned_session( +async def test_sandbox_memory_enqueue_failure_follows_both_data_policies( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + streamed: bool, + model_redacted: bool, + tool_redacted: bool, ) -> None: + secret = "SECRET_SANDBOX_MEMORY_PAYLOAD" + error = RuntimeError(secret) + async def _raise_write_rollout(*args: Any, **kwargs: Any) -> Path: _ = args, kwargs - raise RuntimeError("write_rollout failed") + raise error monkeypatch.setattr(memory_manager_module, "write_rollout", _raise_write_rollout) + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + caplog.set_level(logging.WARNING) client = _DeleteTrackingUnixLocalSandboxClient() agent = SandboxAgent( @@ -1413,16 +1531,40 @@ async def _raise_write_rollout(*args: Any, **kwargs: Any) -> Path: capabilities=[_memory_config()], ) - result = await Runner.run( - agent, - "hello", - run_config=RunConfig(sandbox=SandboxRunConfig(client=client)), - ) + run_config = RunConfig(sandbox=SandboxRunConfig(client=client)) + result: RunResult | RunResultStreaming + if streamed: + result = Runner.run_streamed(agent, "hello", run_config=run_config) + async for _ in result.stream_events(): + pass + expected_message = "Failed to enqueue sandbox memory after streamed run" + else: + result = await Runner.run(agent, "hello", run_config=run_config) + expected_message = "Failed to enqueue sandbox memory after run" assert result.final_output == "done" assert len(client.deleted_roots) == 1 assert not client.deleted_roots[0].exists() + record = next( + record + for record in caplog.records + if expected_message in logging.Formatter().format(record) + ) + redacted = model_redacted or tool_redacted + if redacted: + assert record.msg == "%s" + assert record.args == (expected_message,) + assert record.exc_info is None + assert record.exc_text is None + assert error not in record.__dict__.values() + assert secret not in logging.Formatter().format(record) + else: + assert record.args == (expected_message, error) + assert record.exc_info is not None + assert record.exc_info[1] is error + assert secret in logging.Formatter().format(record) + @pytest.mark.asyncio async def test_sandbox_memory_marks_interrupted_runs_in_phase_one_prompt() -> None: diff --git a/tests/sandbox/test_parse_utils.py b/tests/sandbox/test_parse_utils.py index 136b56e090..549f830d1f 100644 --- a/tests/sandbox/test_parse_utils.py +++ b/tests/sandbox/test_parse_utils.py @@ -63,6 +63,28 @@ def test_parse_ls_la_accepts_special_permission_bits() -> None: assert not (entries[2].permissions.other & FileMode.EXEC) +def test_parse_ls_la_strips_trailing_alternate_access_markers() -> None: + # coreutils/BSD ls append a single marker after the mode field: "." (SELinux + # security context), "+" (ACL), or "@" (macOS extended attributes). + output = ( + "drwxr-xr-x. 2 root root 4096 Jan 1 00:00 selinux-dir\n" + "-rw-r--r--+ 1 root root 123 Jan 1 00:00 acl-file\n" + "-rw-r--r--@ 1 root root 456 Jan 1 00:00 xattr-file\n" + ) + + entries = parse_ls_la(output, base="/") + + assert [entry.path for entry in entries] == [ + "/selinux-dir", + "/acl-file", + "/xattr-file", + ] + assert entries[0].permissions.directory is True + assert entries[0].permissions.owner & FileMode.READ + assert entries[1].permissions.owner & FileMode.WRITE + assert entries[2].permissions.owner & FileMode.READ + + @pytest.mark.parametrize( "permissions", [ diff --git a/tests/sandbox/test_runtime.py b/tests/sandbox/test_runtime.py index ac10423592..0a76aed396 100644 --- a/tests/sandbox/test_runtime.py +++ b/tests/sandbox/test_runtime.py @@ -3,6 +3,7 @@ import asyncio import io import json +import logging import os import re import shutil @@ -18,6 +19,7 @@ from openai.types.responses.response_output_item import LocalShellCall, LocalShellCallAction from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary +import agents._debug as _debug import agents.sandbox.runtime_agent_preparation as runtime_agent_preparation_module from agents import Agent, AgentHooks, LocalShellTool, RunHooks, Runner, function_tool from agents.exceptions import InputGuardrailTripwireTriggered, UserError @@ -204,6 +206,12 @@ async def _apply_entry_batch( return [] +class _RejectingLiveSessionDeltaRecorder(_LiveSessionDeltaRecorder): + async def _validate_manifest_application(self, *, only_ephemeral: bool = False) -> None: + _ = only_ephemeral + raise RuntimeError("live manifest update rejected") + + class _PathGuardingSession(_FakeSession): def __init__(self, manifest: Manifest) -> None: super().__init__(manifest) @@ -2109,14 +2117,17 @@ async def _fake_unmount( assert order == [root / "outer" / "child", root / "outer"] +@pytest.mark.parametrize("redacted", [True, False], ids=["redacted", "diagnostic"]) @pytest.mark.asyncio async def test_unix_local_client_delete_skips_rmtree_when_unmount_fails( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, ) -> None: client = UnixLocalSandboxClient() manifest = _unix_local_manifest( entries={ - "remote": S3Mount( + "SECRET_REMOTE_MOUNT": S3Mount( bucket="bucket", mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()), ), @@ -2133,7 +2144,7 @@ async def _failing_unmount( base_dir: Path, ) -> None: _ = (self, session, dest, base_dir) - raise RuntimeError("busy") + raise RuntimeError("SECRET_UNMOUNT_ERROR") def _fake_rmtree(path: Path, ignore_errors: bool = False) -> None: _ = (path, ignore_errors) @@ -2142,12 +2153,34 @@ def _fake_rmtree(path: Path, ignore_errors: bool = False) -> None: monkeypatch.setattr(S3Mount, "unmount", _failing_unmount) monkeypatch.setattr(shutil, "rmtree", _fake_rmtree) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + caplog.set_level(logging.WARNING) await client.delete(session) assert rmtree_called is False assert workspace_root.exists() + record = next( + record + for record in caplog.records + if "Failed to unmount UnixLocal workspace mount" in logging.Formatter().format(record) + ) + mount_path = str(workspace_root / "SECRET_REMOTE_MOUNT") + if redacted: + assert record.msg == "%s" + assert record.args == ("Failed to unmount UnixLocal workspace mount before deleting root",) + assert record.exc_info is None + assert record.exc_text is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert mount_path not in logging.Formatter().format(record) + assert "SECRET_UNMOUNT_ERROR" not in logging.Formatter().format(record) + else: + assert record.__dict__["openai_agents_diagnostic_context"] == {"mount_path": mount_path} + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert "SECRET_UNMOUNT_ERROR" in logging.Formatter().format(record) + shutil.rmtree(workspace_root, ignore_errors=True) @@ -3372,6 +3405,31 @@ async def test_session_manager_materializes_running_injected_session_manifest_mu assert payload is None +@pytest.mark.asyncio +async def test_session_manager_validates_running_manifest_update_before_materialization() -> None: + inner = _RejectingLiveSessionDeltaRecorder(Manifest()) + inner._running = True + live_session = SandboxSession(inner) + capability = _ManifestMutationCapability() + agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.") + manager = SandboxRuntimeSessionManager( + starting_agent=agent, + sandbox_config=SandboxRunConfig(session=live_session), + run_state=None, + ) + + manager.acquire_agent(agent) + with pytest.raises(RuntimeError, match="live manifest update rejected"): + await manager.ensure_session( + agent=agent, + capabilities=[capability], + is_resumed_state=False, + ) + + assert inner.applied_entry_batches == [] + assert live_session.state.manifest.entries == {} + + @pytest.mark.asyncio async def test_session_manager_retries_running_injected_session_delta_apply_after_failure() -> None: live_session = _LiveSessionDeltaRecorder(Manifest(), fail_entry_batch_times=1) diff --git a/tests/sandbox/test_runtime_agent_preparation.py b/tests/sandbox/test_runtime_agent_preparation.py index eff4a3131a..c532f7e990 100644 --- a/tests/sandbox/test_runtime_agent_preparation.py +++ b/tests/sandbox/test_runtime_agent_preparation.py @@ -17,6 +17,49 @@ from agents.sandbox.manifest import Manifest from agents.sandbox.sandbox_agent import SandboxAgent from agents.sandbox.session.base_sandbox_session import BaseSandboxSession +from agents.sandbox.types import User + + +def test_sandbox_agent_normalizes_first_party_dictionary_configuration() -> None: + agent = SandboxAgent( + name="sandbox", + model_settings={"reasoning": {"context": "all_turns"}}, + default_manifest={"root": "/workspace"}, + run_as={"name": "agent"}, + ) + + assert agent.model_settings.reasoning is not None + assert agent.model_settings.reasoning.context == "all_turns" + assert isinstance(agent.default_manifest, Manifest) + assert isinstance(agent.run_as, User) + assert agent.run_as.name == "agent" + + +def test_sandbox_agent_rejects_untrusted_manifest_path_grants() -> None: + with pytest.raises( + TypeError, + match=( + r"sandbox\.default_manifest\.extra_path_grants must be configured " + r"on a trusted Manifest" + ), + ): + SandboxAgent(name="sandbox", default_manifest={"extra_path_grants": [{"path": "/tmp"}]}) + + +@pytest.mark.parametrize( + "manifest", + [ + Manifest(root="/workspace").model_dump(), + Manifest(root="/workspace").model_dump(mode="json"), + ], +) +def test_sandbox_agent_accepts_serialized_manifest_without_path_grants( + manifest: dict[str, Any], +) -> None: + agent = SandboxAgent(name="sandbox", default_manifest=manifest) + + assert isinstance(agent.default_manifest, Manifest) + assert agent.default_manifest.extra_path_grants == () class _Capability: diff --git a/tests/sandbox/test_runtime_helpers.py b/tests/sandbox/test_runtime_helpers.py index dc95804877..97494d60bd 100644 --- a/tests/sandbox/test_runtime_helpers.py +++ b/tests/sandbox/test_runtime_helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import subprocess import sys from pathlib import Path, PurePosixPath @@ -8,6 +9,7 @@ from agents.sandbox.session.runtime_helpers import ( RESOLVE_WORKSPACE_PATH_HELPER, + WORKSPACE_FINGERPRINT_HELPER, RuntimeHelperScript, ) @@ -24,6 +26,13 @@ def _install_resolve_helper(tmp_path: Path) -> Path: return helper_path +def _install_fingerprint_helper(tmp_path: Path) -> Path: + helper_path = tmp_path / "workspace-fingerprint" + helper_path.write_text(WORKSPACE_FINGERPRINT_HELPER.content, encoding="utf-8") + helper_path.chmod(0o755) + return helper_path + + def test_runtime_helper_from_content_uses_posix_install_path() -> None: helper = RuntimeHelperScript.from_content( name="test-helper", @@ -35,6 +44,43 @@ def test_runtime_helper_from_content_uses_posix_install_path() -> None: assert str(helper.install_path).startswith("/tmp/openai-agents/bin/test-helper-") +@requires_posix_shell +def test_workspace_fingerprint_helper_treats_exclusions_as_literal(tmp_path: Path) -> None: + helper_path = _install_fingerprint_helper(tmp_path) + workspace = tmp_path / "workspace" + excluded = workspace / "cache[1]" + durable = workspace / "cache1" + excluded.mkdir(parents=True) + durable.mkdir() + excluded_file = excluded / "remote.txt" + durable_file = durable / "durable.txt" + excluded_file.write_text("remote-one", encoding="utf-8") + durable_file.write_text("durable-one", encoding="utf-8") + + def fingerprint() -> str: + result = subprocess.run( + [ + str(helper_path), + str(workspace), + "test-version", + str(tmp_path / "fingerprint.json"), + "manifest-digest", + "cache[1]", + ], + check=True, + capture_output=True, + text=True, + ) + return str(json.loads(result.stdout)["fingerprint"]) + + first = fingerprint() + excluded_file.write_text("remote-two", encoding="utf-8") + assert fingerprint() == first + + durable_file.write_text("durable-two", encoding="utf-8") + assert fingerprint() != first + + @requires_posix_shell def test_resolve_workspace_path_helper_allows_extra_root_symlink_target(tmp_path: Path) -> None: helper_path = _install_resolve_helper(tmp_path) diff --git a/tests/sandbox/test_session_manager.py b/tests/sandbox/test_session_manager.py index 67891b74c8..a6f5c6d70f 100644 --- a/tests/sandbox/test_session_manager.py +++ b/tests/sandbox/test_session_manager.py @@ -1,11 +1,13 @@ from __future__ import annotations import asyncio +import logging import uuid from pathlib import Path import pytest +import agents._debug as _debug from agents.sandbox.manifest import Manifest from agents.sandbox.runtime_session_manager import SandboxRuntimeSessionManager from agents.sandbox.sandboxes.unix_local import ( @@ -193,6 +195,54 @@ async def handle(self, event: SandboxSessionEvent) -> None: await instrumentation.emit(event) +@pytest.mark.parametrize("redacted", [True, False], ids=["redacted", "diagnostic"]) +@pytest.mark.asyncio +async def test_logged_sink_failure_conditionally_includes_sink_type( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, +) -> None: + class _FailingLogSink(_EventSink): + async def handle(self, event: SandboxSessionEvent) -> None: + _ = event + raise RuntimeError("SECRET_SINK_ERROR") + + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + caplog.set_level(logging.ERROR) + instrumentation = Instrumentation(sinks=[_FailingLogSink(mode="sync", on_error="log")]) + event = SandboxSessionFinishEvent( + session_id=uuid.uuid4(), + seq=1, + op="running", + span_id="span_running", + ok=True, + duration_ms=0.0, + ) + + await instrumentation.emit(event) + + record = next( + record + for record in caplog.records + if "Sandbox event sink failed" in logging.Formatter().format(record) + ) + if redacted: + assert record.msg == "%s" + assert record.args == ("Sandbox event sink failed (ignored)",) + assert record.exc_info is None + assert record.exc_text is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert "_FailingLogSink" not in logging.Formatter().format(record) + assert "SECRET_SINK_ERROR" not in logging.Formatter().format(record) + else: + assert record.__dict__["openai_agents_diagnostic_context"] == { + "sink_type": "_FailingLogSink" + } + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert "SECRET_SINK_ERROR" in logging.Formatter().format(record) + + def test_session_manager_uses_custom_snapshot_spec_without_resolving_default( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/sandbox/test_session_sinks.py b/tests/sandbox/test_session_sinks.py index 6b2f41505d..6f8708e16c 100644 --- a/tests/sandbox/test_session_sinks.py +++ b/tests/sandbox/test_session_sinks.py @@ -34,6 +34,7 @@ from agents.sandbox.session.base_sandbox_session import BaseSandboxSession from agents.sandbox.session.sandbox_session import _read_with_expected_span_errors from agents.sandbox.snapshot import LocalSnapshot +from agents.sandbox.types import ExecResult from agents.tracing import custom_span, trace from tests.testing_processor import fetch_normalized_spans, fetch_ordered_spans @@ -98,6 +99,33 @@ async def test_sandbox_session_write_does_not_include_bytes_when_disabled( assert "bytes" not in write_start.data +@pytest.mark.asyncio +async def test_sandbox_session_apply_manifest_preserves_write_instrumentation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[SandboxSessionEvent] = [] + instrumentation = Instrumentation( + sinks=[CallbackSink(lambda e, _sess: events.append(e), mode="sync")], + ) + inner = _build_unix_local_session( + tmp_path, + manifest=Manifest(entries={"materialized.txt": File(content=b"hello")}), + ) + + async def successful_exec(*_command: str | Path, timeout: float | None = None) -> ExecResult: + _ = timeout + return ExecResult(stdout=b"", stderr=b"", exit_code=0) + + monkeypatch.setattr(inner, "_exec_internal", successful_exec) + session = SandboxSession(inner, instrumentation=instrumentation) + + await session.apply_manifest() + + write_events = [event for event in events if event.op == "write"] + assert [event.phase for event in write_events] == ["start", "finish"] + + @pytest.mark.asyncio async def test_jsonl_outbox_sink_appends_one_line_per_event(tmp_path: Path) -> None: outbox = tmp_path / "events.jsonl" diff --git a/tests/sandbox/test_tar_utils.py b/tests/sandbox/test_tar_utils.py index 3c8ceec9fa..8f82b70af0 100644 --- a/tests/sandbox/test_tar_utils.py +++ b/tests/sandbox/test_tar_utils.py @@ -312,6 +312,36 @@ def test_validate_tar_bytes_specific_symlink_rejection_does_not_reject_children( ) +@pytest.mark.parametrize( + "member", + [ + _file("remote/data.txt"), + _symlink("remote/link", "../outside"), + _file("remote"), + ], +) +def test_validate_tar_bytes_rejects_members_overlapping_protected_path( + member: _Member, +) -> None: + raw = _tar_bytes(member) + + with pytest.raises(UnsafeTarMemberError, match="overlaps protected path: remote"): + validate_tar_bytes(raw, reject_rel_paths={"remote"}) + + +def test_validate_tar_bytes_rejects_non_directory_ancestor_of_protected_path() -> None: + raw = _tar_bytes(_file("remote")) + + with pytest.raises(UnsafeTarMemberError, match="overlaps protected path: remote/nested"): + validate_tar_bytes(raw, reject_rel_paths={"remote/nested"}) + + +def test_validate_tar_bytes_allows_directory_ancestor_of_protected_path() -> None: + raw = _tar_bytes(_dir("remote")) + + validate_tar_bytes(raw, reject_rel_paths={"remote/nested"}) + + def test_safe_extract_tarfile_rejects_preexisting_symlink_parent( tmp_path: Path, ) -> None: diff --git a/tests/test_agent_as_tool.py b/tests/test_agent_as_tool.py index c28c5a8743..d3187fc1a9 100644 --- a/tests/test_agent_as_tool.py +++ b/tests/test_agent_as_tool.py @@ -10,6 +10,7 @@ from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from pydantic import BaseModel, Field +import agents._debug as _debug from agents import ( Agent, AgentBase, @@ -2064,10 +2065,21 @@ async def _invoke_tool() -> Any: @pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], +) async def test_agent_as_tool_streaming_handler_exception_does_not_fail_call( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + model_redacted: bool, + tool_redacted: bool, ) -> None: - agent = Agent(name="handler_error_agent") + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + agent_name = "SECRET_HANDLER_ERROR_AGENT" + agent = Agent(name=agent_name) + secret = "SECRET_AGENT_STREAM_PAYLOAD" class DummyStreamingResult: def __init__(self) -> None: @@ -2087,7 +2099,7 @@ async def stream_events(self): ) def bad_handler(event: AgentToolStreamEvent) -> None: - raise RuntimeError("boom") + raise RuntimeError(secret) tool_call = ResponseFunctionToolCall( id="call_bad", @@ -2110,9 +2122,27 @@ def bad_handler(event: AgentToolStreamEvent) -> None: tool_call=tool_call, ) - output = await tool.on_invoke_tool(tool_context, '{"input": "go"}') + with caplog.at_level("ERROR", logger="openai.agents"): + output = await tool.on_invoke_tool(tool_context, '{"input": "go"}') assert output == "ok" + record = next( + record + for record in caplog.records + if "Error while handling an agent tool on_stream event" in record.getMessage() + ) + if model_redacted or tool_redacted: + assert record.msg == "%s" + assert record.args == ("Error while handling an agent tool on_stream event",) + assert record.exc_info is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert secret not in caplog.text + assert agent_name not in caplog.text + else: + assert record.__dict__["openai_agents_diagnostic_context"] == {"agent_name": agent_name} + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert secret in caplog.text @pytest.mark.asyncio diff --git a/tests/test_agent_config.py b/tests/test_agent_config.py index ad77eeb3e2..f935cfd7a7 100644 --- a/tests/test_agent_config.py +++ b/tests/test_agent_config.py @@ -1,9 +1,13 @@ +from typing import Any + import pytest +from openai.types.shared import Reasoning from pydantic import BaseModel from agents import Agent, AgentOutputSchema, Handoff, RunContextWrapper, handoff from agents.lifecycle import AgentHooksBase from agents.model_settings import ModelSettings +from agents.retry import ModelRetryBackoffSettings from agents.run_internal.run_loop import get_handoffs, get_output_schema @@ -216,11 +220,66 @@ def test_list_field_validation(self): def test_model_settings_validation(self): """Test model_settings validation - prevents runtime errors""" - # Valid case + # Typed settings and SDK-owned dictionaries are both valid. Agent(name="test", model_settings=ModelSettings()) + agent = Agent(name="test", model_settings={"temperature": 0.25}) + + assert isinstance(agent.model_settings, ModelSettings) + assert agent.model_settings.temperature == 0.25 - # Invalid case that could cause runtime issues + # Invalid values are rejected before model execution. with pytest.raises( - TypeError, match="Agent model_settings must be a ModelSettings instance" + TypeError, match="Agent model_settings must be a ModelSettings instance or a dict" ): - Agent(name="test", model_settings={}) # type: ignore + Agent(name="test", model_settings="invalid") # type: ignore[arg-type] + + +def test_agent_model_settings_dictionary_preserves_openai_reasoning_extensions() -> None: + agent = Agent( + name="test", + model_settings={ + "reasoning": {"context": "all_turns", "future_reasoning_option": "enabled"}, + "context_management": [{"type": "compaction", "compact_threshold": 244800}], + "retry": {"max_retries": 0, "backoff": {"jitter": False}}, + }, + ) + + assert isinstance(agent.model_settings.reasoning, Reasoning) + assert agent.model_settings.reasoning.context == "all_turns" + assert agent.model_settings.reasoning.model_extra == {"future_reasoning_option": "enabled"} + assert agent.model_settings.context_management == [ + {"type": "compaction", "compact_threshold": 244800} + ] + assert agent.model_settings.retry is not None + assert agent.model_settings.retry.max_retries == 0 + assert isinstance(agent.model_settings.retry.backoff, ModelRetryBackoffSettings) + assert agent.model_settings.retry.backoff.jitter is False + + +@pytest.mark.parametrize( + ("settings", "message"), + [ + ({"temperatur": 0.2}, "Unknown model settings: temperatur"), + ({"retry": {"max_retry": 2}}, "Unknown model settings in retry: max_retry"), + ( + {"retry": {"backoff": {"initial_delai": 1}}}, + "Unknown model settings in retry.backoff: initial_delai", + ), + ( + {"context_management": [{"type": "compaction", "compact_threshold_typo": 1}]}, + r"Unknown model settings in context_management\[0\]: compact_threshold_typo", + ), + ], +) +def test_agent_rejects_unknown_first_party_dictionary_model_settings( + settings: dict[str, Any], message: str +) -> None: + with pytest.raises(TypeError, match=message): + Agent(name="test", model_settings=settings) + + +@pytest.mark.parametrize("setting_name", ["reasoning", "context_management", "temperature"]) +def test_agent_does_not_promote_model_settings_to_constructor(setting_name: str) -> None: + arguments: dict[str, Any] = {setting_name: None} + with pytest.raises(TypeError, match=f"unexpected keyword argument '{setting_name}'"): + Agent(name="test", **arguments) diff --git a/tests/test_agent_runner.py b/tests/test_agent_runner.py index e620ecc7aa..da74eb615f 100644 --- a/tests/test_agent_runner.py +++ b/tests/test_agent_runner.py @@ -17,6 +17,7 @@ from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary from typing_extensions import TypedDict +import agents._debug as _debug from agents import ( Agent, GuardrailFunctionOutput, @@ -2564,6 +2565,52 @@ async def test_conversation_lock_rewind_skips_when_no_snapshot() -> None: assert session.pop_calls == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize("session_backend", ["memory", "sqlite"]) +async def test_non_streamed_model_retry_does_not_rewind_committed_session_input( + tmp_path: Path, session_backend: str +) -> None: + model = FakeModel() + model.add_multiple_turn_outputs( + [ + APIConnectionError( + message="connection error", + request=httpx.Request("POST", "https://example.com"), + ), + [get_text_message("done")], + ] + ) + agent = Agent( + name="test", + model=model, + model_settings=ModelSettings( + retry=ModelRetrySettings( + max_retries=1, + policy=retry_policies.network_error(), + ) + ), + ) + session: CountingSession | SQLiteSession + if session_backend == "sqlite": + session = SQLiteSession("retry-session", tmp_path / "retry.sqlite3") + await session.add_items([get_text_input_item("previous")]) + else: + session = CountingSession(history=[get_text_input_item("previous")]) + + try: + result = await Runner.run(agent, input="test", session=session) + saved_items = await session.get_items() + finally: + if isinstance(session, SQLiteSession): + session.close() + + assert result.final_output == "done" + assert [item.get("role") for item in saved_items] == ["user", "user", "assistant"] + assert [item.get("content") for item in saved_items[:2]] == ["previous", "test"] + if isinstance(session, CountingSession): + assert session.pop_calls == 0 + + @pytest.mark.asyncio async def test_get_new_response_uses_agent_retry_settings() -> None: model = FakeModel() @@ -2689,6 +2736,45 @@ async def test_rewind_handles_id_stripped_sessions() -> None: assert session.saved_items == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +async def test_rewind_debug_logging_respects_model_and_tool_policies( + monkeypatch, redacted: bool +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + secret = "SECRET_REWIND_SESSION_CONTENT" + session = IdStrippingSession() + item = cast( + TResponseInputItem, + {"id": "message-1", "type": "message", "role": "user", "content": secret}, + ) + await session.add_items([item]) + + with patch("agents.run_internal.session_persistence.logger") as mock_logger: + await rewind_session_items(session, [item]) + + logged = str(mock_logger.debug.call_args_list) + assert (secret not in logged) is redacted + + +@pytest.mark.asyncio +async def test_rewind_failure_uses_placeholder_free_shared_logger_message() -> None: + class FailingTailSession(SimpleListSession): + async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]: + raise RuntimeError("tail failure") + + item = cast(TResponseInputItem, {"type": "message", "role": "user", "content": "hi"}) + session = FailingTailSession(history=[item]) + + with patch( + "agents.run_internal.session_persistence.log_model_and_tool_action_warning" + ) as mock_warning: + await rewind_session_items(session, [item]) + + assert mock_warning.call_args.args[1] == "Failed to rewind session item" + + @pytest.mark.asyncio async def test_rewind_skips_mismatched_tail_suffix() -> None: target = cast(TResponseInputItem, {"type": "message", "role": "user", "content": "target"}) diff --git a/tests/test_agent_runner_streamed.py b/tests/test_agent_runner_streamed.py index e18e5cd2f4..ad7ba89102 100644 --- a/tests/test_agent_runner_streamed.py +++ b/tests/test_agent_runner_streamed.py @@ -2,6 +2,7 @@ import asyncio import json +import logging from typing import Any, cast import httpx @@ -16,6 +17,7 @@ from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary from typing_extensions import TypedDict +import agents._debug as _debug from agents import ( Agent, GuardrailFunctionOutput, @@ -1170,6 +1172,72 @@ def guardrail_function( pass +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], + ids=["model_redacted", "tool_redacted", "diagnostic"], +) +@pytest.mark.asyncio +async def test_streamed_finalizer_failure_follows_both_data_policies( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + model_redacted: bool, + tool_redacted: bool, +) -> None: + async def safe_guardrail( + context: RunContextWrapper[Any], agent: Agent[Any], input: Any + ) -> GuardrailFunctionOutput: + _ = context, agent, input + return GuardrailFunctionOutput(output_info=None, tripwire_triggered=False) + + error = RuntimeError("SECRET_STREAM_FINALIZER_ERROR") + + async def fail_finalizer(_result: Any) -> bool: + raise error + + monkeypatch.setattr( + run_loop, + "input_guardrail_tripwire_triggered_for_stream", + fail_finalizer, + ) + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + agent_name = "SECRET_STREAM_AGENT_NAME" + agent = Agent( + name=agent_name, + input_guardrails=[InputGuardrail(guardrail_function=safe_guardrail)], + model=FakeModel(initial_output=[get_text_message("done")]), + ) + + with caplog.at_level(logging.DEBUG, logger="openai.agents"): + result = Runner.run_streamed(agent, input="user_message") + async for _ in result.stream_events(): + pass + + assert result.final_output == "done" + record = next( + record + for record in caplog.records + if "Error finalizing streamed result" in record.getMessage() + ) + redacted = model_redacted or tool_redacted + if redacted: + assert record.msg == "%s" + assert record.args == ("Error finalizing streamed result",) + assert record.exc_info is None + assert record.exc_text is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + rendered = logging.Formatter().format(record) + assert agent_name not in rendered + assert "SECRET_STREAM_FINALIZER_ERROR" not in rendered + else: + context = record.__dict__["openai_agents_diagnostic_context"] + assert context == {"agent_name": agent_name} + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "SECRET_STREAM_FINALIZER_ERROR" in logging.Formatter().format(record) + + @pytest.mark.asyncio async def test_input_guardrail_streamed_does_not_save_assistant_message_to_session(): async def guardrail_function( diff --git a/tests/test_computer_tool_lifecycle.py b/tests/test_computer_tool_lifecycle.py index 860dcef9b7..bbb0e04baf 100644 --- a/tests/test_computer_tool_lifecycle.py +++ b/tests/test_computer_tool_lifecycle.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging from typing import Any, cast from unittest.mock import AsyncMock @@ -11,6 +12,7 @@ ResponseComputerToolCall, ) +import agents._debug as _debug from agents import ( Agent, ComputerProvider, @@ -68,6 +70,32 @@ def drag(self, path: list[tuple[int, int]]) -> None: return None +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +async def test_dispose_computer_failure_respects_tool_data_policy( + monkeypatch, caplog, redacted: bool +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + + async def dispose(**_kwargs: Any) -> None: + raise RuntimeError("SECRET_COMPUTER_DISPOSE_FAILURE") + + tool = ComputerTool( + computer=ComputerProvider[FakeComputer]( + create=AsyncMock(return_value=FakeComputer()), + dispose=dispose, + ) + ) + ctx = RunContextWrapper(context=None) + await resolve_computer(tool=tool, run_context=ctx) + + with caplog.at_level(logging.WARNING, logger="openai.agents"): + await dispose_resolved_computers(run_context=ctx) + + assert "Failed to dispose computer for run context" in caplog.text + assert ("SECRET_COMPUTER_DISPOSE_FAILURE" not in caplog.text) is redacted + + def _make_message(text: str) -> ResponseOutputMessage: return ResponseOutputMessage( id="msg-1", diff --git a/tests/test_decorators.py b/tests/test_decorators.py new file mode 100644 index 0000000000..567300da2f --- /dev/null +++ b/tests/test_decorators.py @@ -0,0 +1,42 @@ +import types + +from typing_extensions import assert_type + +import agents.decorators as decorators_module +import agents.tool as tool_module +from agents import ( + FunctionTool, + function_tool, + input_guardrail, + output_guardrail, + tool_input_guardrail, + tool_output_guardrail, +) +from agents.decorators import function_tool as decorators_function_tool, tool + + +def test_decorator_module_preserves_existing_imports_and_identities() -> None: + assert isinstance(decorators_module, types.ModuleType) + assert isinstance(tool_module, types.ModuleType) + assert decorators_function_tool is function_tool + assert tool is function_tool + assert decorators_module.input_guardrail is input_guardrail + assert decorators_module.output_guardrail is output_guardrail + assert decorators_module.tool_input_guardrail is tool_input_guardrail + assert decorators_module.tool_output_guardrail is tool_output_guardrail + assert tool_module.function_tool is function_tool + + +def test_tool_alias_supports_bare_and_configured_decorator_forms() -> None: + @tool + def bare_alias() -> str: + return "bare" + + @tool(name_override="configured_alias") + async def configured_alias() -> str: + return "configured" + + assert_type(bare_alias, FunctionTool) + assert_type(configured_alias, FunctionTool) + assert bare_alias.name == "bare_alias" + assert configured_alias.name == "configured_alias" diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py new file mode 100644 index 0000000000..4f08056915 --- /dev/null +++ b/tests/test_error_logging_redaction.py @@ -0,0 +1,565 @@ +"""Error-path logging must not leak model/tool payloads when data logging is disabled. + +The exception attached to a ``SpanError`` is already redacted based on the tracing +flag, but the sibling ``logger.error`` calls used to log the raw exception (and, for +tool actions, the full traceback) unconditionally. These tests lock in that those log +statements honor ``_debug.DONT_LOG_MODEL_DATA`` / ``_debug.DONT_LOG_TOOL_DATA``. +""" + +from __future__ import annotations + +import logging +import pickle +import threading +from logging.handlers import QueueHandler +from pathlib import Path +from queue import SimpleQueue +from typing import Any +from unittest.mock import patch + +import httpx +import pytest +from openai import AsyncOpenAI + +import agents._debug as _debug +from agents import ( + ModelSettings, + ModelTracing, + OpenAIResponsesModel, + RunConfig, + RunContextWrapper, + trace, +) +from agents.logger import ( + log_model_action_debug, + log_model_action_error, + log_model_action_warning, + log_model_and_tool_action_debug, + log_model_and_tool_action_error, + log_model_and_tool_action_warning, + log_tool_action_debug, + log_tool_action_error as log_shared_tool_action_error, + log_tool_action_warning, +) +from agents.run_internal.tool_execution import ( + log_tool_action_error, + resolve_approval_rejection_message, +) +from agents.tracing.processor_interface import TracingProcessor +from agents.tracing.provider import SynchronousMultiTracingProcessor +from agents.tracing.spans import Span +from agents.tracing.traces import Trace + +_SECRET = "super secret prompt content" + + +class _RecordingHandler(logging.Handler): + def __init__(self) -> None: + super().__init__() + self.records: list[logging.LogRecord] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.records.append(record) + + +class _HostileException(Exception): + def __str__(self) -> str: + raise AssertionError("redacted logging inspected __str__") + + def __repr__(self) -> str: + raise AssertionError("redacted logging inspected __repr__") + + def __getattribute__(self, name: str): + if name in {"__class__", "__traceback__"}: + raise AssertionError(f"redacted logging inspected {name}") + return super().__getattribute__(name) + + +class _TruthinessException(Exception): + def __init__(self, *, truthy: bool) -> None: + super().__init__("diagnostic failure") + self.truthy = truthy + self.bool_calls = 0 + + def __bool__(self) -> bool: + self.bool_calls += 1 + if self.truthy: + raise AssertionError("logging inspected exception truthiness") + return False + + +class _FailingTracingProcessor(TracingProcessor): + def __init__(self) -> None: + self.str_calls = 0 + self.lock = threading.Lock() + + def __str__(self) -> str: + self.str_calls += 1 + return "SECRET_TRACE_PROCESSOR_ID" + + def _fail(self) -> None: + raise ValueError(_SECRET) + + def on_trace_start(self, trace: Trace) -> None: + self._fail() + + def on_trace_end(self, trace: Trace) -> None: + self._fail() + + def on_span_start(self, span: Span[Any]) -> None: + self._fail() + + def on_span_end(self, span: Span[Any]) -> None: + self._fail() + + def shutdown(self) -> None: + self._fail() + + def force_flush(self) -> None: + self._fail() + + +def _emit_shared_error_for_location(test_logger, helper) -> None: + helper(test_logger, "Fixed operational message", ValueError("failure")) + + +def _emit_tool_execution_error_for_location() -> None: + log_tool_action_error("Fixed operational message", ValueError("failure")) + + +def _responses_model() -> OpenAIResponsesModel: + return OpenAIResponsesModel( + model="test-model", + openai_client=AsyncOpenAI( + api_key="test", + http_client=httpx.AsyncClient(trust_env=False), + ), + ) + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_get_response_error_redacts_exception_from_logs(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = _responses_model() + + async def raise_fetch(*args, **kwargs): + raise ValueError(_SECRET) + + monkeypatch.setattr(model, "_fetch_response", raise_fetch) + + with patch("agents.models.openai_responses.logger") as mock_logger: + with trace(workflow_name="test"): + with pytest.raises(ValueError): + await model.get_response( + "instr", + "input", + ModelSettings(), + [], + None, + [], + ModelTracing.ENABLED, + previous_response_id=None, + ) + + mock_logger.error.assert_called_once() + logged = str(mock_logger.error.call_args) + assert _SECRET not in logged + assert "ValueError" not in logged + assert "Error getting response" in logged + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_get_response_error_logs_exception_when_model_data_enabled(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + model = _responses_model() + + async def raise_fetch(*args, **kwargs): + raise ValueError(_SECRET) + + monkeypatch.setattr(model, "_fetch_response", raise_fetch) + + with patch("agents.models.openai_responses.logger") as mock_logger: + with trace(workflow_name="test"): + with pytest.raises(ValueError): + await model.get_response( + "instr", + "input", + ModelSettings(), + [], + None, + [], + ModelTracing.ENABLED, + previous_response_id=None, + ) + + mock_logger.error.assert_called_once() + assert _SECRET in str(mock_logger.error.call_args) + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_stream_response_error_redacts_exception_from_logs(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + model = _responses_model() + + async def raise_fetch(*args, **kwargs): + raise ValueError(_SECRET) + + monkeypatch.setattr(model, "_fetch_response", raise_fetch) + + with patch("agents.models.openai_responses.logger") as mock_logger: + with trace(workflow_name="test"): + with pytest.raises(ValueError): + async for _ in model.stream_response( + "instr", + "input", + ModelSettings(), + [], + None, + [], + ModelTracing.ENABLED, + previous_response_id=None, + ): + pass + + mock_logger.error.assert_called_once() + logged = str(mock_logger.error.call_args) + assert _SECRET not in logged + assert "ValueError" not in logged + assert "Error streaming response" in logged + + +def test_log_tool_action_error_redacts_by_default(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + + with patch("agents.run_internal.tool_execution.logger") as mock_logger: + log_tool_action_error("Shell executor failed", ValueError("rm -rf /secret/path")) + + mock_logger.error.assert_called_once() + assert mock_logger.error.call_args.args == ("%s", "Shell executor failed") + # No traceback either, since it can embed the same sensitive data. + assert mock_logger.error.call_args.kwargs.get("exc_info") in (None, False) + + +@pytest.mark.parametrize( + ("helper", "model_flag", "tool_flag"), + [ + (log_model_action_error, True, False), + (log_model_action_debug, True, False), + (log_model_action_warning, True, False), + (log_tool_action_debug, False, True), + (log_shared_tool_action_error, False, True), + (log_tool_action_warning, False, True), + (log_model_and_tool_action_error, True, False), + (log_model_and_tool_action_error, False, True), + (log_model_and_tool_action_debug, True, False), + (log_model_and_tool_action_warning, False, True), + ], +) +def test_shared_error_helpers_do_not_inspect_or_attach_redacted_exceptions( + monkeypatch, + helper, + model_flag: bool, + tool_flag: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_flag) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_flag) + test_logger = logging.Logger("sensitive-logging-redacted") + handler = _RecordingHandler() + test_logger.addHandler(handler) + hostile = _HostileException() + + helper(test_logger, "Fixed operational message", hostile) + + assert len(handler.records) == 1 + record = handler.records[0] + assert record.msg == "%s" + assert record.args == ("Fixed operational message",) + assert record.exc_info is None + assert record.exc_text is None + assert hostile not in record.__dict__.values() + assert logging.Formatter().format(record) == "Fixed operational message" + + +def test_shared_error_helper_preserves_diagnostics_when_enabled(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + test_logger = logging.Logger("sensitive-logging-diagnostic") + handler = _RecordingHandler() + test_logger.addHandler(handler) + error = ValueError(_SECRET) + + log_shared_tool_action_error(test_logger, "Tool failed", error) + + record = handler.records[0] + assert isinstance(record.args, tuple) + assert error in record.args + assert record.exc_info is not None + assert record.exc_info[1] is error + assert _SECRET in logging.Formatter().format(record) + + +@pytest.mark.parametrize( + "helper", + [log_shared_tool_action_error, log_tool_action_warning], +) +@pytest.mark.parametrize("truthy", [False, True], ids=["falsey", "hostile_bool"]) +def test_shared_error_helpers_do_not_evaluate_exception_truthiness( + monkeypatch, + helper, + truthy: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + test_logger = logging.Logger("sensitive-logging-exception-truthiness") + handler = _RecordingHandler() + test_logger.addHandler(handler) + error = _TruthinessException(truthy=truthy) + + try: + raise error + except _TruthinessException: + helper(test_logger, "Tool failed", error) + + record = handler.records[0] + assert error.bool_calls == 0 + assert record.exc_info is not None + assert record.exc_info[0] is type(error) + assert record.exc_info[1] is error + assert record.exc_info[2] is error.__traceback__ + + +@pytest.mark.parametrize("redacted", [True, False]) +def test_shared_error_helper_conditionally_attaches_diagnostic_extra( + monkeypatch, redacted: bool +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + test_logger = logging.Logger("sensitive-logging-diagnostic-extra") + handler = _RecordingHandler() + test_logger.addHandler(handler) + extra_calls = 0 + + def diagnostic_extra() -> dict[str, object]: + nonlocal extra_calls + extra_calls += 1 + return {"sandbox_id": _SECRET} + + log_tool_action_warning( + test_logger, + "Tool failed", + ValueError("failure"), + diagnostic_extra=diagnostic_extra, + ) + + record = handler.records[0] + assert extra_calls == (0 if redacted else 1) + assert ("openai_agents_diagnostic_context" in record.__dict__) is not redacted + if not redacted: + assert record.__dict__["openai_agents_diagnostic_context"] == {"sandbox_id": _SECRET} + + +def test_shared_error_helper_ignores_diagnostic_extra_failure(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + test_logger = logging.Logger("sensitive-logging-diagnostic-extra-failure") + handler = _RecordingHandler() + test_logger.addHandler(handler) + error = RuntimeError("original failure") + + def diagnostic_extra() -> dict[str, object]: + raise AttributeError("metadata failure") + + log_tool_action_warning( + test_logger, + "Tool failed", + error, + diagnostic_extra=diagnostic_extra, + ) + + record = handler.records[0] + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "original failure" in logging.Formatter().format(record) + + +@pytest.mark.parametrize( + "operation", + [ + "on_trace_start", + "on_trace_end", + "on_span_start", + "on_span_end", + "force_flush", + "shutdown", + ], +) +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], +) +def test_trace_processor_failure_identity_follows_both_data_policies( + monkeypatch, + operation: str, + model_redacted: bool, + tool_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + test_logger = logging.Logger("sensitive-logging-trace-processor", level=logging.DEBUG) + test_logger.propagate = False + handler = _RecordingHandler() + test_logger.addHandler(handler) + failing = _FailingTracingProcessor() + multi = SynchronousMultiTracingProcessor() + multi.add_tracing_processor(failing) + + with patch("agents.tracing.provider.logger", test_logger): + if operation.startswith(("on_trace", "on_span")): + getattr(multi, operation)(object()) + else: + getattr(multi, operation)() + + record = next(record for record in handler.records if record.levelno == logging.ERROR) + redacted = model_redacted or tool_redacted + if redacted: + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert failing not in record.__dict__.values() + assert record.exc_info is None + assert _SECRET not in logging.Formatter().format(record) + assert failing.str_calls == 0 + else: + processor_identity = record.__dict__["openai_agents_diagnostic_context"]["trace_processor"] + assert isinstance(processor_identity, str) + assert type(failing).__module__ in processor_identity + assert type(failing).__qualname__ in processor_identity + assert f"{id(failing):x}" in processor_identity + prepared = QueueHandler(SimpleQueue()).prepare(record) + pickle.dumps(prepared) + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert _SECRET in logging.Formatter().format(record) + + +@pytest.mark.parametrize( + "helper", + [log_shared_tool_action_error, log_tool_action_warning], +) +def test_shared_error_helpers_preserve_direct_caller_location(monkeypatch, helper) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + test_logger = logging.Logger("sensitive-logging-location") + handler = _RecordingHandler() + test_logger.addHandler(handler) + + _emit_shared_error_for_location(test_logger, helper) + + record = handler.records[0] + assert Path(record.pathname).resolve() == Path(__file__).resolve() + assert record.funcName == "_emit_shared_error_for_location" + + +def test_tool_execution_error_helper_preserves_external_caller_location(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + test_logger = logging.Logger("sensitive-logging-wrapped-location") + handler = _RecordingHandler() + test_logger.addHandler(handler) + + with patch("agents.run_internal.tool_execution.logger", test_logger): + _emit_tool_execution_error_for_location() + + record = handler.records[0] + assert Path(record.pathname).resolve() == Path(__file__).resolve() + assert record.funcName == "_emit_tool_execution_error_for_location" + + +def test_shared_error_helper_drops_exception_chains_and_notes(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + test_logger = logging.Logger("sensitive-logging-chain") + handler = _RecordingHandler() + test_logger.addHandler(handler) + cause = ValueError(f"{_SECRET} cause") + error = RuntimeError(f"{_SECRET} outer") + error.__cause__ = cause + if hasattr(error, "add_note"): + error.add_note(f"{_SECRET} note") + else: + error.__notes__ = [f"{_SECRET} note"] + + log_model_action_error(test_logger, "Model failed", error) + + record = handler.records[0] + assert record.exc_info is None + assert record.exc_text is None + assert error not in record.__dict__.values() + assert _SECRET not in logging.Formatter().format(record) + + +def test_log_tool_action_error_logs_full_when_tool_data_enabled(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + + with patch("agents.run_internal.tool_execution.logger") as mock_logger: + log_tool_action_error("Shell executor failed", ValueError("rm -rf /secret/path")) + + mock_logger.error.assert_called_once() + logged = str(mock_logger.error.call_args) + assert "/secret/path" in logged + exc_info = mock_logger.error.call_args.kwargs.get("exc_info") + assert isinstance(exc_info, tuple) + assert exc_info[0] is ValueError + assert isinstance(exc_info[1], ValueError) + assert exc_info[2] is None + + +@pytest.mark.asyncio +async def test_approval_rejection_formatter_error_redacts_exception(monkeypatch, caplog) -> None: + caplog.set_level(logging.DEBUG) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + + def boom(_args): + raise ValueError("formatter blew up SECRET_FMT_123") + + tool_name = "SECRET_FORMATTER_TOOL_NAME" + result = await resolve_approval_rejection_message( + context_wrapper=RunContextWrapper(context=None), + run_config=RunConfig(tool_error_formatter=boom), + tool_type="function", + tool_name=tool_name, + call_id="call_1", + ) + + assert isinstance(result, str) and result + record = next( + record for record in caplog.records if "Tool error formatter failed" in record.getMessage() + ) + assert record.msg == "%s" + assert record.args == ("Tool error formatter failed",) + assert record.exc_info is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert tool_name not in caplog.text + assert "SECRET_FMT_123" not in caplog.text + + +@pytest.mark.asyncio +async def test_approval_rejection_formatter_error_logs_full_when_enabled( + monkeypatch, caplog +) -> None: + caplog.set_level(logging.DEBUG) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + + def boom(_args): + raise ValueError("formatter blew up SECRET_FMT_123") + + tool_name = "diagnostic_tool" + await resolve_approval_rejection_message( + context_wrapper=RunContextWrapper(context=None), + run_config=RunConfig(tool_error_formatter=boom), + tool_type="function", + tool_name=tool_name, + call_id="call_1", + ) + + record = next( + record for record in caplog.records if "Tool error formatter failed" in record.getMessage() + ) + assert record.__dict__["openai_agents_diagnostic_context"] == {"tool_name": tool_name} + assert record.exc_info is not None + assert "SECRET_FMT_123" in caplog.text diff --git a/tests/test_programmatic_tool_calling.py b/tests/test_programmatic_tool_calling.py index 13beeacc82..d7830a28bf 100644 --- a/tests/test_programmatic_tool_calling.py +++ b/tests/test_programmatic_tool_calling.py @@ -1934,7 +1934,7 @@ def lookup_inventory(sku: str) -> InventoryOutput: assert result.final_output == "request rejected" function_outputs = _function_output_raw_items(result) assert len(function_outputs) == 1 - assert function_outputs[0]["output"] == "inventory lookup blocked" + assert json.loads(function_outputs[0]["output"]) == {"error": "inventory lookup blocked"} assert _caller_dict(function_outputs[0]["caller"]) == PROGRAM_CALLER @@ -1964,8 +1964,9 @@ async def lookup_inventory(sku: str) -> InventoryOutput: assert result.final_output == "request timed out" function_outputs = _function_output_raw_items(result) assert len(function_outputs) == 1 - assert isinstance(function_outputs[0]["output"], str) - assert "timed out" in function_outputs[0]["output"].lower() + timeout_output = json.loads(function_outputs[0]["output"]) + assert isinstance(timeout_output, dict) + assert "timed out" in timeout_output["error"].lower() assert _caller_dict(function_outputs[0]["caller"]) == PROGRAM_CALLER @@ -2001,12 +2002,23 @@ def lookup_inventory(sku: str) -> InventoryOutput: assert result.final_output == "request rejected" function_outputs = _function_output_raw_items(result) assert len(function_outputs) == 1 - assert function_outputs[0]["output"] == "inventory result blocked" + assert json.loads(function_outputs[0]["output"]) == {"error": "inventory result blocked"} assert _caller_dict(function_outputs[0]["caller"]) == PROGRAM_CALLER @pytest.mark.asyncio -async def test_typed_programmatic_tool_preserves_approval_rejection() -> None: +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +@pytest.mark.parametrize("serialize_state", [False, True], ids=["in-memory", "serialized"]) +@pytest.mark.parametrize( + "rejection_message", + [None, 'Denied: "東京"'], + ids=["default-rejection", "custom-rejection"], +) +async def test_typed_programmatic_tool_preserves_approval_rejection( + streaming: bool, + serialize_state: bool, + rejection_message: str | None, +) -> None: model = FakeModel() model.add_multiple_turn_outputs( [ @@ -2024,18 +2036,40 @@ def lookup_inventory(sku: str) -> InventoryOutput: model=model, tools=[ProgrammaticToolCallingTool(), lookup_inventory], ) - first_result = await Runner.run(agent, "Check inventory") + first_result: Any + if streaming: + first_result = Runner.run_streamed(agent, "Check inventory") + async for _event in first_result.stream_events(): + pass + else: + first_result = await Runner.run(agent, "Check inventory") assert len(first_result.interruptions) == 1 state = first_result.to_state() - state.reject(first_result.interruptions[0]) - result = await Runner.run(agent, state) + if serialize_state: + state = await RunState.from_json(agent, state.to_json()) + state.reject(state.get_interruptions()[0], rejection_message=rejection_message) + result: Any + if streaming: + result = Runner.run_streamed(agent, state) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, state) assert result.final_output == "request rejected" function_outputs = _function_output_raw_items(result) assert len(function_outputs) == 1 - assert function_outputs[0]["output"] == "Tool execution was not approved." + expected_message = rejection_message or "Tool execution was not approved." + assert json.loads(function_outputs[0]["output"]) == {"error": expected_message} assert _caller_dict(function_outputs[0]["caller"]) == PROGRAM_CALLER + assert model.last_turn_args is not None + replayed_output = next( + item + for item in model.last_turn_args["input"] + if isinstance(item, dict) and item.get("type") == "function_call_output" + ) + assert json.loads(replayed_output["output"]) == {"error": expected_message} @pytest.mark.asyncio @@ -2199,7 +2233,7 @@ def lookup_inventory(sku: str) -> InventoryOutput: assert result.final_output == "request rejected" function_outputs = _function_output_raw_items(result) assert len(function_outputs) == 1 - assert function_outputs[0]["output"] == "inventory lookup blocked" + assert json.loads(function_outputs[0]["output"]) == {"error": "inventory lookup blocked"} assert _caller_dict(function_outputs[0]["caller"]) == PROGRAM_CALLER diff --git a/tests/test_run_config.py b/tests/test_run_config.py index e3f78ae88f..7b99b649f2 100644 --- a/tests/test_run_config.py +++ b/tests/test_run_config.py @@ -2,9 +2,19 @@ import pytest -from agents import Agent, RunConfig, Runner, ToolExecutionConfig, ToolNotFoundBehavior +from agents import ( + Agent, + RunConfig, + Runner, + SessionSettings, + ToolExecutionConfig, + ToolNotFoundBehavior, +) from agents.model_settings import ModelSettings from agents.models.interface import Model, ModelProvider +from agents.run_config import SandboxConcurrencyLimits, SandboxRunConfig +from agents.sandbox.manifest import Manifest +from agents.sandbox.snapshot import NoopSnapshotSpec from .fake_model import FakeModel from .test_responses import get_text_message @@ -24,6 +34,99 @@ def get_model(self, model_name: str | None) -> Model: return self.model_to_return +def test_run_config_normalizes_first_party_dictionary_settings() -> None: + config = RunConfig( + model_settings={"reasoning": {"context": "all_turns"}, "temperature": 0.0}, + session_settings={"limit": 5}, + tool_execution={"max_function_tool_concurrency": 2}, + sandbox={ + "manifest": {"root": "/workspace"}, + "snapshot": {"type": "noop"}, + "concurrency_limits": {"manifest_entries": 3}, + }, + ) + + assert isinstance(config.model_settings, ModelSettings) + assert config.model_settings.reasoning is not None + assert config.model_settings.reasoning.context == "all_turns" + assert config.model_settings.temperature == 0.0 + assert isinstance(config.session_settings, SessionSettings) + assert config.session_settings.limit == 5 + assert isinstance(config.tool_execution, ToolExecutionConfig) + assert config.tool_execution.max_function_tool_concurrency == 2 + assert isinstance(config.sandbox, SandboxRunConfig) + assert isinstance(config.sandbox.manifest, Manifest) + assert isinstance(config.sandbox.snapshot, NoopSnapshotSpec) + assert isinstance(config.sandbox.concurrency_limits, SandboxConcurrencyLimits) + assert config.sandbox.concurrency_limits.manifest_entries == 3 + + +def test_run_config_preserves_typed_configuration_instances() -> None: + settings = ModelSettings(temperature=0.2) + session_settings = SessionSettings(limit=3) + config = RunConfig(model_settings=settings, session_settings=session_settings) + + assert config.model_settings is settings + assert config.session_settings is session_settings + + +def test_run_config_rejects_untrusted_manifest_path_grants() -> None: + with pytest.raises( + TypeError, + match=r"sandbox\.manifest\.extra_path_grants must be configured on a trusted Manifest", + ): + RunConfig(sandbox={"manifest": {"extra_path_grants": [{"path": "/tmp"}]}}) + + +@pytest.mark.parametrize( + "manifest", + [ + Manifest(root="/workspace").model_dump(), + Manifest(root="/workspace").model_dump(mode="json"), + ], +) +def test_run_config_accepts_serialized_manifest_without_path_grants( + manifest: dict[str, object], +) -> None: + config = RunConfig(sandbox={"manifest": manifest}) + + assert config.sandbox is not None + assert isinstance(config.sandbox.manifest, Manifest) + assert config.sandbox.manifest.extra_path_grants == () + + +@pytest.mark.parametrize( + ("settings", "message"), + [ + ({"model_settings": {"temperatur": 0.2}}, "Unknown model settings: temperatur"), + ({"session_settings": {"limitt": 2}}, "Unknown session settings: limitt"), + ( + {"tool_execution": {"max_function_tool_concurrenc": 2}}, + "Unknown run_config.tool_execution settings: max_function_tool_concurrenc", + ), + ], +) +def test_run_config_rejects_unknown_first_party_dictionary_fields( + settings: dict[str, object], message: str +) -> None: + with pytest.raises(TypeError, match=message): + RunConfig(**settings) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_runner_accepts_dictionary_run_configuration() -> None: + model = FakeModel(initial_output=[get_text_message("done")]) + agent = Agent(name="test", model=model) + + result = await Runner.run( + agent, + "hello", + run_config={"model_settings": {"temperature": 0.0}}, + ) + + assert result.final_output == "done" + + @pytest.mark.asyncio async def test_model_provider_on_run_config_is_used_for_agent_model_name() -> None: """ diff --git a/tests/test_run_internal_items.py b/tests/test_run_internal_items.py index 235560c67c..d58830092c 100644 --- a/tests/test_run_internal_items.py +++ b/tests/test_run_internal_items.py @@ -1,6 +1,7 @@ from __future__ import annotations import dataclasses +import json from typing import Any, cast import pytest @@ -9,6 +10,7 @@ ResponseToolSearchCall, ResponseToolSearchOutputItem, ) +from openai.types.responses.response_function_tool_call import CallerProgram from openai.types.responses.response_reasoning_item import ResponseReasoningItem from agents import Agent @@ -27,6 +29,51 @@ from agents.run_internal import items as run_items +@pytest.mark.parametrize("mapping_call", [False, True], ids=["typed-call", "mapping-call"]) +def test_programmatic_structured_tool_errors_are_encoded_as_json_objects( + mapping_call: bool, +) -> None: + caller = {"type": "program", "caller_id": "program-42"} + tool_call: Any + if mapping_call: + tool_call = {"type": "function_call", "call_id": "call-42", "caller": caller} + else: + tool_call = ResponseFunctionToolCall( + type="function_call", + call_id="call-42", + name="lookup", + arguments="{}", + caller=CallerProgram(type="program", caller_id="program-42"), + ) + + output = run_items.function_tool_error_output( + tool_call, + 'Rejected: "東京"', + output_json_schema={"type": "object"}, + ) + + assert json.loads(output) == {"error": 'Rejected: "東京"'} + + +@pytest.mark.parametrize("caller", [None, {"type": "direct"}], ids=["no-caller", "direct"]) +@pytest.mark.parametrize("has_schema", [False, True], ids=["untyped", "typed"]) +def test_direct_function_tool_errors_preserve_plain_text( + caller: dict[str, str] | None, + has_schema: bool, +) -> None: + tool_call: dict[str, Any] = {"type": "function_call", "call_id": "call-42"} + if caller is not None: + tool_call["caller"] = caller + + output = run_items.function_tool_error_output( + tool_call, + "Request rejected.", + output_json_schema={"type": "object"} if has_schema else None, + ) + + assert output == "Request rejected." + + def test_drop_orphan_function_calls_preserves_non_mapping_entries() -> None: payload: list[Any] = [ cast(TResponseInputItem, "plain-text-input"), diff --git a/tests/test_strict_schema.py b/tests/test_strict_schema.py index 0a43f78d78..b431fb39bb 100644 --- a/tests/test_strict_schema.py +++ b/tests/test_strict_schema.py @@ -56,6 +56,40 @@ def test_object_with_true_additional_properties(): ensure_strict_json_schema(schema) +def test_object_with_empty_dict_additional_properties(): + # OpenAPI/MCP schemas commonly use ``additionalProperties: {}`` to mean "allow anything". + # That empty mapping is falsy in Python, but it is still non-strict and must be rejected. + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "additionalProperties": {}, + } + with pytest.raises(UserError): + ensure_strict_json_schema(schema) + + +def test_object_with_schema_additional_properties(): + # A non-empty additionalProperties schema is also non-strict and must be rejected. + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "additionalProperties": {"type": "string"}, + } + with pytest.raises(UserError): + ensure_strict_json_schema(schema) + + +def test_object_with_false_additional_properties_is_allowed(): + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "additionalProperties": False, + } + result = ensure_strict_json_schema(schema) + assert result["additionalProperties"] is False + assert result["required"] == ["a"] + + def test_array_items_processing_and_default_removal(): # When processing an array, the items schema is processed recursively. # Also, any "default": None should be removed. diff --git a/tests/test_tool_context.py b/tests/test_tool_context.py index 5f1f9c1976..05f4a8a859 100644 --- a/tests/test_tool_context.py +++ b/tests/test_tool_context.py @@ -95,6 +95,35 @@ def test_tool_context_constructor_accepts_agent_keyword() -> None: assert tool_ctx.agent is agent +def test_tool_context_constructor_normalizes_dictionary_run_config() -> None: + tool_ctx: ToolContext[dict[str, object]] = ToolContext( + context={}, + tool_name="my_tool", + tool_call_id="call-2", + tool_arguments="{}", + run_config={ + "tracing_disabled": True, + "model_settings": {"temperature": 0.0}, + }, + ) + + assert isinstance(tool_ctx.run_config, RunConfig) + assert tool_ctx.run_config.tracing_disabled is True + assert tool_ctx.run_config.model_settings is not None + assert tool_ctx.run_config.model_settings.temperature == 0.0 + + +def test_tool_context_constructor_rejects_unknown_dictionary_run_config_fields() -> None: + with pytest.raises(TypeError, match="Unknown run_config settings: tracin_disabled"): + ToolContext( + context={}, + tool_name="my_tool", + tool_call_id="call-2", + tool_arguments="{}", + run_config={"tracin_disabled": True}, + ) + + def test_tool_context_constructor_infers_namespace_from_tool_call() -> None: tool_call = ResponseFunctionToolCall( type="function_call", @@ -221,6 +250,25 @@ def test_tool_context_from_agent_context_prefers_explicit_run_config() -> None: assert tool_ctx.run_config is explicit_run_config +def test_tool_context_from_agent_context_normalizes_dictionary_run_config() -> None: + tool_call = ResponseFunctionToolCall( + type="function_call", + name="test_tool", + call_id="call-1", + arguments="{}", + ) + + tool_ctx = ToolContext.from_agent_context( + make_context_wrapper(), + tool_call_id="call-1", + tool_call=tool_call, + run_config={"tracing_disabled": True}, + ) + + assert isinstance(tool_ctx.run_config, RunConfig) + assert tool_ctx.run_config.tracing_disabled is True + + @pytest.mark.asyncio async def test_invoke_function_tool_passes_plain_run_context_when_requested() -> None: captured_context: RunContextWrapper[str] | None = None diff --git a/tests/test_trace_processor.py b/tests/test_trace_processor.py index c0d8898599..ae34114a30 100644 --- a/tests/test_trace_processor.py +++ b/tests/test_trace_processor.py @@ -12,9 +12,10 @@ import httpx import pytest +import agents._debug as _debug from agents.tracing import flush_traces, get_trace_provider from agents.tracing.processor_interface import TracingExporter, TracingProcessor -from agents.tracing.processors import BackendSpanExporter, BatchTraceProcessor +from agents.tracing.processors import BackendSpanExporter, BatchTraceProcessor, ConsoleSpanExporter from agents.tracing.provider import DefaultTraceProvider, TraceProvider from agents.tracing.span_data import AgentSpanData from agents.tracing.spans import Span, SpanImpl @@ -45,6 +46,20 @@ def get_trace(processor: TracingProcessor) -> TraceImpl: ) +@pytest.mark.parametrize("redacted", [True, False]) +def test_console_span_exporter_respects_data_policy(monkeypatch, capsys, redacted: bool) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + span = get_span(mock_processor()) + span.span_data.name = "SECRET_CONSOLE_SPAN" + + ConsoleSpanExporter().export([span]) + + output = capsys.readouterr().out + assert ("SECRET_CONSOLE_SPAN" not in output) is redacted + assert "Export span" in output + + @pytest.fixture def mocked_exporter(): exporter = MagicMock() @@ -438,17 +453,31 @@ def test_backend_span_exporter_2xx_success(mock_client): @patch("httpx.Client") -def test_backend_span_exporter_4xx_client_error(mock_client): - mock_response = MagicMock() - mock_response.status_code = 400 - mock_response.text = "Bad Request" +@pytest.mark.parametrize("redacted", [True, False]) +def test_backend_span_exporter_4xx_client_error(mock_client, monkeypatch, caplog, redacted: bool): + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + + class Response: + status_code = 400 + text_reads = 0 + + @property + def text(self) -> str: + self.text_reads += 1 + return "SECRET_TRACE_RESPONSE_BODY" + + mock_response = Response() mock_client.return_value.post.return_value = mock_response exporter = BackendSpanExporter(api_key="test_key") - exporter.export([get_span(mock_processor())]) + with caplog.at_level(logging.ERROR, logger="openai.agents"): + exporter.export([get_span(mock_processor())]) # 4xx should not be retried mock_client.return_value.post.assert_called_once() + assert ("SECRET_TRACE_RESPONSE_BODY" not in caplog.text) is redacted + assert mock_response.text_reads == (0 if redacted else 1) exporter.close() diff --git a/tests/test_update_rclone_pin.py b/tests/test_update_rclone_pin.py new file mode 100644 index 0000000000..42d8ff1f16 --- /dev/null +++ b/tests/test_update_rclone_pin.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import re +import sys +from datetime import datetime, timezone +from pathlib import Path +from types import ModuleType + +import pytest + + +def _load_updater() -> ModuleType: + path = Path(__file__).parents[1] / ".github/scripts/update_rclone_pin.py" + spec = importlib.util.spec_from_file_location("update_rclone_pin", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +updater = _load_updater() + + +def _checksums(version: str) -> str: + return "\n".join( + f"{index:064x} rclone-v{version}-linux-{arch}.zip" + for index, arch in enumerate(updater._RCLONE_ARCHES, start=1) + ) + + +def _release( + version: str, + published_at: str, + *, + asset_observed_at: str | None = None, + draft: bool = False, + prerelease: bool = False, +) -> dict[str, object]: + asset_timestamp = asset_observed_at or published_at + return { + "tag_name": f"v{version}", + "published_at": published_at, + "draft": draft, + "prerelease": prerelease, + "assets": [ + { + "name": asset_name, + "created_at": asset_timestamp, + "updated_at": asset_timestamp, + } + for asset_name in updater._required_asset_names(version) + ], + } + + +def test_latest_stable_release_respects_default_cooldown() -> None: + now = datetime(2026, 7, 22, tzinfo=timezone.utc) + releases = [ + _release("1.75.0", "2026-07-20T00:00:00Z"), + _release("1.74.5", "2026-07-10T00:00:00Z", prerelease=True), + _release("1.74.4", "2026-07-08T00:00:00Z"), + _release("1.74.3", "2026-07-01T00:00:00Z"), + ] + + selected = updater._latest_stable_release( + releases, + cooldown_days=updater._DEFAULT_COOLDOWN_DAYS, + now=now, + ) + + assert updater._release_version(selected) == "1.74.4" + + +def test_latest_stable_release_skips_release_with_fresh_assets() -> None: + now = datetime(2026, 7, 22, tzinfo=timezone.utc) + releases = [ + _release( + "1.75.0", + "2026-07-01T00:00:00Z", + asset_observed_at="2026-07-20T00:00:00Z", + ), + _release("1.74.4", "2026-07-08T00:00:00Z"), + ] + + selected = updater._latest_stable_release( + releases, + cooldown_days=updater._DEFAULT_COOLDOWN_DAYS, + now=now, + ) + + assert updater._release_version(selected) == "1.74.4" + + +def test_stable_release_cooldown_is_customizable() -> None: + release = _release("1.75.0", "2026-07-20T00:00:00Z") + now = datetime(2026, 7, 22, tzinfo=timezone.utc) + + with pytest.raises(RuntimeError, match="7-day cooldown"): + updater._validate_stable_release(release, cooldown_days=7, now=now) + + updater._validate_stable_release(release, cooldown_days=2, now=now) + + +@pytest.mark.parametrize( + "asset_name", + updater._required_asset_names("1.74.4"), +) +def test_stable_release_cooldown_covers_every_required_asset(asset_name: str) -> None: + release = _release("1.74.4", "2026-07-01T00:00:00Z") + asset = updater._asset(release, asset_name) + asset["created_at"] = "2026-07-20T00:00:00Z" + asset["updated_at"] = "2026-07-20T00:00:00Z" + + with pytest.raises(RuntimeError, match=re.escape(f"asset {asset_name}")): + updater._validate_stable_release( + release, + cooldown_days=7, + now=datetime(2026, 7, 22, tzinfo=timezone.utc), + ) + + +def test_stable_release_cooldown_uses_latest_asset_timestamp() -> None: + release = _release("1.74.4", "2026-07-01T00:00:00Z") + asset = updater._asset(release, "SHA256SUMS") + asset["updated_at"] = "2026-07-20T00:00:00Z" + + with pytest.raises(RuntimeError, match="asset SHA256SUMS"): + updater._validate_stable_release( + release, + cooldown_days=7, + now=datetime(2026, 7, 22, tzinfo=timezone.utc), + ) + + +@pytest.mark.parametrize( + ("draft", "prerelease"), + [(True, False), (False, True)], +) +def test_release_selection_rejects_drafts_and_prereleases( + draft: bool, + prerelease: bool, +) -> None: + release = _release( + "1.74.4", + "2026-07-01T00:00:00Z", + draft=draft, + prerelease=prerelease, + ) + + with pytest.raises(RuntimeError, match="not a stable published release"): + updater._validate_stable_release( + release, + cooldown_days=0, + now=datetime(2026, 7, 22, tzinfo=timezone.utc), + ) + + +def test_parse_sha256s_requires_every_runtime_architecture() -> None: + version = "1.2.3" + + parsed = updater._parse_sha256s(_checksums(version), version) + + assert list(parsed) == list(updater._RCLONE_ARCHES) + assert parsed["amd64"] == f"{2:064x}" + + +def test_headers_do_not_send_github_token_to_release_asset_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GITHUB_TOKEN", "secret") + + assert ( + updater._headers("https://api.github.com/repos/rclone/rclone/releases/latest")[ + "Authorization" + ] + == "Bearer secret" + ) + assert "Authorization" not in updater._headers( + "https://github.com/rclone/rclone/releases/download/v1.2.3/SHA256SUMS" + ) + + +def test_parse_sha256s_rejects_incomplete_release() -> None: + with pytest.raises(RuntimeError, match="arm64"): + updater._parse_sha256s( + "1" * 64 + " rclone-v1.2.3-linux-amd64.zip\n", + "1.2.3", + ) + + +def test_validate_asset_sha256s_requires_github_asset_match() -> None: + version = "1.2.3" + sha256_by_arch = updater._parse_sha256s(_checksums(version), version) + release = { + "assets": [ + { + "name": f"rclone-v{version}-linux-{arch}.zip", + "digest": f"sha256:{sha256_by_arch[arch]}", + } + for arch in updater._RCLONE_ARCHES + ] + } + + updater._validate_asset_sha256s(release, version, sha256_by_arch) + release["assets"][0]["digest"] = f"sha256:{'f' * 64}" + + with pytest.raises(RuntimeError, match="does not match GitHub"): + updater._validate_asset_sha256s(release, version, sha256_by_arch) + + +def test_validate_download_sha256_requires_github_asset_match() -> None: + content = b"rclone checksums" + release = { + "assets": [ + { + "name": "SHA256SUMS", + "digest": f"sha256:{hashlib.sha256(content).hexdigest()}", + } + ] + } + + updater._validate_download_sha256(release, "SHA256SUMS", content) + + with pytest.raises(RuntimeError, match="does not match GitHub"): + updater._validate_download_sha256(release, "SHA256SUMS", b"changed") + + +def test_apply_pin_updates_and_checks_both_consumers(tmp_path: Path) -> None: + runtime_path = tmp_path / updater._RUNTIME_PIN_PATH + docker_path = tmp_path / updater._DOCKER_PIN_PATH + runtime_path.parent.mkdir(parents=True) + docker_path.parent.mkdir(parents=True) + runtime_path.write_text( + f"before\n{updater._PYTHON_PIN_BEGIN}\nold\n{updater._PYTHON_PIN_END}\nafter\n" + ) + docker_path.write_text( + f"before\n{updater._DOCKER_PIN_BEGIN}\nold\n{updater._DOCKER_PIN_END}\nafter\n" + ) + pin = updater.RclonePin( + version="1.2.3", + sha256_by_arch=updater._parse_sha256s(_checksums("1.2.3"), "1.2.3"), + ) + + assert updater.apply_pin(tmp_path, pin, check=True) == [runtime_path, docker_path] + assert "old" in runtime_path.read_text() + + assert updater.apply_pin(tmp_path, pin, check=False) == [runtime_path, docker_path] + assert updater.apply_pin(tmp_path, pin, check=True) == [] + assert '_RCLONE_VERSION = "1.2.3"' in runtime_path.read_text() + assert "ARG RCLONE_VERSION=1.2.3" in docker_path.read_text() diff --git a/tests/tracing/test_tracing_env_disable.py b/tests/tracing/test_tracing_env_disable.py index aa2fd93f20..e49b11ea2f 100644 --- a/tests/tracing/test_tracing_env_disable.py +++ b/tests/tracing/test_tracing_env_disable.py @@ -1,5 +1,8 @@ import logging +import pytest + +import agents._debug as _debug from agents.tracing.provider import DefaultTraceProvider from agents.tracing.scope import Scope from agents.tracing.span_data import AgentSpanData @@ -17,6 +20,25 @@ def test_env_read_on_first_use(monkeypatch): assert isinstance(trace, NoOpTrace) +@pytest.mark.parametrize("redacted", [True, False]) +def test_disabled_span_logging_respects_data_policy(monkeypatch, caplog, redacted: bool): + class SensitiveAgentSpanData(AgentSpanData): + def __repr__(self) -> str: + return "SECRET_SPAN_NAME" + + monkeypatch.setenv("OPENAI_AGENTS_DISABLE_TRACING", "1") + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + provider = DefaultTraceProvider() + + with caplog.at_level(logging.DEBUG, logger="openai.agents"): + span = provider.create_span(SensitiveAgentSpanData(name="agent")) + + assert isinstance(span, NoOpSpan) + assert ("SECRET_SPAN_NAME" not in caplog.text) is redacted + assert "Tracing is disabled. Not creating span" in caplog.text + + def test_env_cached_after_first_use(monkeypatch): """Env flag is cached after the first trace and later env changes do not flip it.""" monkeypatch.setenv("OPENAI_AGENTS_DISABLE_TRACING", "0") diff --git a/tests/voice/test_openai_stt.py b/tests/voice/test_openai_stt.py index 81c7bd56b9..e13532a769 100644 --- a/tests/voice/test_openai_stt.py +++ b/tests/voice/test_openai_stt.py @@ -1,6 +1,7 @@ # test_openai_stt_transcription_session.py import asyncio +import base64 import json import time from unittest.mock import AsyncMock, patch @@ -163,7 +164,24 @@ async def test_session_connects_and_configures_successfully(): @pytest.mark.asyncio -async def test_stream_audio_sends_correct_json(): +@pytest.mark.parametrize( + ("buffer", "expected_pcm16"), + [ + ( + np.array([1, 2, 3, 4], dtype=np.int16), + np.array([1, 2, 3, 4], dtype=np.int16), + ), + ( + np.array([-1.5, -1.0, -0.5, 0.0, 0.5, 1.0, 1.5], dtype=np.float32), + np.array([-32767, -32767, -16383, 0, 16383, 32767, 32767], dtype=np.int16), + ), + ], + ids=["int16", "float32"], +) +async def test_stream_audio_sends_pcm16( + buffer: npt.NDArray[np.int16 | np.float32], + expected_pcm16: npt.NDArray[np.int16], +) -> None: """ Test that when audio is placed on the input queue, the session: 1) Base64-encodes the data. @@ -183,9 +201,9 @@ async def test_stream_audio_sends_correct_json(): ) session._websocket = mock_ws - buffer1 = np.array([1, 2, 3, 4], dtype=np.int16) + original_buffer = buffer.copy() queue: asyncio.Queue[npt.NDArray[np.int16 | np.float32] | None] = asyncio.Queue() - await queue.put(buffer1) + await queue.put(buffer) await queue.put(None) await session._stream_audio(queue) @@ -197,7 +215,8 @@ async def test_stream_audio_sends_correct_json(): ] assert len(append_messages) == 1, "No 'input_audio_buffer.append' message was sent." assert append_messages[0]["type"] == "input_audio_buffer.append" - assert "audio" in append_messages[0] + assert base64.b64decode(append_messages[0]["audio"]) == expected_pcm16.tobytes() + np.testing.assert_array_equal(buffer, original_buffer) await session.close() diff --git a/tests/voice/test_pipeline.py b/tests/voice/test_pipeline.py index c60dbf6161..45db259929 100644 --- a/tests/voice/test_pipeline.py +++ b/tests/voice/test_pipeline.py @@ -1,11 +1,15 @@ from __future__ import annotations import asyncio +import logging +from dataclasses import dataclass, field +from typing import Any import numpy as np import numpy.typing as npt import pytest +import agents._debug as _debug from agents import trace from tests.testing_processor import fetch_events, fetch_span_errors @@ -13,6 +17,7 @@ from agents.voice import ( AudioInput, StreamedAudioResult, + STTModelSettings, TTSModelSettings, VoicePipeline, VoicePipelineConfig, @@ -27,6 +32,22 @@ pass +@dataclass +class _ProviderSTTModelSettings(STTModelSettings): + provider_language: str | None = None + + +@dataclass +class _ProviderTTSModelSettings(TTSModelSettings): + provider_voice: str | None = None + + +@dataclass +class _ProviderVoicePipelineConfig(VoicePipelineConfig): + stt_settings: _ProviderSTTModelSettings = field(default_factory=_ProviderSTTModelSettings) + tts_settings: _ProviderTTSModelSettings = field(default_factory=_ProviderTTSModelSettings) + + def test_streamed_audio_result_odd_length_buffer_int16() -> None: result = StreamedAudioResult( FakeTTS(), @@ -40,6 +61,68 @@ def test_streamed_audio_result_odd_length_buffer_int16() -> None: assert transformed.tolist() == [1] +def test_voice_pipeline_config_normalizes_dictionary_settings() -> None: + config = VoicePipelineConfig( + stt_settings={"language": "ja", "temperature": 0.0}, + tts_settings={"voice": "alloy", "buffer_size": 1}, + ) + + assert isinstance(config.stt_settings, STTModelSettings) + assert config.stt_settings.language == "ja" + assert config.stt_settings.temperature == 0.0 + assert isinstance(config.tts_settings, TTSModelSettings) + assert config.tts_settings.voice == "alloy" + assert config.tts_settings.buffer_size == 1 + + +def test_voice_pipeline_config_subclass_uses_declared_settings_types() -> None: + config = _ProviderVoicePipelineConfig( + stt_settings={"provider_language": "ja"}, # type: ignore[arg-type] + tts_settings={"provider_voice": "voice"}, # type: ignore[arg-type] + ) + + assert isinstance(config.stt_settings, _ProviderSTTModelSettings) + assert config.stt_settings.provider_language == "ja" + assert isinstance(config.tts_settings, _ProviderTTSModelSettings) + assert config.tts_settings.provider_voice == "voice" + + +@pytest.mark.parametrize( + ("settings", "message"), + [ + ({"stt_settings": {"languge": "ja"}}, "Unknown voice.stt settings: languge"), + ({"tts_settings": {"voce": "alloy"}}, "Unknown voice.tts settings: voce"), + ], +) +def test_voice_pipeline_config_rejects_unknown_dictionary_settings( + settings: dict[str, Any], message: str +) -> None: + with pytest.raises(TypeError, match=message): + VoicePipelineConfig(**settings) + + +@pytest.mark.asyncio +async def test_voicepipeline_normalizes_nested_dictionary_config() -> None: + fake_stt = FakeSTT(["first"]) + fake_tts = FakeTTS() + pipeline = VoicePipeline( + workflow=FakeWorkflow([["out_1"]]), + stt_model=fake_stt, + tts_model=fake_tts, + config={ + "stt_settings": {"language": "ja"}, + "tts_settings": {"voice": "alloy", "buffer_size": 1}, + }, + ) + + result = await pipeline.run(AudioInput(buffer=np.zeros(2, dtype=np.int16))) + events, audio_chunks = await extract_events(result) + + assert isinstance(pipeline.config, VoicePipelineConfig) + assert events == ["turn_started", "audio", "turn_ended", "session_ended"] + await fake_tts.verify_audio("out_1", audio_chunks[0]) + + def test_streamed_audio_result_odd_length_buffer_float32() -> None: result = StreamedAudioResult( FakeTTS(), @@ -356,10 +439,24 @@ async def run(self, _: str): yield "out_1" +class _FailingWorkflow(FakeWorkflow): + def __init__(self, error: BaseException): + super().__init__() + self.error = error + + async def run(self, _: str): + raise self.error + yield "" # pragma: no cover + + class _OnStartYieldThenFailWorkflow(FakeWorkflow): + def __init__(self, outputs: list[list[str]], error: BaseException | None = None): + super().__init__(outputs) + self.error = error or RuntimeError("boom") + async def on_start(self): yield "intro" - raise RuntimeError("boom") + raise self.error @pytest.mark.asyncio @@ -412,3 +509,140 @@ async def test_voicepipeline_multi_turn_on_start_exception_does_not_abort() -> N assert events[-1] == "session_ended" assert "error" not in events + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "redacted"), + [ + (True, False, True), + (False, True, True), + (False, False, False), + ], +) +async def test_voice_on_start_errors_apply_model_and_tool_logging_policies( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + model_redacted: bool, + tool_redacted: bool, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + cause = ValueError("SECRET_VOICE_ON_START_TOOL_PAYLOAD") + error = RuntimeError("Voice startup failed") + error.__cause__ = cause + pipeline = VoicePipeline( + workflow=_OnStartYieldThenFailWorkflow([["out_1"]], error), + stt_model=FakeSTT(["first"]), + tts_model=FakeTTS(), + ) + streamed_audio_input = await FakeStreamedAudioInput.get(count=1) + caplog.set_level(logging.WARNING, logger="openai.agents") + + result = await pipeline.run(streamed_audio_input) + events, _ = await extract_events(result) + + assert events[-1] == "session_ended" + records = [ + record + for record in caplog.records + if record.name == "openai.agents" + and ( + record.msg + in { + "Voice workflow on_start failed", + "Voice workflow on_start failed: %s", + } + or ( + isinstance(record.args, tuple) + and record.args + and record.args[0] == "Voice workflow on_start failed" + ) + ) + ] + assert len(records) == 1 + record = records[0] + if redacted: + assert record.msg == "%s" + assert record.args == ("Voice workflow on_start failed",) + assert record.exc_info is None + assert record.exc_text is None + assert error not in record.__dict__.values() + assert cause not in record.__dict__.values() + assert "SECRET_VOICE_ON_START_TOOL_PAYLOAD" not in logging.Formatter().format(record) + else: + assert record.msg == "%s: %s" + assert isinstance(record.args, tuple) + assert error in record.args + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "SECRET_VOICE_ON_START_TOOL_PAYLOAD" in logging.Formatter().format(record) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "redacted"), + [ + (True, False, True), + (False, True, True), + (False, False, False), + ], +) +async def test_voice_workflow_errors_apply_model_and_tool_logging_policies( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + streamed: bool, + model_redacted: bool, + tool_redacted: bool, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + error = RuntimeError("SECRET_VOICE_TOOL_PAYLOAD") + pipeline = VoicePipeline( + workflow=_FailingWorkflow(error), + stt_model=FakeSTT(["first"]), + tts_model=FakeTTS(), + ) + audio_input = ( + await FakeStreamedAudioInput.get(count=1) + if streamed + else AudioInput(buffer=np.zeros(2, dtype=np.int16)) + ) + caplog.set_level(logging.ERROR, logger="openai.agents") + + result = await pipeline.run(audio_input) + with pytest.raises(RuntimeError, match="SECRET_VOICE_TOOL_PAYLOAD"): + await extract_events(result) + assert result.text_generation_task is not None + assert result.text_generation_task.exception() is error + + expected_pipeline_message = ( + "Error processing voice turns" if streamed else "Error processing single voice turn" + ) + records = [ + record + for record in caplog.records + if record.name == "openai.agents" + and isinstance(record.args, tuple) + and record.args + and record.args[0] in {expected_pipeline_message, "Error processing voice output"} + ] + assert len(records) == 2 + for record in records: + if redacted: + assert record.msg == "%s" + assert record.args in { + (expected_pipeline_message,), + ("Error processing voice output",), + } + assert record.exc_info is None + assert "SECRET_VOICE_TOOL_PAYLOAD" not in logging.Formatter().format(record) + else: + assert record.msg == "%s: %s" + assert isinstance(record.args, tuple) + assert error in record.args + assert record.exc_info is not None + assert record.exc_info[1] is error