From 3b93b512815dea507fe039448d20dd41ba0c5440 Mon Sep 17 00:00:00 2001 From: Dhruv Maniya Date: Thu, 30 Jul 2026 09:53:05 +0530 Subject: [PATCH 1/2] feat(sandbox): surface agent transcript errors Signed-off-by: Dhruv Maniya --- containers/entrypoint.py | 2 + src/forge/sandbox/runner.py | 96 +++++++++++++++++++- tests/unit/sandbox/test_transcript_errors.py | 78 ++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 tests/unit/sandbox/test_transcript_errors.py diff --git a/containers/entrypoint.py b/containers/entrypoint.py index 1ee0e129..b702c30a 100644 --- a/containers/entrypoint.py +++ b/containers/entrypoint.py @@ -565,6 +565,8 @@ def _save_conversation_history( "role": getattr(msg, "type", "unknown"), "content": getattr(msg, "content", str(msg)), "tool_calls": getattr(msg, "tool_calls", None), + "status": getattr(msg, "status", None), + "response_metadata": getattr(msg, "response_metadata", None), } for msg in messages ], diff --git a/src/forge/sandbox/runner.py b/src/forge/sandbox/runner.py index 3e5de1e8..82b5c722 100644 --- a/src/forge/sandbox/runner.py +++ b/src/forge/sandbox/runner.py @@ -66,6 +66,82 @@ def _process_cycle( EXIT_CONFIG_ERROR = 3 +def _contains_error_result(value: Any) -> bool: + """Return whether nested tool output contains an explicit error result.""" + if isinstance(value, dict): + if value.get("result") == "error" or value.get("status") == "error": + return True + return any(_contains_error_result(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_error_result(item) for item in value) + return False + + +def _extract_agent_transcript_errors( + workspace_path: Path, + task_key: str, + max_assistant_turns: int = 3, +) -> tuple[Path | None, list[str]]: + """Extract bounded error context from a task's persisted agent history.""" + transcript_path = workspace_path / ".forge" / "history" / f"{task_key}.json" + if not transcript_path.is_file(): + return None, [] + + try: + data = json.loads(transcript_path.read_text()) + messages = data.get("messages", []) + if not isinstance(messages, list): + raise ValueError("'messages' must be a list") + except (OSError, ValueError, json.JSONDecodeError) as exc: + logger.warning(f"Could not parse agent transcript {transcript_path}: {exc}") + return transcript_path, [] + + assistant_indices = [ + index + for index, message in enumerate(messages) + if isinstance(message, dict) + and message.get("role") in {"ai", "assistant"} + ] + start = ( + assistant_indices[-max_assistant_turns] + if len(assistant_indices) >= max_assistant_turns + else 0 + ) + + errors: list[str] = [] + refusal_markers = ("i cannot", "i can't", "i am unable", "i'm unable") + for message in messages[start:]: + if not isinstance(message, dict): + continue + + role = str(message.get("role", "unknown")) + content = message.get("content", "") + content_text = ( + content if isinstance(content, str) else json.dumps(content, default=str) + ) + metadata = message.get("response_metadata") + metadata = metadata if isinstance(metadata, dict) else {} + stop_reason = metadata.get("stop_reason") or metadata.get("finish_reason") + + signal: str | None = None + if stop_reason in {"max_tokens", "length"}: + signal = f"stopped with {stop_reason}" + elif role in {"tool", "function"} and ( + message.get("status") == "error" or _contains_error_result(content) + ): + signal = "tool returned an error" + elif role in {"ai", "assistant"} and any( + marker in content_text.lower() for marker in refusal_markers + ): + signal = "assistant refusal" + + if signal: + excerpt = " ".join(content_text.split())[:1000] + errors.append(f"{signal} ({role}): {excerpt}") + + return transcript_path, errors + + @dataclass class ContainerResult: """Result from container execution.""" @@ -651,6 +727,8 @@ def _build_container_result( stderr_str: str, collected_cycles: list[ReviewCycleData], container_name: str, + transcript_path: Path | None = None, + transcript_errors: list[str] | None = None, ) -> ContainerResult: """Map container exit code to a ContainerResult. @@ -663,6 +741,8 @@ def _build_container_result( stderr_str: Decoded container stderr. collected_cycles: Review cycles collected during execution. container_name: Container name for log messages. + transcript_path: Persisted agent transcript, when available. + transcript_errors: Error signals extracted from recent transcript turns. Returns: ContainerResult reflecting the exit status. @@ -676,6 +756,10 @@ def _build_container_result( logger.info(f"Container stderr:\n{stderr_str}") if stdout_str: logger.debug(f"Container stdout:\n{stdout_str}") + if transcript_path: + logger.error(f"Agent transcript: {transcript_path}") + for error_context in transcript_errors or []: + logger.error(f"Agent transcript error context: {error_context}") if self.settings.container_keep: logger.warning( f"Container kept for debugging (FORGE_CONTAINER_KEEP=true): " @@ -844,9 +928,19 @@ async def run( exit_code = process.returncode or 0 stdout_str = stdout.decode("utf-8", errors="replace") stderr_str = stderr.decode("utf-8", errors="replace") + transcript_path, transcript_errors = _extract_agent_transcript_errors( + workspace_path, + task_key or "UNKNOWN", + ) return self._build_container_result( - exit_code, stdout_str, stderr_str, collected_cycles, container_name + exit_code, + stdout_str, + stderr_str, + collected_cycles, + container_name, + transcript_path, + transcript_errors, ) finally: diff --git a/tests/unit/sandbox/test_transcript_errors.py b/tests/unit/sandbox/test_transcript_errors.py new file mode 100644 index 00000000..fdf6ca40 --- /dev/null +++ b/tests/unit/sandbox/test_transcript_errors.py @@ -0,0 +1,78 @@ +"""Tests for surfacing persisted agent transcript failures.""" + +import json +import logging +from pathlib import Path +from unittest.mock import MagicMock + +from forge.sandbox.runner import ( + ContainerRunner, + _extract_agent_transcript_errors, +) + + +def _runner_without_init() -> ContainerRunner: + runner = object.__new__(ContainerRunner) + runner.settings = MagicMock(container_keep=False) + return runner + + +def test_extracts_errors_from_recent_assistant_turns(tmp_path: Path) -> None: + history_dir = tmp_path / ".forge" / "history" + history_dir.mkdir(parents=True) + transcript = history_dir / "TASK-1.json" + transcript.write_text( + json.dumps( + { + "messages": [ + {"role": "assistant", "content": "I cannot handle the old step."}, + {"role": "assistant", "content": "Trying another approach."}, + { + "role": "tool", + "content": {"result": "error", "message": "permission denied"}, + }, + { + "role": "assistant", + "content": "The context ended.", + "response_metadata": {"stop_reason": "max_tokens"}, + }, + {"role": "assistant", "content": "I'm unable to continue."}, + ] + } + ) + ) + + path, errors = _extract_agent_transcript_errors( + tmp_path, + "TASK-1", + max_assistant_turns=3, + ) + + assert path == transcript + assert len(errors) == 3 + assert any("tool returned an error" in error for error in errors) + assert any("stopped with max_tokens" in error for error in errors) + assert any("assistant refusal" in error for error in errors) + assert all("old step" not in error for error in errors) + + +def test_failed_result_logs_transcript_path_and_context( + tmp_path: Path, + caplog, +) -> None: + runner = _runner_without_init() + transcript = tmp_path / "TASK-1.json" + + with caplog.at_level(logging.ERROR): + runner._build_container_result( + exit_code=1, + stdout_str="", + stderr_str="agent failed", + collected_cycles=[], + container_name="forge-task-1", + transcript_path=transcript, + transcript_errors=["assistant refusal (assistant): cannot continue"], + ) + + assert f"Agent transcript: {transcript}" in caplog.text + assert "assistant refusal (assistant): cannot continue" in caplog.text From d23b3dc1becf4d669741b8a9a841d41d2ac4616d Mon Sep 17 00:00:00 2001 From: Dhruv Maniya Date: Thu, 30 Jul 2026 10:15:56 +0530 Subject: [PATCH 2/2] style: apply Ruff formatting Signed-off-by: Dhruv Maniya --- src/forge/sandbox/runner.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/forge/sandbox/runner.py b/src/forge/sandbox/runner.py index 82b5c722..2ace0e30 100644 --- a/src/forge/sandbox/runner.py +++ b/src/forge/sandbox/runner.py @@ -99,8 +99,7 @@ def _extract_agent_transcript_errors( assistant_indices = [ index for index, message in enumerate(messages) - if isinstance(message, dict) - and message.get("role") in {"ai", "assistant"} + if isinstance(message, dict) and message.get("role") in {"ai", "assistant"} ] start = ( assistant_indices[-max_assistant_turns] @@ -116,9 +115,7 @@ def _extract_agent_transcript_errors( role = str(message.get("role", "unknown")) content = message.get("content", "") - content_text = ( - content if isinstance(content, str) else json.dumps(content, default=str) - ) + content_text = content if isinstance(content, str) else json.dumps(content, default=str) metadata = message.get("response_metadata") metadata = metadata if isinstance(metadata, dict) else {} stop_reason = metadata.get("stop_reason") or metadata.get("finish_reason")