From 85f689612f3fff70274af24739bb22b522455418 Mon Sep 17 00:00:00 2001 From: yaodong-shen Date: Tue, 21 Jul 2026 17:28:31 +0800 Subject: [PATCH] fix(langchain): capture structured output tool calls --- .../callbacks/langchain/callback.py | 19 ++-- .../integration/callbacks/langchain/utils.py | 90 +++++++++++++++++++ .../callbacks/langchain/test_utils.py | 72 +++++++++++++++ 3 files changed, 168 insertions(+), 13 deletions(-) create mode 100644 tests/unit/integration/callbacks/langchain/test_utils.py diff --git a/agentops/integration/callbacks/langchain/callback.py b/agentops/integration/callbacks/langchain/callback.py index 65d07f470..6feebcca9 100644 --- a/agentops/integration/callbacks/langchain/callback.py +++ b/agentops/integration/callbacks/langchain/callback.py @@ -14,7 +14,7 @@ from agentops.logging import logger from agentops.sdk.core import tracer from agentops.semconv import SpanKind, SpanAttributes, LangChainAttributes, LangChainAttributeValues, CoreAttributes -from agentops.integration.callbacks.langchain.utils import get_model_info +from agentops.integration.callbacks.langchain.utils import get_completion_attributes, get_model_info from langchain_core.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler from langchain_core.outputs import LLMResult @@ -237,18 +237,11 @@ def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None: span = self.active_spans.get(run_id) - if hasattr(response, "generations") and response.generations: - completions = [] - for gen_list in response.generations: - for gen in gen_list: - if hasattr(gen, "text"): - completions.append(gen.text) - - if completions: - try: - span.set_attribute(SpanAttributes.LLM_COMPLETIONS, safe_serialize(completions)) - except Exception as e: - logger.warning(f"Failed to set completions: {e}") + try: + for attribute, value in get_completion_attributes(response).items(): + span.set_attribute(attribute, value) + except Exception as e: + logger.warning(f"Failed to set completions: {e}") if hasattr(response, "llm_output") and response.llm_output: token_usage = response.llm_output.get("token_usage", {}) diff --git a/agentops/integration/callbacks/langchain/utils.py b/agentops/integration/callbacks/langchain/utils.py index 20575595a..806657441 100644 --- a/agentops/integration/callbacks/langchain/utils.py +++ b/agentops/integration/callbacks/langchain/utils.py @@ -2,9 +2,12 @@ Utility functions for LangChain integration. """ +from collections.abc import Mapping from typing import Any, Dict, Optional +from agentops.helpers.serialization import safe_serialize from agentops.logging import logger +from agentops.semconv import MessageAttributes, SpanAttributes def get_model_info(serialized: Optional[Dict[str, Any]]) -> Dict[str, str]: @@ -52,3 +55,90 @@ def get_model_info(serialized: Optional[Dict[str, Any]]) -> Dict[str, str]: logger.warning(f"Error extracting model info: {e}") return model_info + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + """Return a mapping for common model and dictionary representations.""" + if isinstance(value, Mapping): + return value + if hasattr(value, "model_dump"): + converted = value.model_dump() + return converted if isinstance(converted, Mapping) else {} + if hasattr(value, "dict"): + converted = value.dict() + return converted if isinstance(converted, Mapping) else {} + return {} + + +def _serialize_tool_arguments(arguments: Any) -> str: + """Serialize tool arguments without double-encoding an existing JSON string.""" + return arguments if isinstance(arguments, str) else safe_serialize(arguments) + + +def get_completion_attributes(response: Any) -> Dict[str, Any]: + """Extract text and tool-call completions from a LangChain LLM result. + + Structured-output models return an ``AIMessage`` whose text content is often + empty while the actual result lives in ``message.tool_calls``. LangChain also + supports an older OpenAI-shaped representation in ``additional_kwargs``. + Emit both forms using AgentOps' indexed completion semantic conventions so + the dashboard can render the structured result instead of an empty response. + """ + attributes: Dict[str, Any] = {} + generations = getattr(response, "generations", None) or [] + completion_index = 0 + + for generation_group in generations: + candidates = generation_group if isinstance(generation_group, (list, tuple)) else [generation_group] + for generation in candidates: + prefix = f"{SpanAttributes.LLM_COMPLETIONS}.{completion_index}" + message = getattr(generation, "message", None) + + if message is not None: + content = getattr(message, "content", None) + raw_role = getattr(message, "type", None) or getattr(message, "role", None) + else: + content = getattr(generation, "text", None) + raw_role = None + + if content not in (None, ""): + attributes[MessageAttributes.COMPLETION_CONTENT.format(i=completion_index)] = ( + content if isinstance(content, str) else safe_serialize(content) + ) + + role = {"ai": "assistant", "human": "user"}.get(raw_role, raw_role or "assistant") + attributes[MessageAttributes.COMPLETION_ROLE.format(i=completion_index)] = role + + generation_info = _as_mapping(getattr(generation, "generation_info", None)) + response_metadata = _as_mapping(getattr(message, "response_metadata", None)) + finish_reason = generation_info.get("finish_reason") or response_metadata.get("finish_reason") + if finish_reason: + attributes[MessageAttributes.COMPLETION_FINISH_REASON.format(i=completion_index)] = finish_reason + + tool_calls = getattr(message, "tool_calls", None) if message is not None else None + if not tool_calls and message is not None: + additional_kwargs = _as_mapping(getattr(message, "additional_kwargs", None)) + tool_calls = additional_kwargs.get("tool_calls") + + for tool_index, raw_tool_call in enumerate(tool_calls or []): + tool_call = _as_mapping(raw_tool_call) + function = _as_mapping(tool_call.get("function")) + tool_prefix = f"{prefix}.tool_calls.{tool_index}" + + tool_id = tool_call.get("id") + tool_type = tool_call.get("type") + tool_name = tool_call.get("name") or function.get("name") + arguments = tool_call.get("args") if "args" in tool_call else function.get("arguments") + + if tool_id: + attributes[f"{tool_prefix}.id"] = tool_id + if tool_type: + attributes[f"{tool_prefix}.type"] = tool_type + if tool_name: + attributes[f"{tool_prefix}.name"] = tool_name + if arguments is not None: + attributes[f"{tool_prefix}.arguments"] = _serialize_tool_arguments(arguments) + + completion_index += 1 + + return attributes diff --git a/tests/unit/integration/callbacks/langchain/test_utils.py b/tests/unit/integration/callbacks/langchain/test_utils.py new file mode 100644 index 000000000..5d4e606de --- /dev/null +++ b/tests/unit/integration/callbacks/langchain/test_utils.py @@ -0,0 +1,72 @@ +import json +from types import SimpleNamespace + +from agentops.integration.callbacks.langchain.utils import get_completion_attributes +from agentops.semconv import MessageAttributes, SpanAttributes + + +def test_get_completion_attributes_serializes_text_generation(): + response = SimpleNamespace(generations=[[SimpleNamespace(text="Hello", generation_info={"finish_reason": "stop"})]]) + + attributes = get_completion_attributes(response) + + assert attributes[MessageAttributes.COMPLETION_CONTENT.format(i=0)] == "Hello" + assert attributes[MessageAttributes.COMPLETION_ROLE.format(i=0)] == "assistant" + assert attributes[MessageAttributes.COMPLETION_FINISH_REASON.format(i=0)] == "stop" + assert SpanAttributes.LLM_COMPLETIONS not in attributes + + +def test_get_completion_attributes_serializes_structured_output_tool_call(): + message = SimpleNamespace( + content="", + type="ai", + tool_calls=[ + { + "id": "call_1", + "type": "tool_call", + "name": "Score", + "args": {"score": 7, "reason": "clear answer"}, + } + ], + additional_kwargs={}, + response_metadata={"finish_reason": "tool_calls"}, + ) + response = SimpleNamespace(generations=[[SimpleNamespace(message=message, text="")]]) + + attributes = get_completion_attributes(response) + + prefix = f"{SpanAttributes.LLM_COMPLETIONS}.0.tool_calls.0" + assert attributes[MessageAttributes.COMPLETION_ROLE.format(i=0)] == "assistant" + assert attributes[MessageAttributes.COMPLETION_FINISH_REASON.format(i=0)] == "tool_calls" + assert attributes[f"{prefix}.id"] == "call_1" + assert attributes[f"{prefix}.type"] == "tool_call" + assert attributes[f"{prefix}.name"] == "Score" + assert json.loads(attributes[f"{prefix}.arguments"]) == {"score": 7, "reason": "clear answer"} + assert MessageAttributes.COMPLETION_CONTENT.format(i=0) not in attributes + + +def test_get_completion_attributes_supports_openai_shaped_tool_calls(): + message = SimpleNamespace( + content=None, + type="ai", + tool_calls=[], + additional_kwargs={ + "tool_calls": [ + { + "id": "call_legacy", + "type": "function", + "function": {"name": "Score", "arguments": '{"score": 9}'}, + } + ] + }, + response_metadata={}, + ) + response = SimpleNamespace(generations=[[SimpleNamespace(message=message, text="")]]) + + attributes = get_completion_attributes(response) + + prefix = f"{SpanAttributes.LLM_COMPLETIONS}.0.tool_calls.0" + assert attributes[f"{prefix}.id"] == "call_legacy" + assert attributes[f"{prefix}.type"] == "function" + assert attributes[f"{prefix}.name"] == "Score" + assert attributes[f"{prefix}.arguments"] == '{"score": 9}'