From cb5bd697fb745f4c88aa3837ed8e849e786e0fe9 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Mon, 3 Aug 2026 22:58:25 +0200 Subject: [PATCH] Add `chat` instrumentation for the smolagents model classes Wrap every smolagents model class that defines its own `generate`, and emit a chat span through the public opentelemetry-util-genai API. The span records the request parameters, provider and endpoint, token usage, finish reason, tool definitions, and the input and output messages. Agent runs and tool calls stay uninstrumented. They follow in separate PRs. Tests run against smolagents 1.24.0 (the declared floor) and against the latest release, plus Weaver conformance scenarios for chat, tool-calling, image input, and reasoning output. Known gaps: - Model.generate_stream is not instrumented, so a call made with stream_outputs=True produces no chat span, no metrics, and no token usage. - A user-defined Model subclass that overrides generate shadows the patched base method and produces no chat span. --- instrumentation/README.md | 2 +- .../.changelog/352.added | 1 + .../README.rst | 37 +- .../pyproject.toml | 2 +- .../genai/smolagents/__init__.py | 93 +- .../genai/smolagents/_messages.py | 316 ++++ .../genai/smolagents/package.py | 2 + .../instrumentation/genai/smolagents/patch.py | 436 ++++++ .../genai/smolagents/provider.py | 151 ++ .../tests/cassettes/litellm_reasoning.yaml | 20 + .../tests/cassettes/openai_model_basic.yaml | 25 + .../cassettes/openai_model_image_url.yaml | 26 + .../tests/cassettes/openai_model_tool.yaml | 32 + .../tests/conformance/__init__.py | 0 .../tests/conformance/_helpers.py | 34 + .../tests/conformance/inference.py | 113 ++ .../tests/conformance/multimodal.py | 163 +++ .../tests/conftest.py | 85 +- .../tests/requirements.latest.txt | 5 +- .../tests/requirements.oldest.txt | 14 +- .../tests/test_conformance.py | 46 + .../tests/test_instrumentor.py | 260 +++- .../tests/test_models.py | 1296 +++++++++++++++++ .../tests/test_utils.py | 222 +++ tox.ini | 4 + 25 files changed, 3340 insertions(+), 45 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py diff --git a/instrumentation/README.md b/instrumentation/README.md index 800e69732..83cf165a8 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -9,6 +9,6 @@ | [opentelemetry-instrumentation-genai-openai](./opentelemetry-instrumentation-genai-openai) | openai >= 1.26.0 | Yes | development | [opentelemetry-instrumentation-genai-openai-agents](./opentelemetry-instrumentation-genai-openai-agents) | openai-agents >= 0.3.3 | No | development | [opentelemetry-instrumentation-genai-qwen-agent](./opentelemetry-instrumentation-genai-qwen-agent) | qwen-agent >= 0.0.20 | No | development -| [opentelemetry-instrumentation-genai-smolagents](./opentelemetry-instrumentation-genai-smolagents) | smolagents >= 1.24.0 | No | development +| [opentelemetry-instrumentation-genai-smolagents](./opentelemetry-instrumentation-genai-smolagents) | smolagents >= 1.24.0 | Yes | development | [opentelemetry-instrumentation-genai-weaviate-client](./opentelemetry-instrumentation-genai-weaviate-client) | weaviate-client >= 3.0.0,<5.0.0 | No | development | [opentelemetry-instrumentation-google-genai](./opentelemetry-instrumentation-google-genai) | google-genai >= 1.32.0, <3 | No | development diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added new file mode 100644 index 000000000..5aca70172 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added @@ -0,0 +1 @@ +Add ``chat`` instrumentation for the smolagents model classes. diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst index 932ad2c12..fa97a7c54 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst @@ -7,7 +7,31 @@ OpenTelemetry smolagents Instrumentation :target: https://pypi.org/project/opentelemetry-instrumentation-genai-smolagents/ This library provides OpenTelemetry instrumentation for `smolagents -`_. +`_. It wraps the smolagents model +classes and emits a GenAI semantic-convention ``chat`` span and the matching +metrics through ``opentelemetry-util-genai``. + +Agent runs (``invoke_agent``) and tool calls (``execute_tool``) are not +instrumented yet. A model call made inside an agent run still gets a ``chat`` +span, but no agent span sits above it. + +A streamed model call, whether it comes from ``stream_outputs=True`` on an agent +or from calling ``Model.generate_stream`` directly, gets a ``chat`` span that +stays open until the caller drains the deltas. The span carries +``gen_ai.request.stream``, and the call also records the +``gen_ai.client.operation.time_to_first_chunk`` and +``gen_ai.client.operation.time_per_output_chunk`` metrics. + +Known gaps: + +* The instrumentation patches ``generate`` and ``generate_stream`` on the model + classes that smolagents ships. A subclass that inherits either method is + instrumented. A subclass that overrides one is not: the override shadows the + patched method, so the call produces no ``chat`` span. +* A streamed ``chat`` span reports no ``gen_ai.response.id`` and no + ``gen_ai.response.model``, because a smolagents stream delta carries neither. + The span reports ``gen_ai.response.finish_reasons`` only when the model + requested tool calls, which is the one stop reason the deltas make visible. Installation ------------ @@ -24,10 +48,13 @@ Usage from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import InferenceClientModel - # Instrument smolagents SmolagentsInstrumentor().instrument() + model = InferenceClientModel() + model.generate([{"role": "user", "content": "How many seconds are in a week?"}]) + Configuration ------------- @@ -71,6 +98,12 @@ environment variable: SmolagentsInstrumentor().instrument(completion_hook=my_hook) +Conformance +----------- + +The scenarios that check this package against the GenAI semantic conventions +live under ``tests/conformance/``. + References ---------- diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml index 9f7ed96ed..67140011a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "opentelemetry-api ~= 1.43", "opentelemetry-instrumentation >= 0.64b0, <1", "opentelemetry-semantic-conventions >= 0.64b0, <1", - "opentelemetry-util-genai >= 1.0b0, <2", + "opentelemetry-util-genai >= 1.1b0.dev, <2", ] [project.optional-dependencies] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index b3cffd043..4e14cda26 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -7,6 +7,9 @@ Instrumentation for `smolagents `_. +Model calls are recorded as ``chat`` spans. Agent runs and tool calls are not +instrumented yet. + Usage ----- @@ -15,10 +18,13 @@ from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import InferenceClientModel - # Enable instrumentation SmolagentsInstrumentor().instrument() + model = InferenceClientModel() + model.generate([{"role": "user", "content": "How many seconds are in a week?"}]) + Configuration ------------- @@ -38,20 +44,63 @@ from __future__ import annotations +from types import ModuleType from typing import Any, Collection +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.utils import unwrap from opentelemetry.util.genai.completion_hook import load_completion_hook from opentelemetry.util.genai.handler import TelemetryHandler from .package import _instruments +from .patch import model_generate, model_generate_stream __all__ = ["SmolagentsInstrumentor"] +def _model_classes_defining(smolagents: ModuleType, method: str) -> list[type]: + """The exported model classes whose ``method`` gets wrapped. + + Only classes that define ``method`` in their own ``__dict__`` are patched, + so a class that inherits it (``AzureOpenAIModel``, ``LiteLLMRouterModel``) + isn't wrapped a second time and can't produce duplicate ``chat`` spans. + A user-defined subclass that overrides the method shadows the patched base + method and emits no ``chat`` span; that limitation is documented in + ``README.rst``. + + Deduplicated by class object, because smolagents exports some classes under + two names (``OpenAIServerModel`` is ``OpenAIModel``) and wrapping the same + class twice would double every ``chat`` span. + """ + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + Model, + ) + + classes: dict[type, None] = {} + for obj in vars(smolagents).values(): + if ( + isinstance(obj, type) + and issubclass(obj, Model) + and method in obj.__dict__ + ): + classes.setdefault(obj, None) + return list(classes) + + class SmolagentsInstrumentor(BaseInstrumentor): """An instrumentor for smolagents.""" + # ``BaseInstrumentor.__new__`` returns a per-class singleton, but Python + # still runs ``__init__`` on every construction. Initializing this state in + # ``__init__`` would let the documented ``SmolagentsInstrumentor() + # .uninstrument()`` form wipe the live instance's bookkeeping and leave + # smolagents permanently patched, so these are class-level defaults that + # only ``_instrument`` / ``_uninstrument`` rebind. + _wrapped_generate_classes: list[type] = [] + _wrapped_generate_stream_classes: list[type] = [] + def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -65,15 +114,51 @@ def _instrument(self, **kwargs: Any) -> None: - logger_provider: LoggerProvider instance - completion_hook: CompletionHook instance """ - TelemetryHandler( + import smolagents # pylint: disable=import-outside-toplevel # noqa: PLC0415 + + handler = TelemetryHandler( tracer_provider=kwargs.get("tracer_provider"), meter_provider=kwargs.get("meter_provider"), logger_provider=kwargs.get("logger_provider"), completion_hook=kwargs.get("completion_hook") or load_completion_hook(), ) - # Patching will be added in follow-up PRs. + + wrapped_generate_classes: list[type] = [] + self._wrapped_generate_classes = wrapped_generate_classes + wrapped_generate_stream_classes: list[type] = [] + self._wrapped_generate_stream_classes = wrapped_generate_stream_classes + try: + for model_cls in _model_classes_defining(smolagents, "generate"): + wrap_function_wrapper( + model_cls, + "generate", + model_generate(handler), + ) + wrapped_generate_classes.append(model_cls) + + for model_cls in _model_classes_defining( + smolagents, "generate_stream" + ): + wrap_function_wrapper( + model_cls, + "generate_stream", + model_generate_stream(handler), + ) + wrapped_generate_stream_classes.append(model_cls) + except Exception: + # BaseInstrumentor.instrument() doesn't mark the instrumentor as + # instrumented when _instrument raises, so uninstrument() would + # refuse to run and leave the patches applied with no way to undo. + self._uninstrument() + raise def _uninstrument(self, **kwargs: Any) -> None: """Disable smolagents instrumentation and restore patched originals.""" - # Unpatching will be added in follow-up PRs. + for model_cls in self._wrapped_generate_classes: + unwrap(model_cls, "generate") + self._wrapped_generate_classes = [] + + for model_cls in self._wrapped_generate_stream_classes: + unwrap(model_cls, "generate_stream") + self._wrapped_generate_stream_classes = [] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py new file mode 100644 index 000000000..797021372 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -0,0 +1,316 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Convert smolagents message and tool shapes into util-genai GenAI types. + +smolagents passes ``generate(messages=...)`` a list of ``ChatMessage`` objects +or plain dicts, and returns a ``ChatMessage``. This module maps those, and the +``tools_to_call_from`` tool objects, onto the types in +``opentelemetry.util.genai.types``. util-genai then serializes them into +``gen_ai.input.messages``, ``gen_ai.output.messages``, and +``gen_ai.tool.definitions``. +""" + +from __future__ import annotations + +import base64 +import binascii +import logging +from collections.abc import Mapping +from enum import Enum +from typing import Any + +from opentelemetry.util.genai.types import ( + Blob, + FunctionToolDefinition, + InputMessage, + OutputMessage, + Reasoning, + Text, + ToolCallRequest, + ToolDefinition, + Uri, +) + +_logger = logging.getLogger(__name__) + +_DEFAULT_IMAGE_MIME_TYPE = "image/png" +_DATA_URL_PREFIX = "data:" + +# smolagents-internal roles -> semconv ``gen_ai`` message roles. smolagents +# applies the same mapping (``models.tool_role_conversions``) inside +# ``generate``, after the wrapper has already read ``messages``, so the wrapper +# has to apply it too. Without this map the input messages would carry roles +# the spec doesn't define. A model configured with ``custom_role_conversions`` +# overrides the mapping and can send different roles. Roles that are already +# spec values pass through unchanged. +_ROLE_MAP: dict[str, str] = { + "tool-call": "assistant", + "tool-response": "user", +} + +# Amazon Bedrock reports the stop reason as ``stopReason`` using Anthropic's +# vocabulary. Normalize it the way the anthropic instrumentation does +# (``anthropic/utils.py``); anything unmapped passes through. +_STOP_REASON_MAP: dict[str, str] = { + "end_turn": "stop", + "stop_sequence": "stop", + "max_tokens": "length", + "tool_use": "tool_calls", +} + + +def _unwrap_role(role: Any) -> str | None: + if role is None: + return None + if isinstance(role, Enum): + role = role.value + name = str(role) + return _ROLE_MAP.get(name, name) + + +def _decode_base64_image(image: str) -> tuple[bytes, str] | None: + """Decode a base64 payload or data URL into ``(bytes, mime_type)``.""" + mime_type = _DEFAULT_IMAGE_MIME_TYPE + if image.startswith(_DATA_URL_PREFIX): + header, _, image = image.partition(",") + media_type = header[len(_DATA_URL_PREFIX) :].split(";")[0] + if media_type: + mime_type = media_type + try: + return base64.b64decode(image, validate=True), mime_type + except (binascii.Error, ValueError): + _logger.debug("Failed to decode a base64 image", exc_info=True) + return None + + +def _encode_image_base64(image: Any) -> str | None: + try: + from smolagents.utils import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + encode_image_base64, + ) + except ImportError: + _logger.debug("smolagents.utils.encode_image_base64 is unavailable") + return None + try: + encoded = encode_image_base64(image) + except Exception: # pylint: disable=broad-except + _logger.debug( + "Failed to encode image of type %s, dropping it from telemetry", + type(image).__name__, + exc_info=True, + ) + return None + return encoded if isinstance(encoded, str) else None + + +def _image_blob(image: Any) -> Blob | None: + """Build a ``Blob`` part from a base64 string, data URL, or PIL image.""" + if isinstance(image, str): + decoded = _decode_base64_image(image) + else: + encoded = _encode_image_base64(image) + decoded = ( + _decode_base64_image(encoded) if encoded is not None else None + ) + if decoded is None: + return None + content, mime_type = decoded + return Blob(mime_type=mime_type, modality="image", content=content) + + +def _image_part_from_element(element: dict[str, Any]) -> Uri | Blob | None: + content_type = element.get("type") + if content_type == "image_url": + image_url = element.get("image_url") + url = image_url.get("url") if isinstance(image_url, dict) else None + if isinstance(url, str) and url: + return Uri(mime_type=None, modality="image", uri=url) + return None + if content_type == "image": + image = element.get("image") + if image is not None: + return _image_blob(image) + return None + + +def _parts_from_content(content: Any) -> list[Any]: + parts: list[Any] = [] + if isinstance(content, str): + parts.append(Text(content=content)) + return parts + if isinstance(content, list): + for element in content: + if not isinstance(element, dict): + _logger.debug( + "Unknown message content dropped from telemetry: %s", + type(element).__name__, + ) + continue + if element.get("type") == "text" and (text := element.get("text")): + parts.append(Text(content=text)) + continue + image_part = _image_part_from_element(element) + if image_part is not None: + parts.append(image_part) + else: + _logger.debug( + "Unknown message part dropped from telemetry: %s", + element.get("type"), + ) + return parts + + +def _get_role_and_content(message: Any) -> tuple[Any, Any]: + if isinstance(message, dict): + return message.get("role"), message.get("content") + return getattr(message, "role", None), getattr(message, "content", None) + + +def to_input_messages(messages: Any) -> list[InputMessage]: + """Map smolagents ``generate`` input messages to ``InputMessage`` objects.""" + result: list[InputMessage] = [] + if not isinstance(messages, list): + return result + for message in messages: + raw_role, content = _get_role_and_content(message) + role = _unwrap_role(raw_role) + if not role: + continue + result.append( + InputMessage(role=role, parts=_parts_from_content(content)) + ) + return result + + +def _raw_value(raw: Any, key: str) -> Any: + """Read ``key`` off a provider response that may be an object or a dict. + + ``ChatMessage.raw`` is whatever the provider handed back: an OpenAI-shaped + object for the API-backed models, and a dict for ``AmazonBedrockModel`` + (the boto3 ``converse`` response) and the local runtimes. + """ + if isinstance(raw, Mapping): + return raw.get(key) + return getattr(raw, key, None) + + +def _first_choice(output_message: Any) -> Any: + choices = _raw_value(getattr(output_message, "raw", None), "choices") + if isinstance(choices, list) and choices: + return choices[0] + return None + + +def _reasoning_from_raw(output_message: Any) -> str | None: + message = _raw_value(_first_choice(output_message), "message") + reasoning = _raw_value(message, "reasoning_content") + return reasoning if isinstance(reasoning, str) and reasoning else None + + +def _tool_call_requests(output_message: Any) -> list[ToolCallRequest]: + # ChatMessage.__post_init__ coerces every entry into a + # ChatMessageToolCall, so the id/function/name/arguments are all present. + tool_calls = getattr(output_message, "tool_calls", None) or [] + return [ + ToolCallRequest( + name=tool_call.function.name, + id=tool_call.id, + arguments=tool_call.function.arguments, + ) + for tool_call in tool_calls + ] + + +def _finish_reason(output_message: Any, has_tool_calls: bool) -> str | None: + """Why the provider stopped generating, or ``None`` if it didn't say. + + The local runtimes (``TransformersModel``, ``VLLMModel``, ``MLXModel``) put + ``{"out": ..., "completion_kwargs": ...}`` on ``raw`` and report no finish + reason at all. Defaulting those to ``"stop"`` would make a generation cut + short by ``max_new_tokens`` look like a natural stop. A response carrying + tool calls is the one case where the reason follows without guessing. + """ + reason = _raw_value(_first_choice(output_message), "finish_reason") + if isinstance(reason, str) and reason: + return reason + raw = getattr(output_message, "raw", None) + stop_reason = _raw_value(raw, "stopReason") + if isinstance(stop_reason, str) and stop_reason: + return _STOP_REASON_MAP.get(stop_reason, stop_reason) + return "tool_calls" if has_tool_calls else None + + +def to_output_message(output_message: Any) -> OutputMessage: + """Map a smolagents ``ChatMessage`` response to an ``OutputMessage``.""" + role = _unwrap_role(getattr(output_message, "role", None)) or "assistant" + parts: list[Any] = _parts_from_content( + getattr(output_message, "content", None) + ) + if reasoning := _reasoning_from_raw(output_message): + parts.append(Reasoning(content=reasoning)) + tool_call_requests = _tool_call_requests(output_message) + parts.extend(tool_call_requests) + # OutputMessage requires the field; util-genai drops an empty value when it + # emits gen_ai.response.finish_reasons. + reason = _finish_reason(output_message, bool(tool_call_requests)) + return OutputMessage(role=role, parts=parts, finish_reason=reason or "") + + +def response_id(output_message: Any) -> str | None: + """Extract ``gen_ai.response.id`` from the provider response on ``.raw``.""" + value = _raw_value(getattr(output_message, "raw", None), "id") + return value if isinstance(value, str) and value else None + + +def response_model_name(output_message: Any) -> str | None: + """Extract ``gen_ai.response.model`` from the provider response on ``.raw``.""" + value = _raw_value(getattr(output_message, "raw", None), "model") + return value if isinstance(value, str) and value else None + + +def _tool_parameters(tool: Any) -> dict[str, Any] | None: + """Return the JSON Schema ``parameters`` object for a smolagents tool. + + A tool's ``inputs`` map is not a JSON Schema on its own: smolagents wraps it + in an object schema, derives ``required`` from ``nullable``, and rewrites its + non-JSON-Schema ``"any"`` type. ``get_tool_json_schema`` builds exactly the + schema the provider receives. + """ + try: + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + get_tool_json_schema, + ) + except ImportError: + _logger.debug("smolagents.models.get_tool_json_schema is unavailable") + return None + try: + schema = get_tool_json_schema(tool) + parameters = schema["function"]["parameters"] + except Exception: # pylint: disable=broad-except + _logger.debug( + "Failed to build a JSON Schema for tool %s", + getattr(tool, "name", None), + exc_info=True, + ) + return None + return parameters if isinstance(parameters, dict) else None + + +def to_tool_definitions(tools: Any) -> list[ToolDefinition] | None: + """Map smolagents tool objects to function tool definitions.""" + if not isinstance(tools, list) or not tools: + return None + definitions: list[ToolDefinition] = [] + for tool in tools: + name = getattr(tool, "name", None) + if not name: + continue + definitions.append( + FunctionToolDefinition( + name=name, + description=getattr(tool, "description", None), + parameters=_tool_parameters(tool), + ) + ) + return definitions or None diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py index b56883424..fa7223304 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py @@ -2,3 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 _instruments = ("smolagents >= 1.24.0",) + +_supports_metrics = True diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py new file mode 100644 index 000000000..4653108aa --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -0,0 +1,436 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""wrapt wrapper factories for smolagents instrumentation. + +Each factory takes the shared :class:`TelemetryHandler` and returns a wrapper +suitable for :func:`wrapt.wrap_function_wrapper`: + +- :func:`model_generate` wraps each defining ``Model.generate`` -> ``chat`` span. +- :func:`model_generate_stream` wraps each defining ``Model.generate_stream`` + -> ``chat`` span, held open until the stream is drained. + +Original library exceptions are always re-raised unmodified; telemetry is +finalized via ``invocation.stop()`` / ``invocation.fail(exc)``. +""" + +from __future__ import annotations + +import logging +from collections.abc import Generator, Mapping +from dataclasses import dataclass +from inspect import signature +from typing import Any, Callable + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import InferenceInvocation +from opentelemetry.util.genai.stream import SyncStreamWrapper +from opentelemetry.util.genai.types import OutputMessage, Text, ToolCallRequest + +from ._messages import ( + response_id, + response_model_name, + to_input_messages, + to_output_message, + to_tool_definitions, +) +from .provider import resolve_provider, resolve_server_address_port + +_logger = logging.getLogger(__name__) + +_Wrapper = Callable[ + [Callable[..., Any], Any, tuple[Any, ...], dict[str, Any]], Any +] + + +def _bind_arguments( + wrapped: Callable[..., Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> dict[str, Any]: + """Bind call args to the wrapped callable's signature, applying defaults. + + smolagents passes the interesting arguments positionally + (``model.generate(input_messages)``), so binding is what makes them + readable by name. On a binding failure the keyword arguments are returned + on their own, without the positional ones and without the defaults. + """ + try: + bound = signature(wrapped).bind(*args, **kwargs) + bound.apply_defaults() + return dict(bound.arguments) + except (TypeError, ValueError): + _logger.debug( + "Failed to bind arguments of %s; falling back to keyword arguments", + getattr(wrapped, "__qualname__", wrapped), + exc_info=True, + ) + return dict(kwargs) + + +def _coerce_float(value: Any) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return None + + +def _coerce_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + return None + + +def _remove_parameter_sentinel() -> Any: + """Return smolagents' ``REMOVE_PARAMETER`` sentinel, or ``None`` if absent.""" + try: + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + REMOVE_PARAMETER, + ) + except ImportError: + _logger.debug("smolagents.models.REMOVE_PARAMETER is unavailable") + return None + return REMOVE_PARAMETER + + +# Model classes that never forward ``stop_sequences``, whatever +# ``supports_stop_parameter`` answers. ``AmazonBedrockModel`` overrides +# ``_prepare_completion_kwargs`` and calls the base with a hardcoded +# ``stop_sequences=None`` (``models.py``), so its ``converse`` request carries +# no stop sequences at all. ``_forwards_stop_sequences`` matches these names +# along the MRO, so subclasses are covered too. +_MODELS_DROPPING_STOP_SEQUENCES = frozenset({"AmazonBedrockModel"}) + + +def _forwards_stop_sequences(instance: Any) -> bool: + if any( + cls.__name__ in _MODELS_DROPPING_STOP_SEQUENCES + for cls in type(instance).__mro__ + ): + return False + return bool(getattr(instance, "supports_stop_parameter", False)) + + +def _merged_request_kwargs( + instance: Any, bound: dict[str, Any] +) -> dict[str, Any]: + """Rebuild the request keyword arguments smolagents will send. + + ``Model._prepare_completion_kwargs`` seeds ``stop`` from the + ``stop_sequences`` argument and ``response_format`` from its own argument, + applies the per-call ``**kwargs`` on top and the model-level ``self.kwargs`` + last, so a key set at two levels reaches the provider with the model-level + value and a model-level ``REMOVE_PARAMETER`` drops it from the request + entirely. Following that order here is what keeps a removed key off the span + as well. + """ + merged: dict[str, Any] = {} + stop_sequences = bound.get("stop_sequences") + if stop_sequences is not None and _forwards_stop_sequences(instance): + merged["stop"] = stop_sequences + response_format = bound.get("response_format") + if response_format is not None: + merged["response_format"] = response_format + call_kwargs = bound.get("kwargs") + if isinstance(call_kwargs, dict): + merged.update(call_kwargs) + model_kwargs = getattr(instance, "kwargs", None) + if not isinstance(model_kwargs, dict): + return merged + remove = _remove_parameter_sentinel() + for name, value in model_kwargs.items(): + if remove is not None and value is remove: + merged.pop(name, None) + else: + merged[name] = value + return merged + + +def _stop_sequences(merged: dict[str, Any]) -> list[str] | None: + """Return the stop sequences the request carries, if any.""" + stop = merged.get("stop", merged.get("stop_sequences")) + if isinstance(stop, str): + return [stop] + if isinstance(stop, list): + return [str(item) for item in stop] + return None + + +# smolagents' ``response_format`` type -> ``gen_ai.output.type`` value, for the +# types whose provider spelling differs from the semconv one. The openai +# instrumentation maps the same two. +_OUTPUT_TYPE_MAP: dict[str, str] = { + "json_object": GenAI.GenAiOutputTypeValues.JSON.value, + "json_schema": GenAI.GenAiOutputTypeValues.JSON.value, +} + +_OUTPUT_TYPE_VALUES = frozenset( + value.value for value in GenAI.GenAiOutputTypeValues +) + + +def _output_type(merged: dict[str, Any]) -> str | None: + """Map the request's ``response_format`` to ``gen_ai.output.type``. + + smolagents forwards ``response_format`` to the provider unchanged, so its + ``type`` is whatever the provider accepts. Only the values the semconv + defines are recorded; a provider-specific one is dropped rather than put on + an enum attribute. + """ + response_format = merged.get("response_format") + if not isinstance(response_format, Mapping): + return None + format_type = response_format.get("type") + if not isinstance(format_type, str): + return None + output_type = _OUTPUT_TYPE_MAP.get(format_type, format_type) + if output_type not in _OUTPUT_TYPE_VALUES: + _logger.debug( + "No gen_ai.output.type value for response_format type %r", + format_type, + ) + return None + return output_type + + +def _apply_request_parameters( + invocation: InferenceInvocation, instance: Any, bound: dict[str, Any] +) -> None: + """Copy the request parameters smolagents will send onto the span.""" + merged = _merged_request_kwargs(instance, bound) + + invocation.temperature = _coerce_float(merged.get("temperature")) + invocation.top_p = _coerce_float(merged.get("top_p")) + invocation.top_k = _coerce_float(merged.get("top_k")) + invocation.frequency_penalty = _coerce_float( + merged.get("frequency_penalty") + ) + invocation.presence_penalty = _coerce_float(merged.get("presence_penalty")) + # TransformersModel takes the generation limit as max_new_tokens (which its + # constructor defaults to 4096) and treats max_tokens as an alias for it. + invocation.max_tokens = _coerce_int( + merged.get("max_tokens", merged.get("max_new_tokens")) + ) + invocation.seed = _coerce_int(merged.get("seed")) + invocation.stop_sequences = _stop_sequences(merged) + invocation.output_type = _output_type(merged) + + +def _apply_token_usage( + invocation: InferenceInvocation, output_message: Any +) -> None: + # ChatMessage.token_usage is the only source: the per-model + # last_input_token_count / last_output_token_count counters were removed + # before the oldest supported smolagents. It is None for the local runtimes + # (TransformersModel, VLLMModel, MLXModel), which report no usage. + token_usage = getattr(output_message, "token_usage", None) + if token_usage is None: + return + invocation.input_tokens = token_usage.input_tokens + invocation.output_tokens = token_usage.output_tokens + + +def _start_inference( + handler: TelemetryHandler, + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> InferenceInvocation: + """Start the ``chat`` span and record the request. + + ``generate`` and ``generate_stream`` take the same parameters. + """ + provider = resolve_provider(instance) + server_address, server_port = resolve_server_address_port(instance) + invocation = handler.inference( + provider, + request_model=getattr(instance, "model_id", None), + server_address=server_address, + server_port=server_port, + ) + bound = _bind_arguments(wrapped, args, kwargs) + _apply_request_parameters(invocation, instance, bound) + invocation.tool_definitions = to_tool_definitions( + bound.get("tools_to_call_from") + ) + if handler.should_capture_content(): + invocation.input_messages = to_input_messages(bound.get("messages")) + return invocation + + +def model_generate(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate`` to emit a ``chat`` span.""" + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + + try: + output_message = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + + _apply_token_usage(invocation, output_message) + invocation.response_model_name = response_model_name(output_message) + invocation.response_id = response_id(output_message) + output = to_output_message(output_message) + # to_output_message leaves finish_reason empty when the provider + # reported none, and an empty value is dropped rather than guessed at. + if output.finish_reason: + invocation.finish_reasons = [output.finish_reason] + if handler.should_capture_content(): + invocation.output_messages = [output] + invocation.stop() + return output_message + + return wrapper + + +@dataclass +class _StreamedToolCall: + """A tool call assembled from stream deltas.""" + + id: str | None = None + name: str = "" + arguments: str = "" + + +class _ModelStreamWrapper(SyncStreamWrapper[Any]): + """Keep the ``chat`` span open until the delta stream is drained. + + Passing the invocation to ``super().__init__()`` turns on + ``gen_ai.request.stream`` and the per-chunk timing metrics. + """ + + def __init__( + self, + stream: Generator[Any, Any, Any], + invocation: InferenceInvocation, + handler: TelemetryHandler, + ) -> None: + super().__init__(stream, invocation=invocation) + self._self_inference = invocation + self._self_handler = handler + self._self_content: list[str] = [] + self._self_tool_calls: dict[int, _StreamedToolCall] = {} + self._self_input_tokens = 0 + self._self_output_tokens = 0 + self._self_saw_token_usage = False + + def _accumulate_tool_call(self, delta: Any) -> None: + index = getattr(delta, "index", None) + if not isinstance(index, int): + # agglomerate_stream_deltas raises here; telemetry must not. + _logger.debug("Dropping a tool call delta that carries no index") + return + tool_call = self._self_tool_calls.setdefault( + index, _StreamedToolCall() + ) + if delta.id: + tool_call.id = delta.id + function = getattr(delta, "function", None) + if function is None: + return + if function.name: + tool_call.name = function.name + if function.arguments: + tool_call.arguments += function.arguments + + def _process_chunk(self, chunk: Any) -> None: + content = getattr(chunk, "content", None) + if content: + self._self_content.append(content) + token_usage = getattr(chunk, "token_usage", None) + if token_usage is not None: + # Summed like agglomerate_stream_deltas, so the span agrees with the + # totals the agent's monitor reports. + self._self_saw_token_usage = True + self._self_input_tokens += token_usage.input_tokens + self._self_output_tokens += token_usage.output_tokens + for delta in getattr(chunk, "tool_calls", None) or []: + self._accumulate_tool_call(delta) + + def _output_message(self) -> OutputMessage | None: + parts: list[Any] = [] + content = "".join(self._self_content) + if content: + parts.append(Text(content=content)) + parts.extend( + ToolCallRequest( + name=tool_call.name, + id=tool_call.id, + arguments=tool_call.arguments or None, + ) + for tool_call in self._self_tool_calls.values() + ) + if not parts: + # Closed before it was drained, so there is no response to report. + return None + # Deltas carry no finish reason, so tool calls are the only evidence. + # Defaulting to "stop" would hide a generation cut short by a token + # limit. + finish_reason = "tool_calls" if self._self_tool_calls else "" + return OutputMessage( + role="assistant", parts=parts, finish_reason=finish_reason + ) + + def _finalize(self, error: BaseException | None = None) -> None: + invocation = self._self_inference + if self._self_saw_token_usage: + invocation.input_tokens = self._self_input_tokens + invocation.output_tokens = self._self_output_tokens + output = self._output_message() + if output is not None: + if output.finish_reason: + invocation.finish_reasons = [output.finish_reason] + if self._self_handler.should_capture_content(): + invocation.output_messages = [output] + if error is not None: + invocation.fail(error) + else: + invocation.stop() + + def _on_stream_end(self) -> None: + self._finalize() + + def _on_stream_error(self, error: BaseException) -> None: + # Records what was streamed before the failure. + self._finalize(error) + + +def model_generate_stream(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate_stream`` to emit a ``chat`` span. + + ``stream_outputs=True`` routes an agent's model calls here. The span stays + open until the caller drains the deltas. + """ + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + + try: + stream = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + + return _ModelStreamWrapper(stream, invocation, handler) + + return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py new file mode 100644 index 000000000..f531606c1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py @@ -0,0 +1,151 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve a smolagents model instance to a ``gen_ai.provider.name`` value. + +Resolution order: + +1. For the LiteLLM model classes, the ``model_id`` vendor prefix + (``anthropic/claude-...`` -> ``anthropic``). +2. The model class name (e.g. ``OpenAIModel`` -> ``openai``), looked up along the + class hierarchy so a user subclass resolves to the provider of the base class + whose ``generate`` it inherits. +3. ``unknown``. + +``gen_ai.provider.name`` is a metric attribute as well as a span attribute, so +every value has to stay low cardinality: deployment-specific detail is reported +as ``server.address`` instead. ``TelemetryHandler.inference`` requires +``provider`` as a string, so this always returns a value rather than ``None``. +""" + +from __future__ import annotations + +import logging +from typing import Any +from urllib.parse import urlparse + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + +_logger = logging.getLogger(__name__) + +_PROVIDER = GenAI.GenAiProviderNameValues + +_UNKNOWN_PROVIDER = "unknown" + +# Model class name -> provider value. The GenAI registry has no value for the +# Hugging Face, vLLM, and MLX runtimes, so those use the product name; a class +# name would look like a provider without being one. +_CLASS_NAME_TO_PROVIDER: dict[str, str] = { + "OpenAIModel": _PROVIDER.OPENAI.value, + "AzureOpenAIModel": _PROVIDER.AZURE_AI_OPENAI.value, + "AmazonBedrockModel": _PROVIDER.AWS_BEDROCK.value, + "InferenceClientModel": "huggingface", + "TransformersModel": "huggingface", + "VLLMModel": "vllm", + "MLXModel": "mlx", +} + +# LiteLLM model_id prefix -> semconv provider value, for the prefixes whose +# LiteLLM vendor slug differs from the semconv value. Every other prefix is +# passed through as-is (``ollama/llama3`` -> ``ollama``). LiteLLM's slugs are a +# closed vocabulary, which keeps the cardinality bounded. +_LITELLM_PREFIX_TO_PROVIDER: dict[str, str] = { + "azure": _PROVIDER.AZURE_AI_OPENAI.value, + "azure_ai": _PROVIDER.AZURE_AI_INFERENCE.value, + "bedrock": _PROVIDER.AWS_BEDROCK.value, + "gemini": _PROVIDER.GCP_GEMINI.value, + "mistral": _PROVIDER.MISTRAL_AI.value, + "vertex_ai": _PROVIDER.GCP_VERTEX_AI.value, + "watsonx": _PROVIDER.IBM_WATSONX_AI.value, + "xai": _PROVIDER.X_AI.value, +} + +_LITELLM_CLASS_NAMES = frozenset({"LiteLLMModel", "LiteLLMRouterModel"}) + + +def _class_names(instance: Any) -> list[str]: + """The instance's class names, most derived first. + + Only the classes that define ``generate`` are patched, so an instrumented + model can be a user subclass of one of them. Matching the exact class name + alone would report ``unknown`` for every such subclass. + """ + return [cls.__name__ for cls in type(instance).__mro__] + + +def _provider_from_litellm(instance: Any) -> str | None: + model_id = getattr(instance, "model_id", None) + if not isinstance(model_id, str) or "/" not in model_id: + return None + prefix = model_id.split("/", 1)[0].lower() + return _LITELLM_PREFIX_TO_PROVIDER.get(prefix, prefix) + + +def _endpoint(instance: Any) -> str | None: + """The endpoint URL the model calls, or ``None`` if it exposes none. + + Three places can hold it, checked in this order: + + 1. ``api_base`` on the instance (``LiteLLMModel``). + 2. ``base_url`` or ``azure_endpoint`` in the SDK client kwargs + (``OpenAIModel``, ``AzureOpenAIModel``, ``InferenceClientModel``). + 3. ``base_url`` on the client the model constructed (e.g. openai's httpx + URL), which holds the effective URL when the caller left it at the + provider default. + """ + api_base = getattr(instance, "api_base", None) + if api_base: + return str(api_base) + client_kwargs = getattr(instance, "client_kwargs", None) + if isinstance(client_kwargs, dict): + for key in ("base_url", "azure_endpoint"): + value = client_kwargs.get(key) + if value: + return str(value) + client_base_url = getattr( + getattr(instance, "client", None), "base_url", None + ) + if client_base_url: + return str(client_base_url) + return None + + +def resolve_server_address_port( + instance: Any, +) -> tuple[str | None, int | None]: + """Return ``(server.address, server.port)`` from the model's endpoint URL. + + Models that don't expose an ``api_base`` / ``base_url`` / ``azure_endpoint`` + (e.g. ``LiteLLMModel`` resolving the host internally, local runtimes) + yield ``(None, None)`` and the caller omits the attributes. + """ + endpoint = _endpoint(instance) + if endpoint is None: + return None, None + parsed = urlparse(endpoint) + port = parsed.port + if port == 443: + port = None + return parsed.hostname or None, port + + +def resolve_provider(instance: Any) -> str: + """Return the ``gen_ai.provider.name`` value for a smolagents model instance.""" + class_names = _class_names(instance) + + if not _LITELLM_CLASS_NAMES.isdisjoint(class_names): + provider = _provider_from_litellm(instance) + if provider is not None: + return provider + + for class_name in class_names: + provider = _CLASS_NAME_TO_PROVIDER.get(class_name) + if provider is not None: + return provider + + _logger.debug( + "No known gen_ai.provider.name for model class %s", class_names[0] + ) + return _UNKNOWN_PROVIDER diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml new file mode 100644 index 000000000..6177ff059 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml @@ -0,0 +1,20 @@ +interactions: +- request: + body: '{"model": "claude-3-7-sonnet-20250219", "messages": [{"role": "user", "content": + [{"type": "text", "text": "Who won the World Cup in 2018? Answer in one word + with no punctuation."}]}], "thinking": {"type": "enabled", "budget_tokens": + 4000}, "max_tokens": 8096}' + headers: {} + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"id":"msg_011KX7d4TtALbugymC3Kb4oE","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[{"type":"thinking","thinking":"The + World Cup in 2018 was won by France. They defeated Croatia 4-2 in the final + match in Moscow, Russia.\n\nI need to answer in one word with no punctuation, + so my answer should simply be:\nFrance","signature":"ErUBCkYIBBgCIkAOLwDXty2UNgzsRPd4O0tNxqhaxPqqLw9isc2bDqyVS7Y87Cefkib8FVE9iTTjTSynEjrrHb+vei4K5pQrVOUrEgzCGfpi49gURMlFYTwaDGTEJyP22/PoNkje2CIwyiqcZqRj6rxDwzG0ayl3JQ2mi8x3iDIEuyP92Pw19EPRhATbpYiORzbaVoPUBuPEKh3gUYNSZwcjUAYNO01Qqo7GYFF08xO0MuULheotLRgC"},{"type":"text","text":"France"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":54,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":65,"service_tier":"standard"}}' + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml new file mode 100644 index 000000000..44f70cc65 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml @@ -0,0 +1,25 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"Who won the + World Cup in 2018? Answer in one word with no punctuation."}]}],"model":"gpt-4o","max_tokens":4096}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb\",\n \"object\": + \"chat.completion\",\n \"created\": 1738649686,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"France\",\n \"refusal\": null\n + \ },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n + \ ],\n \"usage\": {\n \"prompt_tokens\": 25,\n \"completion_tokens\": + 2,\n \"total_tokens\": 27,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_4691090a87\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml new file mode 100644 index 000000000..9fde9a56e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml @@ -0,0 +1,26 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What breed + is this dog?"},{"type":"image_url","image_url":{"url":"https://fastly.picsum.photos/id/237/200/300.jpg?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U"}}]}],"model":"gpt-4o"}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-DVJBfdzKCrLtPYM9ui8SZgIEQv857\",\n \"object\": + \"chat.completion\",\n \"created\": 1776354295,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"This looks like a Labrador Retriever + puppy. They are known for their friendly and outgoing nature.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 268,\n \"completion_tokens\": 18,\n \"total_tokens\": 286,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_07a5e8f420\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml new file mode 100644 index 000000000..a5a57c069 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml @@ -0,0 +1,32 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What is the + weather in Paris?"}]}],"model":"gpt-4o","max_tokens":4096,"tool_choice":"required","tools": + [{"type":"function","function":{"name":"get_weather","description":"Get the weather for a + given city","parameters":{"type":"object","properties":{"location":{"type":"string","description": + "The city to get the weather for"}},"required":["location"]}}}]}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ax7BZfEQe2evzgqbTVXWo0ZqoMIUf\",\n \"object\": + \"chat.completion\",\n \"created\": 1738652337,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_EUuviydGIG5Jau3DLw4v4cue\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n + \ \"arguments\": \"{\\\"location\\\":\\\"Paris\\\"}\"\n }\n + \ }\n ],\n \"refusal\": null\n },\n \"logprobs\": + null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": + {\n \"prompt_tokens\": 61,\n \"completion_tokens\": 15,\n \"total_tokens\": + 76,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n + \ \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_50cad350e4\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py new file mode 100644 index 000000000..be0d0071e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py @@ -0,0 +1,34 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for smolagents conformance scenarios.""" + +from __future__ import annotations + +import json +from typing import Any + + +def attr(span: dict[str, Any], name: str) -> Any: + for entry in span["attributes"]: + if entry["name"] == name: + return entry["value"] + return None + + +def chat_spans(report: Any) -> list[dict[str, Any]]: + return [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + + +def part_fields(messages_json: str | None) -> list[tuple[str, str | None]]: + messages = json.loads(messages_json) if messages_json else [] + return [ + (part["type"], part.get("modality")) + for message in messages + for part in message["parts"] + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py new file mode 100644 index 000000000..f9e60fb0f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py @@ -0,0 +1,113 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios for the ``chat`` operation (plain and tool-calling).""" + +from __future__ import annotations + +from typing import Any + +from smolagents.models import ChatMessage, MessageRole + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +from ..test_utils import GetWeatherTool, openai_model # noqa: TID252 +from ._helpers import attr, chat_spans, part_fields + + +class ChatScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_basic.yaml"): + openai_model().generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? Answer " + "in one word with no punctuation." + ), + } + ], + } + ] + ) + + +class ToolCallingScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_tool.yaml"): + openai_model().generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What is the weather in Paris?", + } + ], + ) + ], + tools_to_call_from=[GetWeatherTool()], + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_part_types = { + part_type + for span in chat_spans(report) + for part_type, _ in part_fields( + attr(span, "gen_ai.output.messages") + ) + } + assert "tool_call" in output_part_types, ( + f"expected a tool_call output part, saw {output_part_types}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py new file mode 100644 index 000000000..5e8025c1b --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py @@ -0,0 +1,163 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios for non-text message parts: an image ``uri`` on a chat +input, and a ``reasoning`` part on a chat output.""" + +from __future__ import annotations + +import os +from typing import Any +from unittest import mock + +from smolagents import LiteLLMModel, OpenAIModel +from smolagents.models import ChatMessage, MessageRole + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) +from opentelemetry.test_util_genai.instrumentor import instrument + +from ._helpers import attr, chat_spans, part_fields + +_IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +class MultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_image_url.yaml"): + model = OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What breed is this dog?", + }, + { + "type": "image_url", + "image_url": {"url": _IMAGE_URL}, + }, + ], + ) + ] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + input_parts = { + fields + for span in chat_spans(report) + for fields in part_fields(attr(span, "gen_ai.input.messages")) + } + assert ("uri", "image") in input_parts, ( + f"expected an image uri input part, saw {input_parts}" + ) + + +class ReasoningScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = ( + # LiteLLM routes to the provider host internally and a LiteLLMModel + # built without an explicit api_base exposes no endpoint URL, so there + # is nothing to derive server.address from on the chat span. + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + env = {"LITELLM_LOCAL_MODEL_COST_MAP": "True"} + with ( + mock.patch.dict(os.environ, env), + mock.patch("tiktoken.get_encoding") as get_encoding, + ): + get_encoding.return_value = mock.MagicMock( + encode=lambda *_: [1, 2, 3] + ) + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("litellm_reasoning.yaml"): + model = LiteLLMModel( + model_id="anthropic/claude-3-7-sonnet-20250219", + api_key="test_anthropic_api_key", + thinking={"type": "enabled", "budget_tokens": 4000}, + ) + model.generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? " + "Answer in one word with no " + "punctuation." + ), + } + ], + } + ] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_parts = { + part_type + for span in chat_spans(report) + for part_type, _ in part_fields( + attr(span, "gen_ai.output.messages") + ) + } + assert "reasoning" in output_parts, ( + f"expected a reasoning output part, saw {output_parts}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py index da97f2987..baebcbf9c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py @@ -5,22 +5,103 @@ from __future__ import annotations +import os +from unittest.mock import MagicMock, patch + import pytest from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) from opentelemetry.test_util_genai.instrumentor import instrument +from opentelemetry.test_util_genai.vcr import ( + scrub_response_headers_overwrite, +) + +pytest_plugins = [ + "opentelemetry.test_util_genai.fixtures", + "opentelemetry.test_util_genai.vcr", +] + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_openai_api_key"), + ("x-api-key", "test_anthropic_api_key"), + ("openai-organization", "test_openai_org_id"), + ("openai-project", "test_openai_project_id"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers_overwrite( + { + "openai-organization": "test_openai_org_id", + "openai-project": "test_openai_project_id", + "Set-Cookie": "test_set_cookie", + } + ), + } + + +@pytest.fixture +def litellm_local_cost_map(): + """Use LiteLLM's bundled model-cost map so it doesn't fetch prices over the + network during cassette playback.""" + previous = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + try: + yield + finally: + if previous is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = previous + + +@pytest.fixture +def patch_tiktoken_encoding(): + """Patch ``tiktoken.get_encoding`` so LiteLLM doesn't download an encoding.""" + with patch("tiktoken.get_encoding") as mock_get_encoding: + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + mock_get_encoding.return_value = mock_encoding + yield + + +@pytest.fixture +def instrument_no_content(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="NO_CONTENT", + ) as instrumentor: + yield instrumentor -pytest_plugins = ["opentelemetry.test_util_genai.fixtures"] + +@pytest.fixture +def instrument_with_content(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ) as instrumentor: + yield instrumentor @pytest.fixture -def instrument_smolagents(tracer_provider, logger_provider, meter_provider): +def instrument_event_only(tracer_provider, logger_provider, meter_provider): with instrument( SmolagentsInstrumentor(), tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, + content_capture="EVENT_ONLY", + emit_event=True, ) as instrumentor: yield instrumentor diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt index d833435b7..649ffc343 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt @@ -26,7 +26,10 @@ # This variant of the requirements aims to test the system using the newest # supported version of external dependencies. -smolagents +# openai/litellm are the model backends exercised by the VCR model tests; they +# are test-only (not declared in pyproject.toml) so they live here. +smolagents[openai,litellm] +wrapt>=2.2.2 -e util/opentelemetry-util-genai -e instrumentation/opentelemetry-instrumentation-genai-smolagents diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt index 1d9b353f3..afb682e96 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt @@ -21,5 +21,15 @@ # pyproject.toml is the single source of truth. The OpenTelemetry SDK and test utilities # come transitively from opentelemetry-test-util-genai. # -# There is nothing to pin yet: the lifecycle tests need no model backend. The pins for -# the backends the model tests drive arrive with those tests. +# openai and litellm are the model backends the VCR model tests instantiate. They are +# test-only (not declared in pyproject.toml) and smolagents does not pull them in through +# the instruments extra, so pin them here to the versions smolagents 1.24.0 declares in its +# own openai/litellm extras. litellm resolves to its floor; openai resolves to 1.61.0 +# rather than 1.58.1 because litellm 1.60.2 itself requires openai>=1.61.0. +openai>=1.58.1 +litellm>=1.60.2 + +# The declared opentelemetry-util-genai floor carries the streaming metrics and the +# gen_ai.request.stream attribute, which are not in a published release yet. +# Drop this once opentelemetry-util-genai 1.1b0 is published. +-e util/opentelemetry-util-genai diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py new file mode 100644 index 000000000..41c60d2db --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py @@ -0,0 +1,46 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-scenario conformance tests for smolagents.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +# Skip collection when weaver_live_check or OTLP exporters aren't installed +# (non-conformance envs). +pytest.importorskip("opentelemetry.test.weaver_live_check") +pytest.importorskip("opentelemetry.exporter.otlp.proto.grpc") + +from opentelemetry.test.weaver_live_check import WeaverLiveCheck # noqa: E402 +from opentelemetry.test_util_genai.conformance import ( # noqa: E402 + Scenario, + run_conformance, +) + +from .conformance.inference import ( # noqa: E402 + ChatScenario, + ToolCallingScenario, +) +from .conformance.multimodal import ( # noqa: E402 + MultimodalScenario, + ReasoningScenario, +) + + +@pytest.mark.parametrize( + "scenario", + [ + pytest.param(ChatScenario()), + pytest.param(ToolCallingScenario()), + pytest.param(MultimodalScenario()), + pytest.param(ReasoningScenario()), + ], + ids=lambda s: type(s).__name__, +) +def test_conformance( + scenario: Scenario, vcr: Any, weaver_live_check: WeaverLiveCheck +) -> None: + run_conformance(scenario, vcr=vcr, weaver=weaver_live_check) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py index 1f4b69905..d0c654181 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py @@ -1,19 +1,40 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Entry-point and instrument/uninstrument lifecycle tests. - -The instrumentor patches nothing yet, so these cover the parts of its contract -that hold before any patching lands: the entry point resolves, the declared -dependency is reported, and repeated instrument/uninstrument cycles stay quiet. -""" +"""Lifecycle, entry-point, completion-hook, and restoration tests.""" from __future__ import annotations +from typing import Any +from unittest.mock import patch + +import pytest +import smolagents +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, + _model_classes_defining, ) +from opentelemetry.test_util_genai.instrumentor import instrument from opentelemetry.util._importlib_metadata import entry_points +from opentelemetry.util.genai.completion_hook import CompletionHook + +from .test_utils import openai_model, stub_openai_client + + +class RecordingHook(CompletionHook): + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def on_completion(self, **kwargs: Any) -> None: + self.calls.append(kwargs) + + +def _generate_a_chat_span() -> None: + model = openai_model() + model.client = stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) def test_entrypoint_loads_instrumentor() -> None: @@ -29,45 +50,42 @@ def test_instrumentation_dependencies() -> None: assert "smolagents >= 1.24.0" in dependencies -def test_instrument_uninstrument_cycle( +def test_instrument_uninstrument_restores_originals( tracer_provider, logger_provider, meter_provider ) -> None: + original_generate = smolagents.OpenAIModel.generate + original_base_generate = smolagents.Model.generate + original_generate_stream = smolagents.OpenAIModel.generate_stream + instrumentor = SmolagentsInstrumentor() instrumentor.instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, ) - assert instrumentor.is_instrumented_by_opentelemetry - instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is not original_generate + assert smolagents.Model.generate is not original_base_generate + assert ( + smolagents.OpenAIModel.generate_stream is not original_generate_stream + ) + instrumentor.uninstrument() -def test_repeated_instrument_uninstrument( - tracer_provider, logger_provider, meter_provider -) -> None: - # BaseInstrumentor returns a per-class singleton, so the lifecycle has to - # survive being driven more than once. - instrumentor = SmolagentsInstrumentor() - for _ in range(2): - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - assert instrumentor.is_instrumented_by_opentelemetry - instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.Model.generate is original_base_generate + assert smolagents.OpenAIModel.generate_stream is original_generate_stream def test_uninstrument_through_a_new_constructor_call( tracer_provider, logger_provider, meter_provider ) -> None: - # BaseInstrumentor.__new__ returns a per-class singleton but Python still - # runs __init__ on every construction, so the documented + # BaseInstrumentor is a per-class singleton, so the documented # SmolagentsInstrumentor().instrument() / SmolagentsInstrumentor() - # .uninstrument() form has to work on the live instance. + # .uninstrument() form must restore everything even though the second + # constructor call re-runs __init__ on the live instance. + original_generate = smolagents.OpenAIModel.generate + SmolagentsInstrumentor().instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, @@ -75,23 +93,201 @@ def test_uninstrument_through_a_new_constructor_call( ) SmolagentsInstrumentor().uninstrument() - assert not SmolagentsInstrumentor().is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is original_generate + + +@pytest.mark.parametrize("method", ["generate", "generate_stream"]) +def test_model_classes_defining(method: str) -> None: + classes = _model_classes_defining(smolagents, method) + + # Each class object appears once, whatever names it is exported under. + assert len(classes) == len(set(classes)) + # Every entry owns the method, so no class is wrapped for one it only + # inherits. + for model_cls in classes: + assert method in model_cls.__dict__ + + # The API-backed model classes define both; the classes that inherit them + # are reached through the base they inherit from. + assert { + smolagents.OpenAIModel, + smolagents.LiteLLMModel, + smolagents.InferenceClientModel, + } <= set(classes) + assert smolagents.AzureOpenAIModel not in classes + assert smolagents.LiteLLMRouterModel not in classes + + +def test_only_generate_covers_the_base_class_and_bedrock() -> None: + # The base Model and AmazonBedrockModel define generate but no + # generate_stream, so streaming is patched on fewer classes. + generate = set(_model_classes_defining(smolagents, "generate")) + generate_stream = set( + _model_classes_defining(smolagents, "generate_stream") + ) + + assert {smolagents.Model, smolagents.AmazonBedrockModel} <= generate + assert generate_stream.isdisjoint( + {smolagents.Model, smolagents.AmazonBedrockModel} + ) + + +def test_repeated_instrument_uninstrument( + tracer_provider, logger_provider, meter_provider +) -> None: + # BaseInstrumentor returns a per-class singleton, so the wrapped-class + # bookkeeping has to survive being filled and drained more than once. + original_generate = smolagents.OpenAIModel.generate + + instrumentor = SmolagentsInstrumentor() + for _ in range(2): + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + assert smolagents.OpenAIModel.generate is not original_generate + instrumentor.uninstrument() + assert smolagents.OpenAIModel.generate is original_generate def test_uninstrument_without_instrument() -> None: # BaseInstrumentor.uninstrument() short-circuits, but _uninstrument() must - # also be a no-op on its own: the rollback path in _instrument() calls it - # after a partial patch. + # also be a no-op on unpatched attributes: the rollback in _instrument() + # calls it after a partial patch. + original_generate = smolagents.OpenAIModel.generate + SmolagentsInstrumentor().uninstrument() SmolagentsInstrumentor()._uninstrument() # noqa: SLF001 + assert smolagents.OpenAIModel.generate is original_generate + def test_instrument_with_no_providers() -> None: # Without providers the handler falls back to the globals; instrumenting # must not require a caller to pass them. + original_generate = smolagents.OpenAIModel.generate + instrumentor = SmolagentsInstrumentor() instrumentor.instrument() try: - assert instrumentor.is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is not original_generate finally: instrumentor.uninstrument() + + assert smolagents.OpenAIModel.generate is original_generate + + +def test_failed_instrument_rolls_back_partial_patches( + tracer_provider, logger_provider, meter_provider +) -> None: + # A failure part-way through must leave no class patched, because + # uninstrument() cannot clean up after a failed _instrument(). + model_classes = _model_classes_defining(smolagents, "generate") + assert len(model_classes) > 1, ( + "the rollback needs more than one class to patch" + ) + originals = { + model_cls: model_cls.__dict__["generate"] + for model_cls in model_classes + } + stream_classes = _model_classes_defining(smolagents, "generate_stream") + stream_originals = { + model_cls: model_cls.__dict__["generate_stream"] + for model_cls in stream_classes + } + + real_wrap = wrap_function_wrapper + calls = 0 + + def fail_on_the_second_class(target: Any, name: str, wrapper: Any) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("boom") + real_wrap(target, name, wrapper) + + with patch( + "opentelemetry.instrumentation.genai.smolagents.wrap_function_wrapper", + fail_on_the_second_class, + ): + with pytest.raises(RuntimeError, match="boom"): + SmolagentsInstrumentor().instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + + assert calls == 2, "the first class was expected to be patched" + for model_cls, original in originals.items(): + assert model_cls.__dict__["generate"] is original + for model_cls, original in stream_originals.items(): + assert model_cls.__dict__["generate_stream"] is original + + +def test_inherited_generate_wrapped_only_on_defining_classes( + tracer_provider, logger_provider, meter_provider +) -> None: + # AzureOpenAIModel and LiteLLMRouterModel inherit generate; they must not be + # wrapped separately or they would emit duplicate chat spans. + assert "generate" not in smolagents.AzureOpenAIModel.__dict__ + assert "generate" not in smolagents.LiteLLMRouterModel.__dict__ + + instrumentor = SmolagentsInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + try: + # wrapt returns a fresh bound wrapper per attribute access, so the + # wrapper objects differ; the underlying wrapped function is shared, + # proving AzureOpenAIModel inherits the single wrapped generate. + assert ( + smolagents.AzureOpenAIModel.generate.__wrapped__ + is smolagents.OpenAIModel.generate.__wrapped__ + ) + finally: + instrumentor.uninstrument() + + +def test_explicit_completion_hook_takes_precedence( + tracer_provider, logger_provider, meter_provider, span_exporter +) -> None: + explicit_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook" + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + completion_hook=explicit_hook, + ): + _generate_a_chat_span() + + load_hook.assert_not_called() + assert explicit_hook.calls, "explicit completion hook was not invoked" + + +def test_env_completion_hook_used_when_no_explicit_hook( + tracer_provider, logger_provider, meter_provider +) -> None: + env_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook", + return_value=env_hook, + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + _generate_a_chat_span() + + load_hook.assert_called_once() + assert env_hook.calls, "env-resolved completion hook was not invoked" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py new file mode 100644 index 000000000..ca72d4e86 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -0,0 +1,1296 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Model (``chat``) instrumentation tests: VCR-backed runs of the real +smolagents model classes, provider/endpoint resolution, request-parameter +mapping, and message conversion. +""" + +from __future__ import annotations + +import inspect +import json +import sys +from collections.abc import Generator +from types import ModuleType, SimpleNamespace +from typing import Any + +import pytest +from smolagents import LiteLLMModel, OpenAIModel +from smolagents.models import ( + ChatMessage, + ChatMessageToolCall, + ChatMessageToolCallFunction, + MessageRole, +) + +from opentelemetry.instrumentation.genai.smolagents._messages import ( + response_id, + response_model_name, + to_input_messages, + to_output_message, + to_tool_definitions, +) +from opentelemetry.instrumentation.genai.smolagents.provider import ( + resolve_provider, + resolve_server_address_port, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import ( + error_attributes, + server_attributes, +) +from opentelemetry.trace import StatusCode +from opentelemetry.util.genai.types import ( + Blob, + Reasoning, + Text, + ToolCallRequest, + Uri, +) + +from .test_utils import ( + GetWeatherTool, + attr, + data_point_attributes, + metrics_by_name, + openai_model, + parse_messages, + part_types, + spans_by_operation, + stub_openai_client, + stub_streaming_openai_client, + text_chunk, + tool_call_chunk, + usage_chunk, +) + +IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +def test_openai_model_basic( + instrument_with_content, span_exporter, metric_reader, vcr +) -> None: + model = openai_model() + text = "Who won the World Cup in 2018? Answer in one word with no punctuation." + with vcr.use_cassette("openai_model_basic.yaml"): + output = model.generate( + messages=[ + {"role": "user", "content": [{"type": "text", "text": text}]} + ] + ) + assert output.content == "France" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.name == "chat gpt-4o" + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 25 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + assert ( + attr(span, GenAI.GEN_AI_RESPONSE_ID) + == "chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb" + ) + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) == "gpt-4o-2024-08-06" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert attr(span, server_attributes.SERVER_ADDRESS) == "api.openai.com" + assert attr(span, server_attributes.SERVER_PORT) is None + + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0]["role"] == "user" + assert inputs[0]["parts"][0] == {"type": "text", "content": text} + + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["role"] == "assistant" + assert outputs[0]["parts"][0] == {"type": "text", "content": "France"} + + metrics = metrics_by_name(metric_reader) + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + assert duration.unit == "s" + assert data_point_attributes(duration) == [ + { + GenAI.GEN_AI_OPERATION_NAME: "chat", + GenAI.GEN_AI_PROVIDER_NAME: "openai", + GenAI.GEN_AI_REQUEST_MODEL: "gpt-4o", + GenAI.GEN_AI_RESPONSE_MODEL: "gpt-4o-2024-08-06", + server_attributes.SERVER_ADDRESS: "api.openai.com", + } + ] + token_usage = metrics[gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE] + assert { + point.attributes[GenAI.GEN_AI_TOKEN_TYPE]: point.sum + for point in token_usage.data.data_points + } == {"input": 25, "output": 2} + + +def test_openai_model_no_content( + instrument_no_content, span_exporter, vcr +) -> None: + model = openai_model() + with vcr.use_cassette("openai_model_basic.yaml"): + model.generate( + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Hi"}]} + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert isinstance(attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS), int) + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_openai_model_image_url( + instrument_with_content, span_exporter, vcr +) -> None: + model = openai_model() + with vcr.use_cassette("openai_model_image_url.yaml"): + model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + {"type": "text", "text": "What breed is this dog?"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + ], + ) + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + parts = inputs[0]["parts"] + assert parts[0] == {"type": "text", "content": "What breed is this dog?"} + assert parts[1]["type"] == "uri" + assert parts[1]["modality"] == "image" + assert parts[1]["uri"] == IMAGE_URL + + +def test_openai_model_with_tools( + instrument_with_content, span_exporter, vcr +) -> None: + model = openai_model() + with vcr.use_cassette("openai_model_tool.yaml"): + output = model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What is the weather in Paris?", + } + ], + ) + ], + tools_to_call_from=[GetWeatherTool()], + ) + assert output.tool_calls[0].function.name == "get_weather" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_DEFINITIONS)) == [ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather for a given city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + }, + } + ] + + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + tool_call_parts = [ + part for part in outputs[0]["parts"] if part["type"] == "tool_call" + ] + assert tool_call_parts[0]["name"] == "get_weather" + # smolagents hands the provider's raw argument payload through unparsed. + assert tool_call_parts[0]["arguments"] == '{"location":"Paris"}' + assert tool_call_parts[0]["id"] == "call_EUuviydGIG5Jau3DLw4v4cue" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) + + +def _litellm_supports_reasoning() -> bool: + """litellm surfaces Anthropic ``reasoning_content`` only from ~1.63 onward. + + The oldest supported litellm (smolagents' 1.60.2 floor) doesn't parse the + Anthropic thinking blocks into ``reasoning_content``, so the reasoning part + can't be mapped there. Gate the reasoning-specific assertion on the version. + """ + from importlib.metadata import version # noqa: PLC0415 + + parts = version("litellm").split(".") + try: + return (int(parts[0]), int(parts[1])) >= (1, 63) + except (IndexError, ValueError): + return True + + +def test_litellm_reasoning( + instrument_with_content, + span_exporter, + litellm_local_cost_map, + patch_tiktoken_encoding, + vcr, +) -> None: + model = LiteLLMModel( + model_id="anthropic/claude-3-7-sonnet-20250219", + api_key="test_anthropic_api_key", + thinking={"type": "enabled", "budget_tokens": 4000}, + ) + with vcr.use_cassette("litellm_reasoning.yaml"): + model.generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? Answer in one " + "word with no punctuation." + ), + } + ], + } + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "anthropic" + assert ( + attr(span, GenAI.GEN_AI_REQUEST_MODEL) + == "anthropic/claude-3-7-sonnet-20250219" + ) + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + if _litellm_supports_reasoning(): + assert "reasoning" in part_types(outputs) + + +def test_model_generate_reraises_and_records_error( + instrument_with_content, span_exporter +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model(model_id="broken-model") + with pytest.raises(NotImplementedError): + model.generate(messages=[{"role": "user", "content": "hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "NotImplementedError" + + +def test_inherited_azure_generate_emits_one_chat_span( + instrument_with_content, span_exporter +) -> None: + # AzureOpenAIModel inherits generate from OpenAIModel, so only the defining + # class is patched. Exercise the inherited method end to end to prove the + # single patch still produces exactly one span with the Azure provider. + from smolagents import AzureOpenAIModel # noqa: PLC0415 + + model = AzureOpenAIModel( + model_id="gpt-4o-deployment", + azure_endpoint="https://example-resource.openai.azure.com", + api_key="test_azure_api_key", + api_version="2024-10-21", + ) + model.client = stub_openai_client("Bonjour") + + output = model.generate(messages=[{"role": "user", "content": "Hi"}]) + assert output.content == "Bonjour" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "azure.ai.openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o-deployment" + assert ( + attr(span, server_attributes.SERVER_ADDRESS) + == "example-resource.openai.azure.com" + ) + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 1 + + +def _fake_model(class_name: str, **attrs: Any) -> Any: + instance = type(class_name, (), {})() + for key, value in attrs.items(): + setattr(instance, key, value) + return instance + + +@pytest.mark.parametrize( + "class_name, attrs, expected", + [ + ("OpenAIModel", {}, "openai"), + ("AzureOpenAIModel", {}, "azure.ai.openai"), + ("AmazonBedrockModel", {}, "aws.bedrock"), + # No GenAI registry value exists for these runtimes, so the product + # name is used rather than the class name. + ("InferenceClientModel", {}, "huggingface"), + ("TransformersModel", {}, "huggingface"), + ("VLLMModel", {}, "vllm"), + ("MLXModel", {}, "mlx"), + # LiteLLM vendor prefixes: remapped where the slug differs from the + # semconv value, passed through otherwise. + ("LiteLLMModel", {"model_id": "anthropic/claude-3"}, "anthropic"), + ("LiteLLMModel", {"model_id": "mistral/large"}, "mistral_ai"), + ("LiteLLMModel", {"model_id": "xai/grok"}, "x_ai"), + ("LiteLLMModel", {"model_id": "gemini/gemini-2.0"}, "gcp.gemini"), + ( + "LiteLLMModel", + {"model_id": "vertex_ai/gemini-2.0"}, + "gcp.vertex_ai", + ), + ("LiteLLMModel", {"model_id": "watsonx/granite"}, "ibm.watsonx.ai"), + ( + "LiteLLMModel", + {"model_id": "azure_ai/phi-4"}, + "azure.ai.inference", + ), + ("LiteLLMModel", {"model_id": "ollama/llama3"}, "ollama"), + # LiteLLMRouterModel takes a model-group name, not a provider/model + # slug, so there is nothing to resolve. + ("LiteLLMRouterModel", {"model_id": "model-group-1"}, "unknown"), + # gen_ai.provider.name is also a metric attribute. An unmapped model + # must not fall back to the deployment-specific host or a class name. + ( + "CustomModel", + {"api_base": "https://llm.example.com/v1"}, + "unknown", + ), + ("CustomModel", {}, "unknown"), + ], +) +def test_resolve_provider( + class_name: str, attrs: dict[str, Any], expected: str +) -> None: + assert resolve_provider(_fake_model(class_name, **attrs)) == expected + + +@pytest.mark.parametrize( + "model_class, expected", + [ + ("OpenAIModel", "openai"), + ("AzureOpenAIModel", "azure.ai.openai"), + ("AmazonBedrockModel", "aws.bedrock"), + ("InferenceClientModel", "huggingface"), + ("TransformersModel", "huggingface"), + ("VLLMModel", "vllm"), + ("MLXModel", "mlx"), + ("LiteLLMModel", "unknown"), + ("LiteLLMRouterModel", "unknown"), + ], +) +def test_resolve_provider_covers_every_real_model_class( + model_class: str, expected: str +) -> None: + # The mapping is keyed by class name, so pin it against the real classes + # rather than only against synthetic stand-ins. + import smolagents # noqa: PLC0415 + + instance = object.__new__(getattr(smolagents, model_class)) + assert resolve_provider(instance) == expected + + +@pytest.mark.parametrize( + "attrs, expected", + [ + ({"api_base": "https://api.openai.com/v1"}, ("api.openai.com", None)), + # The default HTTPS port is omitted per the semconv server.port guidance. + ( + {"api_base": "https://api.openai.com:443/v1"}, + ("api.openai.com", None), + ), + ({"api_base": "http://localhost:11434/v1"}, ("localhost", 11434)), + ( + { + "client_kwargs": { + "azure_endpoint": "https://x.openai.azure.com" + } + }, + ("x.openai.azure.com", None), + ), + ({}, (None, None)), + ], +) +def test_resolve_server_address_port( + attrs: dict[str, Any], expected: tuple[str | None, int | None] +) -> None: + assert resolve_server_address_port(_fake_model("M", **attrs)) == expected + + +def test_server_address_falls_back_to_the_sdk_client() -> None: + # The common configuration: no api_base, so the URL is only known to the + # client the model built for itself. + model = OpenAIModel(model_id="gpt-4o", api_key="test_openai_api_key") + assert resolve_server_address_port(model) == ("api.openai.com", None) + + +def test_request_parameters_recorded( + instrument_with_content, span_exporter +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model( + model_id="broken-model", + temperature=0.5, + top_p=0.9, + top_k=40, + frequency_penalty=0.25, + presence_penalty=1, + max_tokens=256, + seed=7, + ) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + stop_sequences=[""], + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.5 + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.9 + # top_k is a float attribute in the spec even though callers pass an int. + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_K) == 40.0 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_TOP_K), float) + assert attr(span, GenAI.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.25 + assert attr(span, GenAI.GEN_AI_REQUEST_PRESENCE_PENALTY) == 1.0 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == 256 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS), int) + assert attr(span, GenAI.GEN_AI_REQUEST_SEED) == 7 + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == ("",) + + +def test_model_kwargs_win_over_call_kwargs( + instrument_with_content, span_exporter +) -> None: + # _prepare_completion_kwargs applies the call kwargs first and the model + # kwargs on top, and drops any key whose model-level value is the + # REMOVE_PARAMETER sentinel. + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model = Model( + model_id="broken-model", + temperature=0.1, + max_tokens=REMOVE_PARAMETER, + ) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + temperature=0.9, + max_tokens=512, + top_p=0.5, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.1 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) is None + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.5 + + +@pytest.mark.parametrize( + "model_id, model_kwargs, expected", + [ + # gpt-5 doesn't accept `stop`, so smolagents truncates the generated + # text locally instead of sending the sequences. + ("gpt-5", {}, None), + ("gpt-4o", {}, ("",)), + # An explicit `stop` overrides the stop_sequences argument. + ("gpt-4o", {"stop": ["STOP"]}, ("STOP",)), + # The model-level sentinel pops the `stop` that + # _prepare_completion_kwargs seeded from stop_sequences, leaving the + # request with none. + ("gpt-4o", {"stop": "REMOVE"}, None), + ], +) +def test_stop_sequences_follow_what_is_sent( + instrument_with_content, + span_exporter, + model_id: str, + model_kwargs: dict[str, Any], + expected: tuple[str, ...] | None, +) -> None: + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + model = Model(model_id=model_id, **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + stop_sequences=[""], + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == expected + + +def _bedrock_model(response: dict[str, Any]) -> Any: + from smolagents import AmazonBedrockModel # noqa: PLC0415 + + # A caller-supplied client keeps boto3 out of the test. + return AmazonBedrockModel( + model_id="us.amazon.nova-pro-v1:0", + client=SimpleNamespace(converse=lambda **_: response), + ) + + +BEDROCK_RESPONSE: dict[str, Any] = { + "output": { + "message": { + "role": "assistant", + "content": [{"text": "done"}], + "tool_calls": None, + } + }, + "usage": {"inputTokens": 3, "outputTokens": 2}, + "stopReason": "end_turn", +} + + +def test_bedrock_stop_sequences_are_not_recorded( + instrument_with_content, span_exporter +) -> None: + # supports_stop_parameter says yes, but the prepared request carries no + # stop sequences, so the span must not claim any either. + model = _bedrock_model(BEDROCK_RESPONSE) + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + request = model._prepare_completion_kwargs( # noqa: SLF001 + messages=messages, stop_sequences=[""] + ) + assert model.supports_stop_parameter is True + assert "stop" not in request + + model.generate(messages=messages, stop_sequences=[""]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) is None + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "aws.bedrock" + # Bedrock's "end_turn" is normalized to the semconv "stop". + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + + +@pytest.mark.parametrize( + "model_kwargs, call_kwargs, expected", + [ + ({"max_tokens": 256}, {}, 256), + ({}, {"max_tokens": 256}, 256), + # max_new_tokens is the TransformersModel spelling of the same limit, + # and max_tokens wins when a caller sets both. + ({"max_new_tokens": 4096}, {}, 4096), + ({"max_new_tokens": 4096, "max_tokens": 256}, {}, 256), + ({}, {}, None), + ], +) +def test_max_tokens_covers_both_spellings( + instrument_with_content, + span_exporter, + model_kwargs: dict[str, Any], + call_kwargs: dict[str, Any], + expected: int | None, +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model(model_id="broken-model", **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], **call_kwargs + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == expected + + +@pytest.mark.parametrize( + "response_format, model_kwargs, expected", + [ + ({"type": "json_object"}, {}, "json"), + ({"type": "json_schema", "json_schema": {}}, {}, "json"), + ({"type": "text"}, {}, "text"), + (None, {}, None), + # smolagents forwards response_format unchanged, so its type is + # whatever the provider accepts. An unknown one is dropped rather than + # recorded on an enum attribute. + ({"type": "xml"}, {}, None), + ({}, {}, None), + # The model-level kwargs win over the argument, and the sentinel drops + # the key from the request, the same as for every other parameter. + ( + {"type": "json_object"}, + {"response_format": {"type": "text"}}, + "text", + ), + ({"type": "json_object"}, {"response_format": "REMOVE"}, None), + (None, {"response_format": {"type": "json_object"}}, "json"), + ], +) +def test_output_type_follows_the_response_format( + instrument_with_content, + span_exporter, + response_format: dict[str, Any] | None, + model_kwargs: dict[str, Any], + expected: str | None, +) -> None: + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + model = Model(model_id="broken-model", **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + response_format=response_format, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_OUTPUT_TYPE) == expected + + +def test_user_subclass_of_a_patched_model_keeps_its_provider( + instrument_with_content, span_exporter, metric_reader +) -> None: + # A subclass inherits the patched generate, so it is instrumented; resolving + # the provider by exact class name would report "unknown" on both the span + # and the metrics. + class TenantOpenAIModel(OpenAIModel): + pass + + model = TenantOpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="http://localhost:11434/v1", + ) + model.client = stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + # A non-default port is part of the endpoint, unlike the HTTPS default. + assert attr(span, server_attributes.SERVER_ADDRESS) == "localhost" + assert attr(span, server_attributes.SERVER_PORT) == 11434 + duration = metrics_by_name(metric_reader)[ + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + ] + assert data_point_attributes(duration)[0][GenAI.GEN_AI_PROVIDER_NAME] == ( + "openai" + ) + + +def test_provider_error_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_openai_client( + "", error=ConnectionError("connection reset") + ) + + with pytest.raises(ConnectionError, match="connection reset"): + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def _drain_stream(model: OpenAIModel, **kwargs: Any) -> list[Any]: + return list( + model.generate_stream( + messages=[{"role": "user", "content": "Hi"}], **kwargs + ) + ) + + +def test_generate_stream_is_lazy_and_records_the_drained_response( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] + ) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + # A streamed response isn't finished until the caller drains it. + assert spans_by_operation(span_exporter.get_finished_spans(), "chat") == [] + + deltas = list(stream) + assert "".join(delta.content or "" for delta in deltas) == "Bonjour" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" + assert attr(span, GenAI.GEN_AI_REQUEST_STREAM) is True + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "Bonjour"}] + # Deltas carry no finish reason, and "stop" would hide a truncation. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def test_generate_stream_stays_a_generator( + instrument_with_content, span_exporter +) -> None: + # Instrumentation observes; it must not change what generate_stream returns. + model = openai_model() + model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + assert isinstance(stream, Generator) + assert inspect.isgenerator(stream) + list(stream) + + +def test_generate_stream_accumulates_tool_calls( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [ + tool_call_chunk(0, call_id="call_1", name="get_weather"), + tool_call_chunk(0, arguments='{"location":'), + tool_call_chunk(0, arguments='"Paris"}'), + ] + ) + + _drain_stream(model, tools_to_call_from=[GetWeatherTool()]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + (part,) = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES)[0]["parts"] + assert part == { + "type": "tool_call", + "id": "call_1", + "name": "get_weather", + "arguments": '{"location":"Paris"}', + } + # Tool calls are the only evidence of why a streamed generation stopped. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) + + +def test_generate_stream_error_mid_iteration_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon")], error=ConnectionError("stream died") + ) + + with pytest.raises(ConnectionError, match="stream died"): + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" + # What was streamed before the failure is still recorded. + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "Bon"}] + + +def test_generate_stream_close_before_drain_finalizes_once( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + stream.close() + stream.close() # idempotent + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_generate_stream_records_chunk_metrics( + instrument_with_content, span_exporter, metric_reader +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] + ) + + _drain_stream(model) + + metrics = metrics_by_name(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metrics + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE in metrics + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK in metrics + ) + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_PER_OUTPUT_CHUNK in metrics + ) + + +def test_generate_stream_no_content( + instrument_no_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bonjour"), usage_chunk(3, 2)] + ) + + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + + +def test_event_only_content_capture( + instrument_event_only, span_exporter, log_exporter +) -> None: + model = openai_model() + model.client = stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + # Metadata still goes on the span. + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.event_name == "gen_ai.client.inference.operation.details" + # Event attributes carry the messages as structured values, not JSON text. + attributes = record.attributes or {} + inputs = attributes[GenAI.GEN_AI_INPUT_MESSAGES] + outputs = attributes[GenAI.GEN_AI_OUTPUT_MESSAGES] + assert inputs[0]["parts"][0]["content"] == "Hi" + assert outputs[0]["parts"][0]["content"] == "Bonjour" + assert outputs[0]["finish_reason"] == "stop" + + +def test_to_tool_definitions_uses_json_schema() -> None: + (definition,) = to_tool_definitions([GetWeatherTool()]) + assert definition.name == "get_weather" + assert definition.description == "Get the weather for a given city" + # smolagents' raw ``inputs`` map is not a JSON Schema on its own. + assert definition.parameters == { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + } + + +def test_to_input_messages_maps_smolagents_only_roles() -> None: + # smolagents converts these roles itself inside generate(), after the + # wrapper has already read the messages. + messages = to_input_messages( + [ + {"role": MessageRole.TOOL_CALL, "content": "call"}, + {"role": MessageRole.TOOL_RESPONSE, "content": "response"}, + {"role": MessageRole.SYSTEM, "content": "sys"}, + ] + ) + assert [message.role for message in messages] == [ + "assistant", + "user", + "system", + ] + + +def test_to_input_messages_dict_and_chatmessage() -> None: + messages = to_input_messages( + [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + ChatMessage(role=MessageRole.ASSISTANT, content="Hi there"), + ] + ) + assert messages[0].role == "user" + assert messages[0].parts == [Text(content="Hello")] + assert messages[1].role == "assistant" + assert messages[1].parts == [Text(content="Hi there")] + + +def test_to_input_messages_image_and_base64() -> None: + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image", "image": "aVZCT1J3MEtHZ28="}, + ], + } + ] + ) + parts = messages[0].parts + assert parts[0] == Uri(mime_type=None, modality="image", uri=IMAGE_URL) + assert isinstance(parts[1], Blob) + assert parts[1].modality == "image" + assert parts[1].mime_type == "image/png" + assert isinstance(parts[1].content, bytes) + + +def test_to_input_messages_data_url_keeps_media_type() -> None: + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + { + "type": "image", + "image": "data:image/jpeg;base64,aVZCT1J3MEtHZ28=", + } + ], + } + ] + ) + (part,) = messages[0].parts + assert isinstance(part, Blob) + assert part.mime_type == "image/jpeg" + + +@pytest.mark.parametrize( + "image", + [ + "not base64!!!!", + # A path is not base64, but a non-validating decode silently turns this + # one into 15 bytes of garbage after dropping the invalid characters. + "/tmp/photos/my-cat.png", + ], +) +def test_to_input_messages_drops_malformed_base64_image(image: str) -> None: + messages = to_input_messages( + [{"role": "user", "content": [{"type": "image", "image": image}]}] + ) + assert messages[0].parts == [] + + +def test_to_output_message_text_reasoning_and_tool_calls() -> None: + class _Msg: + role = MessageRole.ASSISTANT + content = "The answer" + tool_calls = [ + ChatMessageToolCall( + id="call_1", + type="function", + function=ChatMessageToolCallFunction( + name="get_weather", arguments='{"location": "Paris"}' + ), + ) + ] + + class raw: # noqa: N801 + class _Choice: + class message: # noqa: N801 + reasoning_content = "thinking about it" + + choices = [_Choice()] + + output = to_output_message(_Msg()) + assert output.role == "assistant" + assert Text(content="The answer") in output.parts + assert Reasoning(content="thinking about it") in output.parts + tool_calls = [p for p in output.parts if isinstance(p, ToolCallRequest)] + assert tool_calls[0].name == "get_weather" + assert tool_calls[0].id == "call_1" + assert output.finish_reason == "tool_calls" + + +def test_to_output_message_from_a_dict_raw_response() -> None: + # AmazonBedrockModel, TransformersModel, VLLMModel and MLXModel put a plain + # dict on ChatMessage.raw rather than an SDK object. + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="done", + raw={"stopReason": "max_tokens"}, + ) + output = to_output_message(message) + assert output.parts == [Text(content="done")] + # "max_tokens" is Bedrock's stopReason spelling for a length cutoff. + assert output.finish_reason == "length" + assert response_id(message) is None + assert response_model_name(message) is None + + +def test_to_output_message_unwraps_the_role_enum() -> None: + output = to_output_message( + ChatMessage(role=MessageRole.ASSISTANT, content="done") + ) + assert output.role == "assistant" + + +def test_to_output_message_maps_image_content() -> None: + # A response whose content is a list carries the same element shapes as a + # request, so an image in it maps the same way. + output = to_output_message( + ChatMessage( + role=MessageRole.ASSISTANT, + content=[ + {"type": "text", "text": "Here it is"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image", "image": "aVZCT1J3MEtHZ28="}, + ], + ) + ) + assert output.parts[0] == Text(content="Here it is") + assert output.parts[1] == Uri( + mime_type=None, modality="image", uri=IMAGE_URL + ) + blob = output.parts[2] + assert isinstance(blob, Blob) + assert blob.mime_type == "image/png" + assert blob.modality == "image" + + +LOCAL_RUNTIME_RAW = { + "out": "done", + "completion_kwargs": {"max_new_tokens": 4096}, +} + + +@pytest.mark.parametrize( + "raw, tool_calls, expected", + [ + # API-backed models: the provider's own value, whatever it is. + ( + SimpleNamespace(choices=[SimpleNamespace(finish_reason="stop")]), + [], + "stop", + ), + ( + SimpleNamespace(choices=[SimpleNamespace(finish_reason="length")]), + [], + "length", + ), + # Bedrock's stopReason values map onto the semconv vocabulary. + ({"stopReason": "max_tokens"}, [], "length"), + ({"stopReason": "tool_use"}, [], "tool_calls"), + # An unmapped value passes through rather than being guessed at. + ({"stopReason": "guardrail_intervened"}, [], "guardrail_intervened"), + # The local runtimes report no reason, so none is recorded and + # util-genai omits the empty value. + (LOCAL_RUNTIME_RAW, [], ""), + (None, [], ""), + # Tool calls in the response give the reason without guessing. + ( + LOCAL_RUNTIME_RAW, + [ + ChatMessageToolCall( + id="call_1", + type="function", + function=ChatMessageToolCallFunction( + name="get_weather", arguments="{}" + ), + ) + ], + "tool_calls", + ), + ], +) +def test_finish_reason_follows_the_provider_response( + raw: Any, tool_calls: list[ChatMessageToolCall], expected: str +) -> None: + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="done", + tool_calls=tool_calls or None, + raw=raw, + ) + assert to_output_message(message).finish_reason == expected + + +def test_model_reporting_no_finish_reason_omits_the_attribute( + instrument_with_content, span_exporter +) -> None: + # InferenceClientModel is a patched class whose provider response can come + # back without a finish reason; the span must then carry none. + from smolagents import InferenceClientModel # noqa: PLC0415 + + model = InferenceClientModel( + model_id="Qwen/Qwen2.5-Coder-32B-Instruct", token="hf_test" + ) + model.client = SimpleNamespace( + chat_completion=lambda **_: SimpleNamespace( + id="hf-1", + model="Qwen/Qwen2.5-Coder-32B-Instruct", + choices=[ + SimpleNamespace( + message=SimpleNamespace( + role="assistant", content="ok", tool_calls=None + ) + ) + ], + usage=SimpleNamespace(prompt_tokens=5, completion_tokens=2), + ) + ) + + model.generate(messages=[{"role": "user", "content": "hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["finish_reason"] == "" + + +# The local runtimes flatten a message's content as text, so the content has to +# be a list of parts rather than a bare string. +LOCAL_RUNTIME_MESSAGES: list[dict[str, Any]] = [ + { + "role": "user", + "content": [{"type": "text", "text": "Where is the Louvre?"}], + } +] + + +def _mlx_model(monkeypatch: pytest.MonkeyPatch) -> Any: + """An ``MLXModel`` whose ``mlx_lm`` pieces are stubbed. + + ``MLXModel.generate`` imports nothing itself; it drives ``stream_generate`` + over ``self.model`` and ``self.tokenizer``, which ``__init__`` loads from + ``mlx_lm``. Bypassing ``__init__`` is therefore enough to run the real + ``generate``, and the runtime doesn't have to be installed. ``monkeypatch`` + is unused here; it keeps the factory signature uniform with the vllm one, + which does have modules to stub. + """ + from smolagents.models import MLXModel # noqa: PLC0415 + + model = object.__new__(MLXModel) + model.model_id = "mlx-community/Qwen2.5-0.5B-Instruct-4bit" + model.kwargs = {} + model.flatten_messages_as_text = True + model.apply_chat_template_kwargs = {} + model.model = object() + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, tools=None, **_: [1, 2, 3] + ) + model.stream_generate = lambda *_, **__: iter( + [SimpleNamespace(text="In "), SimpleNamespace(text="Paris")] + ) + return model + + +def _vllm_model(monkeypatch: pytest.MonkeyPatch) -> Any: + """A ``VLLMModel`` with ``vllm`` itself stubbed. + + ``VLLMModel.generate`` imports ``SamplingParams`` and + ``StructuredOutputsParams`` from ``vllm`` when it runs, and neither test env + installs vllm, so both modules are faked for the duration of the test. + Everything the wrapper reads still comes from the real ``generate``. + """ + from smolagents.models import VLLMModel # noqa: PLC0415 + + def fake_params(**kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(**kwargs) + + vllm = ModuleType("vllm") + sampling_params = ModuleType("vllm.sampling_params") + setattr(vllm, "SamplingParams", fake_params) + setattr(sampling_params, "StructuredOutputsParams", fake_params) + setattr(vllm, "sampling_params", sampling_params) + monkeypatch.setitem(sys.modules, "vllm", vllm) + monkeypatch.setitem(sys.modules, "vllm.sampling_params", sampling_params) + + completion = SimpleNamespace( + prompt_token_ids=[1, 2, 3, 4], + outputs=[SimpleNamespace(text="In Paris", token_ids=[5, 6])], + ) + model = object.__new__(VLLMModel) + model.model_id = "Qwen/Qwen2.5-0.5B-Instruct" + model.kwargs = {} + model.flatten_messages_as_text = True + model._is_vlm = False + model.apply_chat_template_kwargs = {} + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, **_: "prompt" + ) + model.model = SimpleNamespace(generate=lambda *_, **__: [completion]) + return model + + +@pytest.mark.parametrize( + "model_factory, provider, request_model, input_tokens, output_tokens", + [ + ( + _mlx_model, + "mlx", + "mlx-community/Qwen2.5-0.5B-Instruct-4bit", + 3, + 2, + ), + (_vllm_model, "vllm", "Qwen/Qwen2.5-0.5B-Instruct", 4, 2), + ], + ids=["mlx", "vllm"], +) +def test_local_runtime_response_is_recorded( + instrument_with_content, + span_exporter, + monkeypatch: pytest.MonkeyPatch, + model_factory: Any, + provider: str, + request_model: str, + input_tokens: int, + output_tokens: int, +) -> None: + output = model_factory(monkeypatch).generate( + messages=LOCAL_RUNTIME_MESSAGES + ) + assert output.content == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == provider + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == request_model + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == input_tokens + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == output_tokens + # A local runtime returns no provider response envelope and listens on no + # socket, so it reports no finish reason, id or response model, and there is + # no endpoint to derive server.address from. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_ID) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) is None + assert attr(span, server_attributes.SERVER_ADDRESS) is None + assert attr(span, server_attributes.SERVER_PORT) is None + assert span.status.status_code == StatusCode.UNSET + + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0]["parts"] == [ + {"type": "text", "content": "Where is the Louvre?"} + ] + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["role"] == "assistant" + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] + assert outputs[0]["finish_reason"] == "" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py new file mode 100644 index 000000000..eeb8d78e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py @@ -0,0 +1,222 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers, tools, and model stubs for smolagents instrumentation tests.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import Any + +from smolagents import OpenAIModel, Tool + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + + +class GetWeatherTool(Tool): + name = "get_weather" + description = "Get the weather for a given city" + inputs = { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + } + output_type = "string" + + def forward(self, location: str) -> str: + return "sunny" + + +def openai_model() -> OpenAIModel: + return OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + + +def stub_openai_client( + content: str, finish_reason: str = "stop", error: Exception | None = None +) -> Any: + """An object shaped like the bits of ``openai.OpenAI`` that ``generate`` uses. + + Building a real ``ChatCompletion`` keeps the response shape honest (the + wrapper reads ``raw.model``, ``raw.id``, and ``raw.choices[0].finish_reason``) + without needing a cassette for a deployment we can't record against. + """ + from openai.types.chat import ( # noqa: PLC0415 + ChatCompletion, + ChatCompletionMessage, + ) + from openai.types.chat.chat_completion import Choice # noqa: PLC0415 + from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 + + completion = ChatCompletion( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion", + created=0, + choices=[ + Choice( + index=0, + finish_reason=finish_reason, + message=ChatCompletionMessage( + role="assistant", content=content + ), + ) + ], + usage=CompletionUsage( + prompt_tokens=3, completion_tokens=1, total_tokens=4 + ), + ) + + def create(**_: Any) -> ChatCompletion: + if error is not None: + raise error + return completion + + return SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + +def stub_streaming_openai_client( + chunks: list[Any], error: Exception | None = None +) -> Any: + """An ``openai.OpenAI`` stand-in whose ``create`` returns a chunk stream. + + ``OpenAIModel.generate_stream`` reads ``event.usage`` and + ``event.choices[0].delta``, so the chunks are real ``ChatCompletionChunk`` + objects. ``error`` is raised after the chunks are yielded, which is how a + provider failure part-way through a stream reaches the caller. + """ + + def create(**_: Any) -> Any: + def stream() -> Any: + yield from chunks + if error is not None: + raise error + + return stream() + + return SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + +def text_chunk(content: str) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 + Choice, + ChoiceDelta, + ) + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[Choice(index=0, delta=ChoiceDelta(content=content))], + ) + + +def tool_call_chunk( + index: int, + call_id: str | None = None, + name: str | None = None, + arguments: str | None = None, +) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 + Choice, + ChoiceDelta, + ChoiceDeltaToolCall, + ChoiceDeltaToolCallFunction, + ) + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[ + Choice( + index=0, + delta=ChoiceDelta( + tool_calls=[ + ChoiceDeltaToolCall( + index=index, + id=call_id, + type="function", + function=ChoiceDeltaToolCallFunction( + name=name, arguments=arguments + ), + ) + ] + ), + ) + ], + ) + + +def usage_chunk(prompt_tokens: int, completion_tokens: int) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[], + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +def spans_by_operation( + spans: list[ReadableSpan], operation: str +) -> list[ReadableSpan]: + return [ + span + for span in spans + if (span.attributes or {}).get(GenAI.GEN_AI_OPERATION_NAME) + == operation + ] + + +def attr(span: ReadableSpan, name: str) -> Any: + return (span.attributes or {}).get(name) + + +def parse_messages(span: ReadableSpan, name: str) -> list[dict[str, Any]]: + raw = attr(span, name) + return json.loads(raw) if isinstance(raw, str) else [] + + +def part_types(messages: list[dict[str, Any]]) -> list[str]: + return [part["type"] for message in messages for part in message["parts"]] + + +def metrics_by_name(metric_reader: Any) -> dict[str, Any]: + data = metric_reader.get_metrics_data() + if data is None: + return {} + return { + metric.name: metric + for resource_metric in data.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + } + + +def data_point_attributes(metric: Any) -> list[dict[str, Any]]: + return [dict(point.attributes) for point in metric.data.data_points] diff --git a/tox.ini b/tox.ini index 55d61bfaf..7216c0952 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,7 @@ envlist = ; instrumentation-genai-smolagents py3{10,11,12,13,14}-test-instrumentation-genai-smolagents-latest py310-test-instrumentation-genai-smolagents-oldest + py314-test-instrumentation-genai-smolagents-conformance lint-instrumentation-genai-smolagents ; instrumentation-genai-anthropic @@ -153,6 +154,8 @@ deps = smolagents-latest: {[testenv]test_deps} smolagents-latest: {[testenv]pytest_deps} smolagents-latest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt + smolagents-conformance: {[testenv]pytest_deps} + smolagents-conformance: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt anthropic-oldest: {[testenv]pytest_deps} anthropic-oldest: -e {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic[instruments] @@ -245,6 +248,7 @@ commands = lint-instrumentation-genai-agno: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-agno" test-instrumentation-genai-smolagents-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests --vcr-record=none {posargs} + test-instrumentation-genai-smolagents-conformance: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py --vcr-record=none {posargs} lint-instrumentation-genai-smolagents: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-smolagents" test-instrumentation-genai-anthropic-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests --vcr-record=none {posargs}