diff --git a/instrumentation/README.md b/instrumentation/README.md index 5c8bb36f4..1700f4731 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -10,6 +10,6 @@ | [opentelemetry-instrumentation-genai-openai](./opentelemetry-instrumentation-genai-openai) | openai >= 1.26.0 | Yes | development | [opentelemetry-instrumentation-genai-openai-agents](./opentelemetry-instrumentation-genai-openai-agents) | openai-agents >= 0.3.3 | No | development | [opentelemetry-instrumentation-genai-qwen-agent](./opentelemetry-instrumentation-genai-qwen-agent) | qwen-agent >= 0.0.20 | No | development -| [opentelemetry-instrumentation-genai-smolagents](./opentelemetry-instrumentation-genai-smolagents) | smolagents >= 1.24.0 | No | development +| [opentelemetry-instrumentation-genai-smolagents](./opentelemetry-instrumentation-genai-smolagents) | smolagents >= 1.24.0 | Yes | development | [opentelemetry-instrumentation-genai-weaviate-client](./opentelemetry-instrumentation-genai-weaviate-client) | weaviate-client >= 3.0.0,<5.0.0 | No | development | [opentelemetry-instrumentation-google-genai](./opentelemetry-instrumentation-google-genai) | google-genai >= 1.32.0, <3 | No | development diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added new file mode 100644 index 000000000..5aca70172 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/352.added @@ -0,0 +1 @@ +Add ``chat`` instrumentation for the smolagents model classes. diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst index 932ad2c12..fa97a7c54 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst @@ -7,7 +7,31 @@ OpenTelemetry smolagents Instrumentation :target: https://pypi.org/project/opentelemetry-instrumentation-genai-smolagents/ This library provides OpenTelemetry instrumentation for `smolagents -`_. +`_. It wraps the smolagents model +classes and emits a GenAI semantic-convention ``chat`` span and the matching +metrics through ``opentelemetry-util-genai``. + +Agent runs (``invoke_agent``) and tool calls (``execute_tool``) are not +instrumented yet. A model call made inside an agent run still gets a ``chat`` +span, but no agent span sits above it. + +A streamed model call, whether it comes from ``stream_outputs=True`` on an agent +or from calling ``Model.generate_stream`` directly, gets a ``chat`` span that +stays open until the caller drains the deltas. The span carries +``gen_ai.request.stream``, and the call also records the +``gen_ai.client.operation.time_to_first_chunk`` and +``gen_ai.client.operation.time_per_output_chunk`` metrics. + +Known gaps: + +* The instrumentation patches ``generate`` and ``generate_stream`` on the model + classes that smolagents ships. A subclass that inherits either method is + instrumented. A subclass that overrides one is not: the override shadows the + patched method, so the call produces no ``chat`` span. +* A streamed ``chat`` span reports no ``gen_ai.response.id`` and no + ``gen_ai.response.model``, because a smolagents stream delta carries neither. + The span reports ``gen_ai.response.finish_reasons`` only when the model + requested tool calls, which is the one stop reason the deltas make visible. Installation ------------ @@ -24,10 +48,13 @@ Usage from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import InferenceClientModel - # Instrument smolagents SmolagentsInstrumentor().instrument() + model = InferenceClientModel() + model.generate([{"role": "user", "content": "How many seconds are in a week?"}]) + Configuration ------------- @@ -71,6 +98,12 @@ environment variable: SmolagentsInstrumentor().instrument(completion_hook=my_hook) +Conformance +----------- + +The scenarios that check this package against the GenAI semantic conventions +live under ``tests/conformance/``. + References ---------- diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml index 9f7ed96ed..67140011a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "opentelemetry-api ~= 1.43", "opentelemetry-instrumentation >= 0.64b0, <1", "opentelemetry-semantic-conventions >= 0.64b0, <1", - "opentelemetry-util-genai >= 1.0b0, <2", + "opentelemetry-util-genai >= 1.1b0.dev, <2", ] [project.optional-dependencies] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index b3cffd043..4e14cda26 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -7,6 +7,9 @@ Instrumentation for `smolagents `_. +Model calls are recorded as ``chat`` spans. Agent runs and tool calls are not +instrumented yet. + Usage ----- @@ -15,10 +18,13 @@ from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import InferenceClientModel - # Enable instrumentation SmolagentsInstrumentor().instrument() + model = InferenceClientModel() + model.generate([{"role": "user", "content": "How many seconds are in a week?"}]) + Configuration ------------- @@ -38,20 +44,63 @@ from __future__ import annotations +from types import ModuleType from typing import Any, Collection +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.utils import unwrap from opentelemetry.util.genai.completion_hook import load_completion_hook from opentelemetry.util.genai.handler import TelemetryHandler from .package import _instruments +from .patch import model_generate, model_generate_stream __all__ = ["SmolagentsInstrumentor"] +def _model_classes_defining(smolagents: ModuleType, method: str) -> list[type]: + """The exported model classes whose ``method`` gets wrapped. + + Only classes that define ``method`` in their own ``__dict__`` are patched, + so a class that inherits it (``AzureOpenAIModel``, ``LiteLLMRouterModel``) + isn't wrapped a second time and can't produce duplicate ``chat`` spans. + A user-defined subclass that overrides the method shadows the patched base + method and emits no ``chat`` span; that limitation is documented in + ``README.rst``. + + Deduplicated by class object, because smolagents exports some classes under + two names (``OpenAIServerModel`` is ``OpenAIModel``) and wrapping the same + class twice would double every ``chat`` span. + """ + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + Model, + ) + + classes: dict[type, None] = {} + for obj in vars(smolagents).values(): + if ( + isinstance(obj, type) + and issubclass(obj, Model) + and method in obj.__dict__ + ): + classes.setdefault(obj, None) + return list(classes) + + class SmolagentsInstrumentor(BaseInstrumentor): """An instrumentor for smolagents.""" + # ``BaseInstrumentor.__new__`` returns a per-class singleton, but Python + # still runs ``__init__`` on every construction. Initializing this state in + # ``__init__`` would let the documented ``SmolagentsInstrumentor() + # .uninstrument()`` form wipe the live instance's bookkeeping and leave + # smolagents permanently patched, so these are class-level defaults that + # only ``_instrument`` / ``_uninstrument`` rebind. + _wrapped_generate_classes: list[type] = [] + _wrapped_generate_stream_classes: list[type] = [] + def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -65,15 +114,51 @@ def _instrument(self, **kwargs: Any) -> None: - logger_provider: LoggerProvider instance - completion_hook: CompletionHook instance """ - TelemetryHandler( + import smolagents # pylint: disable=import-outside-toplevel # noqa: PLC0415 + + handler = TelemetryHandler( tracer_provider=kwargs.get("tracer_provider"), meter_provider=kwargs.get("meter_provider"), logger_provider=kwargs.get("logger_provider"), completion_hook=kwargs.get("completion_hook") or load_completion_hook(), ) - # Patching will be added in follow-up PRs. + + wrapped_generate_classes: list[type] = [] + self._wrapped_generate_classes = wrapped_generate_classes + wrapped_generate_stream_classes: list[type] = [] + self._wrapped_generate_stream_classes = wrapped_generate_stream_classes + try: + for model_cls in _model_classes_defining(smolagents, "generate"): + wrap_function_wrapper( + model_cls, + "generate", + model_generate(handler), + ) + wrapped_generate_classes.append(model_cls) + + for model_cls in _model_classes_defining( + smolagents, "generate_stream" + ): + wrap_function_wrapper( + model_cls, + "generate_stream", + model_generate_stream(handler), + ) + wrapped_generate_stream_classes.append(model_cls) + except Exception: + # BaseInstrumentor.instrument() doesn't mark the instrumentor as + # instrumented when _instrument raises, so uninstrument() would + # refuse to run and leave the patches applied with no way to undo. + self._uninstrument() + raise def _uninstrument(self, **kwargs: Any) -> None: """Disable smolagents instrumentation and restore patched originals.""" - # Unpatching will be added in follow-up PRs. + for model_cls in self._wrapped_generate_classes: + unwrap(model_cls, "generate") + self._wrapped_generate_classes = [] + + for model_cls in self._wrapped_generate_stream_classes: + unwrap(model_cls, "generate_stream") + self._wrapped_generate_stream_classes = [] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py new file mode 100644 index 000000000..797021372 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -0,0 +1,316 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Convert smolagents message and tool shapes into util-genai GenAI types. + +smolagents passes ``generate(messages=...)`` a list of ``ChatMessage`` objects +or plain dicts, and returns a ``ChatMessage``. This module maps those, and the +``tools_to_call_from`` tool objects, onto the types in +``opentelemetry.util.genai.types``. util-genai then serializes them into +``gen_ai.input.messages``, ``gen_ai.output.messages``, and +``gen_ai.tool.definitions``. +""" + +from __future__ import annotations + +import base64 +import binascii +import logging +from collections.abc import Mapping +from enum import Enum +from typing import Any + +from opentelemetry.util.genai.types import ( + Blob, + FunctionToolDefinition, + InputMessage, + OutputMessage, + Reasoning, + Text, + ToolCallRequest, + ToolDefinition, + Uri, +) + +_logger = logging.getLogger(__name__) + +_DEFAULT_IMAGE_MIME_TYPE = "image/png" +_DATA_URL_PREFIX = "data:" + +# smolagents-internal roles -> semconv ``gen_ai`` message roles. smolagents +# applies the same mapping (``models.tool_role_conversions``) inside +# ``generate``, after the wrapper has already read ``messages``, so the wrapper +# has to apply it too. Without this map the input messages would carry roles +# the spec doesn't define. A model configured with ``custom_role_conversions`` +# overrides the mapping and can send different roles. Roles that are already +# spec values pass through unchanged. +_ROLE_MAP: dict[str, str] = { + "tool-call": "assistant", + "tool-response": "user", +} + +# Amazon Bedrock reports the stop reason as ``stopReason`` using Anthropic's +# vocabulary. Normalize it the way the anthropic instrumentation does +# (``anthropic/utils.py``); anything unmapped passes through. +_STOP_REASON_MAP: dict[str, str] = { + "end_turn": "stop", + "stop_sequence": "stop", + "max_tokens": "length", + "tool_use": "tool_calls", +} + + +def _unwrap_role(role: Any) -> str | None: + if role is None: + return None + if isinstance(role, Enum): + role = role.value + name = str(role) + return _ROLE_MAP.get(name, name) + + +def _decode_base64_image(image: str) -> tuple[bytes, str] | None: + """Decode a base64 payload or data URL into ``(bytes, mime_type)``.""" + mime_type = _DEFAULT_IMAGE_MIME_TYPE + if image.startswith(_DATA_URL_PREFIX): + header, _, image = image.partition(",") + media_type = header[len(_DATA_URL_PREFIX) :].split(";")[0] + if media_type: + mime_type = media_type + try: + return base64.b64decode(image, validate=True), mime_type + except (binascii.Error, ValueError): + _logger.debug("Failed to decode a base64 image", exc_info=True) + return None + + +def _encode_image_base64(image: Any) -> str | None: + try: + from smolagents.utils import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + encode_image_base64, + ) + except ImportError: + _logger.debug("smolagents.utils.encode_image_base64 is unavailable") + return None + try: + encoded = encode_image_base64(image) + except Exception: # pylint: disable=broad-except + _logger.debug( + "Failed to encode image of type %s, dropping it from telemetry", + type(image).__name__, + exc_info=True, + ) + return None + return encoded if isinstance(encoded, str) else None + + +def _image_blob(image: Any) -> Blob | None: + """Build a ``Blob`` part from a base64 string, data URL, or PIL image.""" + if isinstance(image, str): + decoded = _decode_base64_image(image) + else: + encoded = _encode_image_base64(image) + decoded = ( + _decode_base64_image(encoded) if encoded is not None else None + ) + if decoded is None: + return None + content, mime_type = decoded + return Blob(mime_type=mime_type, modality="image", content=content) + + +def _image_part_from_element(element: dict[str, Any]) -> Uri | Blob | None: + content_type = element.get("type") + if content_type == "image_url": + image_url = element.get("image_url") + url = image_url.get("url") if isinstance(image_url, dict) else None + if isinstance(url, str) and url: + return Uri(mime_type=None, modality="image", uri=url) + return None + if content_type == "image": + image = element.get("image") + if image is not None: + return _image_blob(image) + return None + + +def _parts_from_content(content: Any) -> list[Any]: + parts: list[Any] = [] + if isinstance(content, str): + parts.append(Text(content=content)) + return parts + if isinstance(content, list): + for element in content: + if not isinstance(element, dict): + _logger.debug( + "Unknown message content dropped from telemetry: %s", + type(element).__name__, + ) + continue + if element.get("type") == "text" and (text := element.get("text")): + parts.append(Text(content=text)) + continue + image_part = _image_part_from_element(element) + if image_part is not None: + parts.append(image_part) + else: + _logger.debug( + "Unknown message part dropped from telemetry: %s", + element.get("type"), + ) + return parts + + +def _get_role_and_content(message: Any) -> tuple[Any, Any]: + if isinstance(message, dict): + return message.get("role"), message.get("content") + return getattr(message, "role", None), getattr(message, "content", None) + + +def to_input_messages(messages: Any) -> list[InputMessage]: + """Map smolagents ``generate`` input messages to ``InputMessage`` objects.""" + result: list[InputMessage] = [] + if not isinstance(messages, list): + return result + for message in messages: + raw_role, content = _get_role_and_content(message) + role = _unwrap_role(raw_role) + if not role: + continue + result.append( + InputMessage(role=role, parts=_parts_from_content(content)) + ) + return result + + +def _raw_value(raw: Any, key: str) -> Any: + """Read ``key`` off a provider response that may be an object or a dict. + + ``ChatMessage.raw`` is whatever the provider handed back: an OpenAI-shaped + object for the API-backed models, and a dict for ``AmazonBedrockModel`` + (the boto3 ``converse`` response) and the local runtimes. + """ + if isinstance(raw, Mapping): + return raw.get(key) + return getattr(raw, key, None) + + +def _first_choice(output_message: Any) -> Any: + choices = _raw_value(getattr(output_message, "raw", None), "choices") + if isinstance(choices, list) and choices: + return choices[0] + return None + + +def _reasoning_from_raw(output_message: Any) -> str | None: + message = _raw_value(_first_choice(output_message), "message") + reasoning = _raw_value(message, "reasoning_content") + return reasoning if isinstance(reasoning, str) and reasoning else None + + +def _tool_call_requests(output_message: Any) -> list[ToolCallRequest]: + # ChatMessage.__post_init__ coerces every entry into a + # ChatMessageToolCall, so the id/function/name/arguments are all present. + tool_calls = getattr(output_message, "tool_calls", None) or [] + return [ + ToolCallRequest( + name=tool_call.function.name, + id=tool_call.id, + arguments=tool_call.function.arguments, + ) + for tool_call in tool_calls + ] + + +def _finish_reason(output_message: Any, has_tool_calls: bool) -> str | None: + """Why the provider stopped generating, or ``None`` if it didn't say. + + The local runtimes (``TransformersModel``, ``VLLMModel``, ``MLXModel``) put + ``{"out": ..., "completion_kwargs": ...}`` on ``raw`` and report no finish + reason at all. Defaulting those to ``"stop"`` would make a generation cut + short by ``max_new_tokens`` look like a natural stop. A response carrying + tool calls is the one case where the reason follows without guessing. + """ + reason = _raw_value(_first_choice(output_message), "finish_reason") + if isinstance(reason, str) and reason: + return reason + raw = getattr(output_message, "raw", None) + stop_reason = _raw_value(raw, "stopReason") + if isinstance(stop_reason, str) and stop_reason: + return _STOP_REASON_MAP.get(stop_reason, stop_reason) + return "tool_calls" if has_tool_calls else None + + +def to_output_message(output_message: Any) -> OutputMessage: + """Map a smolagents ``ChatMessage`` response to an ``OutputMessage``.""" + role = _unwrap_role(getattr(output_message, "role", None)) or "assistant" + parts: list[Any] = _parts_from_content( + getattr(output_message, "content", None) + ) + if reasoning := _reasoning_from_raw(output_message): + parts.append(Reasoning(content=reasoning)) + tool_call_requests = _tool_call_requests(output_message) + parts.extend(tool_call_requests) + # OutputMessage requires the field; util-genai drops an empty value when it + # emits gen_ai.response.finish_reasons. + reason = _finish_reason(output_message, bool(tool_call_requests)) + return OutputMessage(role=role, parts=parts, finish_reason=reason or "") + + +def response_id(output_message: Any) -> str | None: + """Extract ``gen_ai.response.id`` from the provider response on ``.raw``.""" + value = _raw_value(getattr(output_message, "raw", None), "id") + return value if isinstance(value, str) and value else None + + +def response_model_name(output_message: Any) -> str | None: + """Extract ``gen_ai.response.model`` from the provider response on ``.raw``.""" + value = _raw_value(getattr(output_message, "raw", None), "model") + return value if isinstance(value, str) and value else None + + +def _tool_parameters(tool: Any) -> dict[str, Any] | None: + """Return the JSON Schema ``parameters`` object for a smolagents tool. + + A tool's ``inputs`` map is not a JSON Schema on its own: smolagents wraps it + in an object schema, derives ``required`` from ``nullable``, and rewrites its + non-JSON-Schema ``"any"`` type. ``get_tool_json_schema`` builds exactly the + schema the provider receives. + """ + try: + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + get_tool_json_schema, + ) + except ImportError: + _logger.debug("smolagents.models.get_tool_json_schema is unavailable") + return None + try: + schema = get_tool_json_schema(tool) + parameters = schema["function"]["parameters"] + except Exception: # pylint: disable=broad-except + _logger.debug( + "Failed to build a JSON Schema for tool %s", + getattr(tool, "name", None), + exc_info=True, + ) + return None + return parameters if isinstance(parameters, dict) else None + + +def to_tool_definitions(tools: Any) -> list[ToolDefinition] | None: + """Map smolagents tool objects to function tool definitions.""" + if not isinstance(tools, list) or not tools: + return None + definitions: list[ToolDefinition] = [] + for tool in tools: + name = getattr(tool, "name", None) + if not name: + continue + definitions.append( + FunctionToolDefinition( + name=name, + description=getattr(tool, "description", None), + parameters=_tool_parameters(tool), + ) + ) + return definitions or None diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py index b56883424..fa7223304 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py @@ -2,3 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 _instruments = ("smolagents >= 1.24.0",) + +_supports_metrics = True diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py new file mode 100644 index 000000000..4653108aa --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -0,0 +1,436 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""wrapt wrapper factories for smolagents instrumentation. + +Each factory takes the shared :class:`TelemetryHandler` and returns a wrapper +suitable for :func:`wrapt.wrap_function_wrapper`: + +- :func:`model_generate` wraps each defining ``Model.generate`` -> ``chat`` span. +- :func:`model_generate_stream` wraps each defining ``Model.generate_stream`` + -> ``chat`` span, held open until the stream is drained. + +Original library exceptions are always re-raised unmodified; telemetry is +finalized via ``invocation.stop()`` / ``invocation.fail(exc)``. +""" + +from __future__ import annotations + +import logging +from collections.abc import Generator, Mapping +from dataclasses import dataclass +from inspect import signature +from typing import Any, Callable + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import InferenceInvocation +from opentelemetry.util.genai.stream import SyncStreamWrapper +from opentelemetry.util.genai.types import OutputMessage, Text, ToolCallRequest + +from ._messages import ( + response_id, + response_model_name, + to_input_messages, + to_output_message, + to_tool_definitions, +) +from .provider import resolve_provider, resolve_server_address_port + +_logger = logging.getLogger(__name__) + +_Wrapper = Callable[ + [Callable[..., Any], Any, tuple[Any, ...], dict[str, Any]], Any +] + + +def _bind_arguments( + wrapped: Callable[..., Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> dict[str, Any]: + """Bind call args to the wrapped callable's signature, applying defaults. + + smolagents passes the interesting arguments positionally + (``model.generate(input_messages)``), so binding is what makes them + readable by name. On a binding failure the keyword arguments are returned + on their own, without the positional ones and without the defaults. + """ + try: + bound = signature(wrapped).bind(*args, **kwargs) + bound.apply_defaults() + return dict(bound.arguments) + except (TypeError, ValueError): + _logger.debug( + "Failed to bind arguments of %s; falling back to keyword arguments", + getattr(wrapped, "__qualname__", wrapped), + exc_info=True, + ) + return dict(kwargs) + + +def _coerce_float(value: Any) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return None + + +def _coerce_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + return None + + +def _remove_parameter_sentinel() -> Any: + """Return smolagents' ``REMOVE_PARAMETER`` sentinel, or ``None`` if absent.""" + try: + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + REMOVE_PARAMETER, + ) + except ImportError: + _logger.debug("smolagents.models.REMOVE_PARAMETER is unavailable") + return None + return REMOVE_PARAMETER + + +# Model classes that never forward ``stop_sequences``, whatever +# ``supports_stop_parameter`` answers. ``AmazonBedrockModel`` overrides +# ``_prepare_completion_kwargs`` and calls the base with a hardcoded +# ``stop_sequences=None`` (``models.py``), so its ``converse`` request carries +# no stop sequences at all. ``_forwards_stop_sequences`` matches these names +# along the MRO, so subclasses are covered too. +_MODELS_DROPPING_STOP_SEQUENCES = frozenset({"AmazonBedrockModel"}) + + +def _forwards_stop_sequences(instance: Any) -> bool: + if any( + cls.__name__ in _MODELS_DROPPING_STOP_SEQUENCES + for cls in type(instance).__mro__ + ): + return False + return bool(getattr(instance, "supports_stop_parameter", False)) + + +def _merged_request_kwargs( + instance: Any, bound: dict[str, Any] +) -> dict[str, Any]: + """Rebuild the request keyword arguments smolagents will send. + + ``Model._prepare_completion_kwargs`` seeds ``stop`` from the + ``stop_sequences`` argument and ``response_format`` from its own argument, + applies the per-call ``**kwargs`` on top and the model-level ``self.kwargs`` + last, so a key set at two levels reaches the provider with the model-level + value and a model-level ``REMOVE_PARAMETER`` drops it from the request + entirely. Following that order here is what keeps a removed key off the span + as well. + """ + merged: dict[str, Any] = {} + stop_sequences = bound.get("stop_sequences") + if stop_sequences is not None and _forwards_stop_sequences(instance): + merged["stop"] = stop_sequences + response_format = bound.get("response_format") + if response_format is not None: + merged["response_format"] = response_format + call_kwargs = bound.get("kwargs") + if isinstance(call_kwargs, dict): + merged.update(call_kwargs) + model_kwargs = getattr(instance, "kwargs", None) + if not isinstance(model_kwargs, dict): + return merged + remove = _remove_parameter_sentinel() + for name, value in model_kwargs.items(): + if remove is not None and value is remove: + merged.pop(name, None) + else: + merged[name] = value + return merged + + +def _stop_sequences(merged: dict[str, Any]) -> list[str] | None: + """Return the stop sequences the request carries, if any.""" + stop = merged.get("stop", merged.get("stop_sequences")) + if isinstance(stop, str): + return [stop] + if isinstance(stop, list): + return [str(item) for item in stop] + return None + + +# smolagents' ``response_format`` type -> ``gen_ai.output.type`` value, for the +# types whose provider spelling differs from the semconv one. The openai +# instrumentation maps the same two. +_OUTPUT_TYPE_MAP: dict[str, str] = { + "json_object": GenAI.GenAiOutputTypeValues.JSON.value, + "json_schema": GenAI.GenAiOutputTypeValues.JSON.value, +} + +_OUTPUT_TYPE_VALUES = frozenset( + value.value for value in GenAI.GenAiOutputTypeValues +) + + +def _output_type(merged: dict[str, Any]) -> str | None: + """Map the request's ``response_format`` to ``gen_ai.output.type``. + + smolagents forwards ``response_format`` to the provider unchanged, so its + ``type`` is whatever the provider accepts. Only the values the semconv + defines are recorded; a provider-specific one is dropped rather than put on + an enum attribute. + """ + response_format = merged.get("response_format") + if not isinstance(response_format, Mapping): + return None + format_type = response_format.get("type") + if not isinstance(format_type, str): + return None + output_type = _OUTPUT_TYPE_MAP.get(format_type, format_type) + if output_type not in _OUTPUT_TYPE_VALUES: + _logger.debug( + "No gen_ai.output.type value for response_format type %r", + format_type, + ) + return None + return output_type + + +def _apply_request_parameters( + invocation: InferenceInvocation, instance: Any, bound: dict[str, Any] +) -> None: + """Copy the request parameters smolagents will send onto the span.""" + merged = _merged_request_kwargs(instance, bound) + + invocation.temperature = _coerce_float(merged.get("temperature")) + invocation.top_p = _coerce_float(merged.get("top_p")) + invocation.top_k = _coerce_float(merged.get("top_k")) + invocation.frequency_penalty = _coerce_float( + merged.get("frequency_penalty") + ) + invocation.presence_penalty = _coerce_float(merged.get("presence_penalty")) + # TransformersModel takes the generation limit as max_new_tokens (which its + # constructor defaults to 4096) and treats max_tokens as an alias for it. + invocation.max_tokens = _coerce_int( + merged.get("max_tokens", merged.get("max_new_tokens")) + ) + invocation.seed = _coerce_int(merged.get("seed")) + invocation.stop_sequences = _stop_sequences(merged) + invocation.output_type = _output_type(merged) + + +def _apply_token_usage( + invocation: InferenceInvocation, output_message: Any +) -> None: + # ChatMessage.token_usage is the only source: the per-model + # last_input_token_count / last_output_token_count counters were removed + # before the oldest supported smolagents. It is None for the local runtimes + # (TransformersModel, VLLMModel, MLXModel), which report no usage. + token_usage = getattr(output_message, "token_usage", None) + if token_usage is None: + return + invocation.input_tokens = token_usage.input_tokens + invocation.output_tokens = token_usage.output_tokens + + +def _start_inference( + handler: TelemetryHandler, + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> InferenceInvocation: + """Start the ``chat`` span and record the request. + + ``generate`` and ``generate_stream`` take the same parameters. + """ + provider = resolve_provider(instance) + server_address, server_port = resolve_server_address_port(instance) + invocation = handler.inference( + provider, + request_model=getattr(instance, "model_id", None), + server_address=server_address, + server_port=server_port, + ) + bound = _bind_arguments(wrapped, args, kwargs) + _apply_request_parameters(invocation, instance, bound) + invocation.tool_definitions = to_tool_definitions( + bound.get("tools_to_call_from") + ) + if handler.should_capture_content(): + invocation.input_messages = to_input_messages(bound.get("messages")) + return invocation + + +def model_generate(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate`` to emit a ``chat`` span.""" + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + + try: + output_message = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + + _apply_token_usage(invocation, output_message) + invocation.response_model_name = response_model_name(output_message) + invocation.response_id = response_id(output_message) + output = to_output_message(output_message) + # to_output_message leaves finish_reason empty when the provider + # reported none, and an empty value is dropped rather than guessed at. + if output.finish_reason: + invocation.finish_reasons = [output.finish_reason] + if handler.should_capture_content(): + invocation.output_messages = [output] + invocation.stop() + return output_message + + return wrapper + + +@dataclass +class _StreamedToolCall: + """A tool call assembled from stream deltas.""" + + id: str | None = None + name: str = "" + arguments: str = "" + + +class _ModelStreamWrapper(SyncStreamWrapper[Any]): + """Keep the ``chat`` span open until the delta stream is drained. + + Passing the invocation to ``super().__init__()`` turns on + ``gen_ai.request.stream`` and the per-chunk timing metrics. + """ + + def __init__( + self, + stream: Generator[Any, Any, Any], + invocation: InferenceInvocation, + handler: TelemetryHandler, + ) -> None: + super().__init__(stream, invocation=invocation) + self._self_inference = invocation + self._self_handler = handler + self._self_content: list[str] = [] + self._self_tool_calls: dict[int, _StreamedToolCall] = {} + self._self_input_tokens = 0 + self._self_output_tokens = 0 + self._self_saw_token_usage = False + + def _accumulate_tool_call(self, delta: Any) -> None: + index = getattr(delta, "index", None) + if not isinstance(index, int): + # agglomerate_stream_deltas raises here; telemetry must not. + _logger.debug("Dropping a tool call delta that carries no index") + return + tool_call = self._self_tool_calls.setdefault( + index, _StreamedToolCall() + ) + if delta.id: + tool_call.id = delta.id + function = getattr(delta, "function", None) + if function is None: + return + if function.name: + tool_call.name = function.name + if function.arguments: + tool_call.arguments += function.arguments + + def _process_chunk(self, chunk: Any) -> None: + content = getattr(chunk, "content", None) + if content: + self._self_content.append(content) + token_usage = getattr(chunk, "token_usage", None) + if token_usage is not None: + # Summed like agglomerate_stream_deltas, so the span agrees with the + # totals the agent's monitor reports. + self._self_saw_token_usage = True + self._self_input_tokens += token_usage.input_tokens + self._self_output_tokens += token_usage.output_tokens + for delta in getattr(chunk, "tool_calls", None) or []: + self._accumulate_tool_call(delta) + + def _output_message(self) -> OutputMessage | None: + parts: list[Any] = [] + content = "".join(self._self_content) + if content: + parts.append(Text(content=content)) + parts.extend( + ToolCallRequest( + name=tool_call.name, + id=tool_call.id, + arguments=tool_call.arguments or None, + ) + for tool_call in self._self_tool_calls.values() + ) + if not parts: + # Closed before it was drained, so there is no response to report. + return None + # Deltas carry no finish reason, so tool calls are the only evidence. + # Defaulting to "stop" would hide a generation cut short by a token + # limit. + finish_reason = "tool_calls" if self._self_tool_calls else "" + return OutputMessage( + role="assistant", parts=parts, finish_reason=finish_reason + ) + + def _finalize(self, error: BaseException | None = None) -> None: + invocation = self._self_inference + if self._self_saw_token_usage: + invocation.input_tokens = self._self_input_tokens + invocation.output_tokens = self._self_output_tokens + output = self._output_message() + if output is not None: + if output.finish_reason: + invocation.finish_reasons = [output.finish_reason] + if self._self_handler.should_capture_content(): + invocation.output_messages = [output] + if error is not None: + invocation.fail(error) + else: + invocation.stop() + + def _on_stream_end(self) -> None: + self._finalize() + + def _on_stream_error(self, error: BaseException) -> None: + # Records what was streamed before the failure. + self._finalize(error) + + +def model_generate_stream(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate_stream`` to emit a ``chat`` span. + + ``stream_outputs=True`` routes an agent's model calls here. The span stays + open until the caller drains the deltas. + """ + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + + try: + stream = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + + return _ModelStreamWrapper(stream, invocation, handler) + + return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py new file mode 100644 index 000000000..f531606c1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py @@ -0,0 +1,151 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve a smolagents model instance to a ``gen_ai.provider.name`` value. + +Resolution order: + +1. For the LiteLLM model classes, the ``model_id`` vendor prefix + (``anthropic/claude-...`` -> ``anthropic``). +2. The model class name (e.g. ``OpenAIModel`` -> ``openai``), looked up along the + class hierarchy so a user subclass resolves to the provider of the base class + whose ``generate`` it inherits. +3. ``unknown``. + +``gen_ai.provider.name`` is a metric attribute as well as a span attribute, so +every value has to stay low cardinality: deployment-specific detail is reported +as ``server.address`` instead. ``TelemetryHandler.inference`` requires +``provider`` as a string, so this always returns a value rather than ``None``. +""" + +from __future__ import annotations + +import logging +from typing import Any +from urllib.parse import urlparse + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + +_logger = logging.getLogger(__name__) + +_PROVIDER = GenAI.GenAiProviderNameValues + +_UNKNOWN_PROVIDER = "unknown" + +# Model class name -> provider value. The GenAI registry has no value for the +# Hugging Face, vLLM, and MLX runtimes, so those use the product name; a class +# name would look like a provider without being one. +_CLASS_NAME_TO_PROVIDER: dict[str, str] = { + "OpenAIModel": _PROVIDER.OPENAI.value, + "AzureOpenAIModel": _PROVIDER.AZURE_AI_OPENAI.value, + "AmazonBedrockModel": _PROVIDER.AWS_BEDROCK.value, + "InferenceClientModel": "huggingface", + "TransformersModel": "huggingface", + "VLLMModel": "vllm", + "MLXModel": "mlx", +} + +# LiteLLM model_id prefix -> semconv provider value, for the prefixes whose +# LiteLLM vendor slug differs from the semconv value. Every other prefix is +# passed through as-is (``ollama/llama3`` -> ``ollama``). LiteLLM's slugs are a +# closed vocabulary, which keeps the cardinality bounded. +_LITELLM_PREFIX_TO_PROVIDER: dict[str, str] = { + "azure": _PROVIDER.AZURE_AI_OPENAI.value, + "azure_ai": _PROVIDER.AZURE_AI_INFERENCE.value, + "bedrock": _PROVIDER.AWS_BEDROCK.value, + "gemini": _PROVIDER.GCP_GEMINI.value, + "mistral": _PROVIDER.MISTRAL_AI.value, + "vertex_ai": _PROVIDER.GCP_VERTEX_AI.value, + "watsonx": _PROVIDER.IBM_WATSONX_AI.value, + "xai": _PROVIDER.X_AI.value, +} + +_LITELLM_CLASS_NAMES = frozenset({"LiteLLMModel", "LiteLLMRouterModel"}) + + +def _class_names(instance: Any) -> list[str]: + """The instance's class names, most derived first. + + Only the classes that define ``generate`` are patched, so an instrumented + model can be a user subclass of one of them. Matching the exact class name + alone would report ``unknown`` for every such subclass. + """ + return [cls.__name__ for cls in type(instance).__mro__] + + +def _provider_from_litellm(instance: Any) -> str | None: + model_id = getattr(instance, "model_id", None) + if not isinstance(model_id, str) or "/" not in model_id: + return None + prefix = model_id.split("/", 1)[0].lower() + return _LITELLM_PREFIX_TO_PROVIDER.get(prefix, prefix) + + +def _endpoint(instance: Any) -> str | None: + """The endpoint URL the model calls, or ``None`` if it exposes none. + + Three places can hold it, checked in this order: + + 1. ``api_base`` on the instance (``LiteLLMModel``). + 2. ``base_url`` or ``azure_endpoint`` in the SDK client kwargs + (``OpenAIModel``, ``AzureOpenAIModel``, ``InferenceClientModel``). + 3. ``base_url`` on the client the model constructed (e.g. openai's httpx + URL), which holds the effective URL when the caller left it at the + provider default. + """ + api_base = getattr(instance, "api_base", None) + if api_base: + return str(api_base) + client_kwargs = getattr(instance, "client_kwargs", None) + if isinstance(client_kwargs, dict): + for key in ("base_url", "azure_endpoint"): + value = client_kwargs.get(key) + if value: + return str(value) + client_base_url = getattr( + getattr(instance, "client", None), "base_url", None + ) + if client_base_url: + return str(client_base_url) + return None + + +def resolve_server_address_port( + instance: Any, +) -> tuple[str | None, int | None]: + """Return ``(server.address, server.port)`` from the model's endpoint URL. + + Models that don't expose an ``api_base`` / ``base_url`` / ``azure_endpoint`` + (e.g. ``LiteLLMModel`` resolving the host internally, local runtimes) + yield ``(None, None)`` and the caller omits the attributes. + """ + endpoint = _endpoint(instance) + if endpoint is None: + return None, None + parsed = urlparse(endpoint) + port = parsed.port + if port == 443: + port = None + return parsed.hostname or None, port + + +def resolve_provider(instance: Any) -> str: + """Return the ``gen_ai.provider.name`` value for a smolagents model instance.""" + class_names = _class_names(instance) + + if not _LITELLM_CLASS_NAMES.isdisjoint(class_names): + provider = _provider_from_litellm(instance) + if provider is not None: + return provider + + for class_name in class_names: + provider = _CLASS_NAME_TO_PROVIDER.get(class_name) + if provider is not None: + return provider + + _logger.debug( + "No known gen_ai.provider.name for model class %s", class_names[0] + ) + return _UNKNOWN_PROVIDER diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml new file mode 100644 index 000000000..6177ff059 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml @@ -0,0 +1,20 @@ +interactions: +- request: + body: '{"model": "claude-3-7-sonnet-20250219", "messages": [{"role": "user", "content": + [{"type": "text", "text": "Who won the World Cup in 2018? Answer in one word + with no punctuation."}]}], "thinking": {"type": "enabled", "budget_tokens": + 4000}, "max_tokens": 8096}' + headers: {} + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"id":"msg_011KX7d4TtALbugymC3Kb4oE","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[{"type":"thinking","thinking":"The + World Cup in 2018 was won by France. They defeated Croatia 4-2 in the final + match in Moscow, Russia.\n\nI need to answer in one word with no punctuation, + so my answer should simply be:\nFrance","signature":"ErUBCkYIBBgCIkAOLwDXty2UNgzsRPd4O0tNxqhaxPqqLw9isc2bDqyVS7Y87Cefkib8FVE9iTTjTSynEjrrHb+vei4K5pQrVOUrEgzCGfpi49gURMlFYTwaDGTEJyP22/PoNkje2CIwyiqcZqRj6rxDwzG0ayl3JQ2mi8x3iDIEuyP92Pw19EPRhATbpYiORzbaVoPUBuPEKh3gUYNSZwcjUAYNO01Qqo7GYFF08xO0MuULheotLRgC"},{"type":"text","text":"France"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":54,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":65,"service_tier":"standard"}}' + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml new file mode 100644 index 000000000..44f70cc65 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml @@ -0,0 +1,25 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"Who won the + World Cup in 2018? Answer in one word with no punctuation."}]}],"model":"gpt-4o","max_tokens":4096}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb\",\n \"object\": + \"chat.completion\",\n \"created\": 1738649686,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"France\",\n \"refusal\": null\n + \ },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n + \ ],\n \"usage\": {\n \"prompt_tokens\": 25,\n \"completion_tokens\": + 2,\n \"total_tokens\": 27,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_4691090a87\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml new file mode 100644 index 000000000..9fde9a56e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml @@ -0,0 +1,26 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What breed + is this dog?"},{"type":"image_url","image_url":{"url":"https://fastly.picsum.photos/id/237/200/300.jpg?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U"}}]}],"model":"gpt-4o"}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-DVJBfdzKCrLtPYM9ui8SZgIEQv857\",\n \"object\": + \"chat.completion\",\n \"created\": 1776354295,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"This looks like a Labrador Retriever + puppy. They are known for their friendly and outgoing nature.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 268,\n \"completion_tokens\": 18,\n \"total_tokens\": 286,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_07a5e8f420\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml new file mode 100644 index 000000000..a5a57c069 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml @@ -0,0 +1,32 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What is the + weather in Paris?"}]}],"model":"gpt-4o","max_tokens":4096,"tool_choice":"required","tools": + [{"type":"function","function":{"name":"get_weather","description":"Get the weather for a + given city","parameters":{"type":"object","properties":{"location":{"type":"string","description": + "The city to get the weather for"}},"required":["location"]}}}]}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ax7BZfEQe2evzgqbTVXWo0ZqoMIUf\",\n \"object\": + \"chat.completion\",\n \"created\": 1738652337,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_EUuviydGIG5Jau3DLw4v4cue\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n + \ \"arguments\": \"{\\\"location\\\":\\\"Paris\\\"}\"\n }\n + \ }\n ],\n \"refusal\": null\n },\n \"logprobs\": + null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": + {\n \"prompt_tokens\": 61,\n \"completion_tokens\": 15,\n \"total_tokens\": + 76,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n + \ \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_50cad350e4\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py new file mode 100644 index 000000000..be0d0071e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py @@ -0,0 +1,34 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for smolagents conformance scenarios.""" + +from __future__ import annotations + +import json +from typing import Any + + +def attr(span: dict[str, Any], name: str) -> Any: + for entry in span["attributes"]: + if entry["name"] == name: + return entry["value"] + return None + + +def chat_spans(report: Any) -> list[dict[str, Any]]: + return [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + + +def part_fields(messages_json: str | None) -> list[tuple[str, str | None]]: + messages = json.loads(messages_json) if messages_json else [] + return [ + (part["type"], part.get("modality")) + for message in messages + for part in message["parts"] + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py new file mode 100644 index 000000000..f9e60fb0f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py @@ -0,0 +1,113 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios for the ``chat`` operation (plain and tool-calling).""" + +from __future__ import annotations + +from typing import Any + +from smolagents.models import ChatMessage, MessageRole + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +from ..test_utils import GetWeatherTool, openai_model # noqa: TID252 +from ._helpers import attr, chat_spans, part_fields + + +class ChatScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_basic.yaml"): + openai_model().generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? Answer " + "in one word with no punctuation." + ), + } + ], + } + ] + ) + + +class ToolCallingScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_tool.yaml"): + openai_model().generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What is the weather in Paris?", + } + ], + ) + ], + tools_to_call_from=[GetWeatherTool()], + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_part_types = { + part_type + for span in chat_spans(report) + for part_type, _ in part_fields( + attr(span, "gen_ai.output.messages") + ) + } + assert "tool_call" in output_part_types, ( + f"expected a tool_call output part, saw {output_part_types}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py new file mode 100644 index 000000000..5e8025c1b --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py @@ -0,0 +1,163 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios for non-text message parts: an image ``uri`` on a chat +input, and a ``reasoning`` part on a chat output.""" + +from __future__ import annotations + +import os +from typing import Any +from unittest import mock + +from smolagents import LiteLLMModel, OpenAIModel +from smolagents.models import ChatMessage, MessageRole + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) +from opentelemetry.test_util_genai.instrumentor import instrument + +from ._helpers import attr, chat_spans, part_fields + +_IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +class MultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_image_url.yaml"): + model = OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What breed is this dog?", + }, + { + "type": "image_url", + "image_url": {"url": _IMAGE_URL}, + }, + ], + ) + ] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + input_parts = { + fields + for span in chat_spans(report) + for fields in part_fields(attr(span, "gen_ai.input.messages")) + } + assert ("uri", "image") in input_parts, ( + f"expected an image uri input part, saw {input_parts}" + ) + + +class ReasoningScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = ( + # LiteLLM routes to the provider host internally and a LiteLLMModel + # built without an explicit api_base exposes no endpoint URL, so there + # is nothing to derive server.address from on the chat span. + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + env = {"LITELLM_LOCAL_MODEL_COST_MAP": "True"} + with ( + mock.patch.dict(os.environ, env), + mock.patch("tiktoken.get_encoding") as get_encoding, + ): + get_encoding.return_value = mock.MagicMock( + encode=lambda *_: [1, 2, 3] + ) + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("litellm_reasoning.yaml"): + model = LiteLLMModel( + model_id="anthropic/claude-3-7-sonnet-20250219", + api_key="test_anthropic_api_key", + thinking={"type": "enabled", "budget_tokens": 4000}, + ) + model.generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? " + "Answer in one word with no " + "punctuation." + ), + } + ], + } + ] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_parts = { + part_type + for span in chat_spans(report) + for part_type, _ in part_fields( + attr(span, "gen_ai.output.messages") + ) + } + assert "reasoning" in output_parts, ( + f"expected a reasoning output part, saw {output_parts}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py index da97f2987..baebcbf9c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py @@ -5,22 +5,103 @@ from __future__ import annotations +import os +from unittest.mock import MagicMock, patch + import pytest from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) from opentelemetry.test_util_genai.instrumentor import instrument +from opentelemetry.test_util_genai.vcr import ( + scrub_response_headers_overwrite, +) + +pytest_plugins = [ + "opentelemetry.test_util_genai.fixtures", + "opentelemetry.test_util_genai.vcr", +] + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_openai_api_key"), + ("x-api-key", "test_anthropic_api_key"), + ("openai-organization", "test_openai_org_id"), + ("openai-project", "test_openai_project_id"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers_overwrite( + { + "openai-organization": "test_openai_org_id", + "openai-project": "test_openai_project_id", + "Set-Cookie": "test_set_cookie", + } + ), + } + + +@pytest.fixture +def litellm_local_cost_map(): + """Use LiteLLM's bundled model-cost map so it doesn't fetch prices over the + network during cassette playback.""" + previous = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + try: + yield + finally: + if previous is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = previous + + +@pytest.fixture +def patch_tiktoken_encoding(): + """Patch ``tiktoken.get_encoding`` so LiteLLM doesn't download an encoding.""" + with patch("tiktoken.get_encoding") as mock_get_encoding: + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + mock_get_encoding.return_value = mock_encoding + yield + + +@pytest.fixture +def instrument_no_content(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="NO_CONTENT", + ) as instrumentor: + yield instrumentor -pytest_plugins = ["opentelemetry.test_util_genai.fixtures"] + +@pytest.fixture +def instrument_with_content(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ) as instrumentor: + yield instrumentor @pytest.fixture -def instrument_smolagents(tracer_provider, logger_provider, meter_provider): +def instrument_event_only(tracer_provider, logger_provider, meter_provider): with instrument( SmolagentsInstrumentor(), tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, + content_capture="EVENT_ONLY", + emit_event=True, ) as instrumentor: yield instrumentor diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt index d833435b7..649ffc343 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt @@ -26,7 +26,10 @@ # This variant of the requirements aims to test the system using the newest # supported version of external dependencies. -smolagents +# openai/litellm are the model backends exercised by the VCR model tests; they +# are test-only (not declared in pyproject.toml) so they live here. +smolagents[openai,litellm] +wrapt>=2.2.2 -e util/opentelemetry-util-genai -e instrumentation/opentelemetry-instrumentation-genai-smolagents diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt index 1d9b353f3..afb682e96 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt @@ -21,5 +21,15 @@ # pyproject.toml is the single source of truth. The OpenTelemetry SDK and test utilities # come transitively from opentelemetry-test-util-genai. # -# There is nothing to pin yet: the lifecycle tests need no model backend. The pins for -# the backends the model tests drive arrive with those tests. +# openai and litellm are the model backends the VCR model tests instantiate. They are +# test-only (not declared in pyproject.toml) and smolagents does not pull them in through +# the instruments extra, so pin them here to the versions smolagents 1.24.0 declares in its +# own openai/litellm extras. litellm resolves to its floor; openai resolves to 1.61.0 +# rather than 1.58.1 because litellm 1.60.2 itself requires openai>=1.61.0. +openai>=1.58.1 +litellm>=1.60.2 + +# The declared opentelemetry-util-genai floor carries the streaming metrics and the +# gen_ai.request.stream attribute, which are not in a published release yet. +# Drop this once opentelemetry-util-genai 1.1b0 is published. +-e util/opentelemetry-util-genai diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py new file mode 100644 index 000000000..41c60d2db --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py @@ -0,0 +1,46 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-scenario conformance tests for smolagents.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +# Skip collection when weaver_live_check or OTLP exporters aren't installed +# (non-conformance envs). +pytest.importorskip("opentelemetry.test.weaver_live_check") +pytest.importorskip("opentelemetry.exporter.otlp.proto.grpc") + +from opentelemetry.test.weaver_live_check import WeaverLiveCheck # noqa: E402 +from opentelemetry.test_util_genai.conformance import ( # noqa: E402 + Scenario, + run_conformance, +) + +from .conformance.inference import ( # noqa: E402 + ChatScenario, + ToolCallingScenario, +) +from .conformance.multimodal import ( # noqa: E402 + MultimodalScenario, + ReasoningScenario, +) + + +@pytest.mark.parametrize( + "scenario", + [ + pytest.param(ChatScenario()), + pytest.param(ToolCallingScenario()), + pytest.param(MultimodalScenario()), + pytest.param(ReasoningScenario()), + ], + ids=lambda s: type(s).__name__, +) +def test_conformance( + scenario: Scenario, vcr: Any, weaver_live_check: WeaverLiveCheck +) -> None: + run_conformance(scenario, vcr=vcr, weaver=weaver_live_check) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py index 1f4b69905..d0c654181 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py @@ -1,19 +1,40 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Entry-point and instrument/uninstrument lifecycle tests. - -The instrumentor patches nothing yet, so these cover the parts of its contract -that hold before any patching lands: the entry point resolves, the declared -dependency is reported, and repeated instrument/uninstrument cycles stay quiet. -""" +"""Lifecycle, entry-point, completion-hook, and restoration tests.""" from __future__ import annotations +from typing import Any +from unittest.mock import patch + +import pytest +import smolagents +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, + _model_classes_defining, ) +from opentelemetry.test_util_genai.instrumentor import instrument from opentelemetry.util._importlib_metadata import entry_points +from opentelemetry.util.genai.completion_hook import CompletionHook + +from .test_utils import openai_model, stub_openai_client + + +class RecordingHook(CompletionHook): + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def on_completion(self, **kwargs: Any) -> None: + self.calls.append(kwargs) + + +def _generate_a_chat_span() -> None: + model = openai_model() + model.client = stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) def test_entrypoint_loads_instrumentor() -> None: @@ -29,45 +50,42 @@ def test_instrumentation_dependencies() -> None: assert "smolagents >= 1.24.0" in dependencies -def test_instrument_uninstrument_cycle( +def test_instrument_uninstrument_restores_originals( tracer_provider, logger_provider, meter_provider ) -> None: + original_generate = smolagents.OpenAIModel.generate + original_base_generate = smolagents.Model.generate + original_generate_stream = smolagents.OpenAIModel.generate_stream + instrumentor = SmolagentsInstrumentor() instrumentor.instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, ) - assert instrumentor.is_instrumented_by_opentelemetry - instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is not original_generate + assert smolagents.Model.generate is not original_base_generate + assert ( + smolagents.OpenAIModel.generate_stream is not original_generate_stream + ) + instrumentor.uninstrument() -def test_repeated_instrument_uninstrument( - tracer_provider, logger_provider, meter_provider -) -> None: - # BaseInstrumentor returns a per-class singleton, so the lifecycle has to - # survive being driven more than once. - instrumentor = SmolagentsInstrumentor() - for _ in range(2): - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - assert instrumentor.is_instrumented_by_opentelemetry - instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.Model.generate is original_base_generate + assert smolagents.OpenAIModel.generate_stream is original_generate_stream def test_uninstrument_through_a_new_constructor_call( tracer_provider, logger_provider, meter_provider ) -> None: - # BaseInstrumentor.__new__ returns a per-class singleton but Python still - # runs __init__ on every construction, so the documented + # BaseInstrumentor is a per-class singleton, so the documented # SmolagentsInstrumentor().instrument() / SmolagentsInstrumentor() - # .uninstrument() form has to work on the live instance. + # .uninstrument() form must restore everything even though the second + # constructor call re-runs __init__ on the live instance. + original_generate = smolagents.OpenAIModel.generate + SmolagentsInstrumentor().instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, @@ -75,23 +93,201 @@ def test_uninstrument_through_a_new_constructor_call( ) SmolagentsInstrumentor().uninstrument() - assert not SmolagentsInstrumentor().is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is original_generate + + +@pytest.mark.parametrize("method", ["generate", "generate_stream"]) +def test_model_classes_defining(method: str) -> None: + classes = _model_classes_defining(smolagents, method) + + # Each class object appears once, whatever names it is exported under. + assert len(classes) == len(set(classes)) + # Every entry owns the method, so no class is wrapped for one it only + # inherits. + for model_cls in classes: + assert method in model_cls.__dict__ + + # The API-backed model classes define both; the classes that inherit them + # are reached through the base they inherit from. + assert { + smolagents.OpenAIModel, + smolagents.LiteLLMModel, + smolagents.InferenceClientModel, + } <= set(classes) + assert smolagents.AzureOpenAIModel not in classes + assert smolagents.LiteLLMRouterModel not in classes + + +def test_only_generate_covers_the_base_class_and_bedrock() -> None: + # The base Model and AmazonBedrockModel define generate but no + # generate_stream, so streaming is patched on fewer classes. + generate = set(_model_classes_defining(smolagents, "generate")) + generate_stream = set( + _model_classes_defining(smolagents, "generate_stream") + ) + + assert {smolagents.Model, smolagents.AmazonBedrockModel} <= generate + assert generate_stream.isdisjoint( + {smolagents.Model, smolagents.AmazonBedrockModel} + ) + + +def test_repeated_instrument_uninstrument( + tracer_provider, logger_provider, meter_provider +) -> None: + # BaseInstrumentor returns a per-class singleton, so the wrapped-class + # bookkeeping has to survive being filled and drained more than once. + original_generate = smolagents.OpenAIModel.generate + + instrumentor = SmolagentsInstrumentor() + for _ in range(2): + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + assert smolagents.OpenAIModel.generate is not original_generate + instrumentor.uninstrument() + assert smolagents.OpenAIModel.generate is original_generate def test_uninstrument_without_instrument() -> None: # BaseInstrumentor.uninstrument() short-circuits, but _uninstrument() must - # also be a no-op on its own: the rollback path in _instrument() calls it - # after a partial patch. + # also be a no-op on unpatched attributes: the rollback in _instrument() + # calls it after a partial patch. + original_generate = smolagents.OpenAIModel.generate + SmolagentsInstrumentor().uninstrument() SmolagentsInstrumentor()._uninstrument() # noqa: SLF001 + assert smolagents.OpenAIModel.generate is original_generate + def test_instrument_with_no_providers() -> None: # Without providers the handler falls back to the globals; instrumenting # must not require a caller to pass them. + original_generate = smolagents.OpenAIModel.generate + instrumentor = SmolagentsInstrumentor() instrumentor.instrument() try: - assert instrumentor.is_instrumented_by_opentelemetry + assert smolagents.OpenAIModel.generate is not original_generate finally: instrumentor.uninstrument() + + assert smolagents.OpenAIModel.generate is original_generate + + +def test_failed_instrument_rolls_back_partial_patches( + tracer_provider, logger_provider, meter_provider +) -> None: + # A failure part-way through must leave no class patched, because + # uninstrument() cannot clean up after a failed _instrument(). + model_classes = _model_classes_defining(smolagents, "generate") + assert len(model_classes) > 1, ( + "the rollback needs more than one class to patch" + ) + originals = { + model_cls: model_cls.__dict__["generate"] + for model_cls in model_classes + } + stream_classes = _model_classes_defining(smolagents, "generate_stream") + stream_originals = { + model_cls: model_cls.__dict__["generate_stream"] + for model_cls in stream_classes + } + + real_wrap = wrap_function_wrapper + calls = 0 + + def fail_on_the_second_class(target: Any, name: str, wrapper: Any) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("boom") + real_wrap(target, name, wrapper) + + with patch( + "opentelemetry.instrumentation.genai.smolagents.wrap_function_wrapper", + fail_on_the_second_class, + ): + with pytest.raises(RuntimeError, match="boom"): + SmolagentsInstrumentor().instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + + assert calls == 2, "the first class was expected to be patched" + for model_cls, original in originals.items(): + assert model_cls.__dict__["generate"] is original + for model_cls, original in stream_originals.items(): + assert model_cls.__dict__["generate_stream"] is original + + +def test_inherited_generate_wrapped_only_on_defining_classes( + tracer_provider, logger_provider, meter_provider +) -> None: + # AzureOpenAIModel and LiteLLMRouterModel inherit generate; they must not be + # wrapped separately or they would emit duplicate chat spans. + assert "generate" not in smolagents.AzureOpenAIModel.__dict__ + assert "generate" not in smolagents.LiteLLMRouterModel.__dict__ + + instrumentor = SmolagentsInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + try: + # wrapt returns a fresh bound wrapper per attribute access, so the + # wrapper objects differ; the underlying wrapped function is shared, + # proving AzureOpenAIModel inherits the single wrapped generate. + assert ( + smolagents.AzureOpenAIModel.generate.__wrapped__ + is smolagents.OpenAIModel.generate.__wrapped__ + ) + finally: + instrumentor.uninstrument() + + +def test_explicit_completion_hook_takes_precedence( + tracer_provider, logger_provider, meter_provider, span_exporter +) -> None: + explicit_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook" + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + completion_hook=explicit_hook, + ): + _generate_a_chat_span() + + load_hook.assert_not_called() + assert explicit_hook.calls, "explicit completion hook was not invoked" + + +def test_env_completion_hook_used_when_no_explicit_hook( + tracer_provider, logger_provider, meter_provider +) -> None: + env_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook", + return_value=env_hook, + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + _generate_a_chat_span() + + load_hook.assert_called_once() + assert env_hook.calls, "env-resolved completion hook was not invoked" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py new file mode 100644 index 000000000..ca72d4e86 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -0,0 +1,1296 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Model (``chat``) instrumentation tests: VCR-backed runs of the real +smolagents model classes, provider/endpoint resolution, request-parameter +mapping, and message conversion. +""" + +from __future__ import annotations + +import inspect +import json +import sys +from collections.abc import Generator +from types import ModuleType, SimpleNamespace +from typing import Any + +import pytest +from smolagents import LiteLLMModel, OpenAIModel +from smolagents.models import ( + ChatMessage, + ChatMessageToolCall, + ChatMessageToolCallFunction, + MessageRole, +) + +from opentelemetry.instrumentation.genai.smolagents._messages import ( + response_id, + response_model_name, + to_input_messages, + to_output_message, + to_tool_definitions, +) +from opentelemetry.instrumentation.genai.smolagents.provider import ( + resolve_provider, + resolve_server_address_port, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import ( + error_attributes, + server_attributes, +) +from opentelemetry.trace import StatusCode +from opentelemetry.util.genai.types import ( + Blob, + Reasoning, + Text, + ToolCallRequest, + Uri, +) + +from .test_utils import ( + GetWeatherTool, + attr, + data_point_attributes, + metrics_by_name, + openai_model, + parse_messages, + part_types, + spans_by_operation, + stub_openai_client, + stub_streaming_openai_client, + text_chunk, + tool_call_chunk, + usage_chunk, +) + +IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +def test_openai_model_basic( + instrument_with_content, span_exporter, metric_reader, vcr +) -> None: + model = openai_model() + text = "Who won the World Cup in 2018? Answer in one word with no punctuation." + with vcr.use_cassette("openai_model_basic.yaml"): + output = model.generate( + messages=[ + {"role": "user", "content": [{"type": "text", "text": text}]} + ] + ) + assert output.content == "France" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.name == "chat gpt-4o" + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 25 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + assert ( + attr(span, GenAI.GEN_AI_RESPONSE_ID) + == "chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb" + ) + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) == "gpt-4o-2024-08-06" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert attr(span, server_attributes.SERVER_ADDRESS) == "api.openai.com" + assert attr(span, server_attributes.SERVER_PORT) is None + + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0]["role"] == "user" + assert inputs[0]["parts"][0] == {"type": "text", "content": text} + + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["role"] == "assistant" + assert outputs[0]["parts"][0] == {"type": "text", "content": "France"} + + metrics = metrics_by_name(metric_reader) + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + assert duration.unit == "s" + assert data_point_attributes(duration) == [ + { + GenAI.GEN_AI_OPERATION_NAME: "chat", + GenAI.GEN_AI_PROVIDER_NAME: "openai", + GenAI.GEN_AI_REQUEST_MODEL: "gpt-4o", + GenAI.GEN_AI_RESPONSE_MODEL: "gpt-4o-2024-08-06", + server_attributes.SERVER_ADDRESS: "api.openai.com", + } + ] + token_usage = metrics[gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE] + assert { + point.attributes[GenAI.GEN_AI_TOKEN_TYPE]: point.sum + for point in token_usage.data.data_points + } == {"input": 25, "output": 2} + + +def test_openai_model_no_content( + instrument_no_content, span_exporter, vcr +) -> None: + model = openai_model() + with vcr.use_cassette("openai_model_basic.yaml"): + model.generate( + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Hi"}]} + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert isinstance(attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS), int) + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_openai_model_image_url( + instrument_with_content, span_exporter, vcr +) -> None: + model = openai_model() + with vcr.use_cassette("openai_model_image_url.yaml"): + model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + {"type": "text", "text": "What breed is this dog?"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + ], + ) + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + parts = inputs[0]["parts"] + assert parts[0] == {"type": "text", "content": "What breed is this dog?"} + assert parts[1]["type"] == "uri" + assert parts[1]["modality"] == "image" + assert parts[1]["uri"] == IMAGE_URL + + +def test_openai_model_with_tools( + instrument_with_content, span_exporter, vcr +) -> None: + model = openai_model() + with vcr.use_cassette("openai_model_tool.yaml"): + output = model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What is the weather in Paris?", + } + ], + ) + ], + tools_to_call_from=[GetWeatherTool()], + ) + assert output.tool_calls[0].function.name == "get_weather" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_DEFINITIONS)) == [ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather for a given city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + }, + } + ] + + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + tool_call_parts = [ + part for part in outputs[0]["parts"] if part["type"] == "tool_call" + ] + assert tool_call_parts[0]["name"] == "get_weather" + # smolagents hands the provider's raw argument payload through unparsed. + assert tool_call_parts[0]["arguments"] == '{"location":"Paris"}' + assert tool_call_parts[0]["id"] == "call_EUuviydGIG5Jau3DLw4v4cue" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) + + +def _litellm_supports_reasoning() -> bool: + """litellm surfaces Anthropic ``reasoning_content`` only from ~1.63 onward. + + The oldest supported litellm (smolagents' 1.60.2 floor) doesn't parse the + Anthropic thinking blocks into ``reasoning_content``, so the reasoning part + can't be mapped there. Gate the reasoning-specific assertion on the version. + """ + from importlib.metadata import version # noqa: PLC0415 + + parts = version("litellm").split(".") + try: + return (int(parts[0]), int(parts[1])) >= (1, 63) + except (IndexError, ValueError): + return True + + +def test_litellm_reasoning( + instrument_with_content, + span_exporter, + litellm_local_cost_map, + patch_tiktoken_encoding, + vcr, +) -> None: + model = LiteLLMModel( + model_id="anthropic/claude-3-7-sonnet-20250219", + api_key="test_anthropic_api_key", + thinking={"type": "enabled", "budget_tokens": 4000}, + ) + with vcr.use_cassette("litellm_reasoning.yaml"): + model.generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? Answer in one " + "word with no punctuation." + ), + } + ], + } + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "anthropic" + assert ( + attr(span, GenAI.GEN_AI_REQUEST_MODEL) + == "anthropic/claude-3-7-sonnet-20250219" + ) + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + if _litellm_supports_reasoning(): + assert "reasoning" in part_types(outputs) + + +def test_model_generate_reraises_and_records_error( + instrument_with_content, span_exporter +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model(model_id="broken-model") + with pytest.raises(NotImplementedError): + model.generate(messages=[{"role": "user", "content": "hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "NotImplementedError" + + +def test_inherited_azure_generate_emits_one_chat_span( + instrument_with_content, span_exporter +) -> None: + # AzureOpenAIModel inherits generate from OpenAIModel, so only the defining + # class is patched. Exercise the inherited method end to end to prove the + # single patch still produces exactly one span with the Azure provider. + from smolagents import AzureOpenAIModel # noqa: PLC0415 + + model = AzureOpenAIModel( + model_id="gpt-4o-deployment", + azure_endpoint="https://example-resource.openai.azure.com", + api_key="test_azure_api_key", + api_version="2024-10-21", + ) + model.client = stub_openai_client("Bonjour") + + output = model.generate(messages=[{"role": "user", "content": "Hi"}]) + assert output.content == "Bonjour" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "azure.ai.openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o-deployment" + assert ( + attr(span, server_attributes.SERVER_ADDRESS) + == "example-resource.openai.azure.com" + ) + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 1 + + +def _fake_model(class_name: str, **attrs: Any) -> Any: + instance = type(class_name, (), {})() + for key, value in attrs.items(): + setattr(instance, key, value) + return instance + + +@pytest.mark.parametrize( + "class_name, attrs, expected", + [ + ("OpenAIModel", {}, "openai"), + ("AzureOpenAIModel", {}, "azure.ai.openai"), + ("AmazonBedrockModel", {}, "aws.bedrock"), + # No GenAI registry value exists for these runtimes, so the product + # name is used rather than the class name. + ("InferenceClientModel", {}, "huggingface"), + ("TransformersModel", {}, "huggingface"), + ("VLLMModel", {}, "vllm"), + ("MLXModel", {}, "mlx"), + # LiteLLM vendor prefixes: remapped where the slug differs from the + # semconv value, passed through otherwise. + ("LiteLLMModel", {"model_id": "anthropic/claude-3"}, "anthropic"), + ("LiteLLMModel", {"model_id": "mistral/large"}, "mistral_ai"), + ("LiteLLMModel", {"model_id": "xai/grok"}, "x_ai"), + ("LiteLLMModel", {"model_id": "gemini/gemini-2.0"}, "gcp.gemini"), + ( + "LiteLLMModel", + {"model_id": "vertex_ai/gemini-2.0"}, + "gcp.vertex_ai", + ), + ("LiteLLMModel", {"model_id": "watsonx/granite"}, "ibm.watsonx.ai"), + ( + "LiteLLMModel", + {"model_id": "azure_ai/phi-4"}, + "azure.ai.inference", + ), + ("LiteLLMModel", {"model_id": "ollama/llama3"}, "ollama"), + # LiteLLMRouterModel takes a model-group name, not a provider/model + # slug, so there is nothing to resolve. + ("LiteLLMRouterModel", {"model_id": "model-group-1"}, "unknown"), + # gen_ai.provider.name is also a metric attribute. An unmapped model + # must not fall back to the deployment-specific host or a class name. + ( + "CustomModel", + {"api_base": "https://llm.example.com/v1"}, + "unknown", + ), + ("CustomModel", {}, "unknown"), + ], +) +def test_resolve_provider( + class_name: str, attrs: dict[str, Any], expected: str +) -> None: + assert resolve_provider(_fake_model(class_name, **attrs)) == expected + + +@pytest.mark.parametrize( + "model_class, expected", + [ + ("OpenAIModel", "openai"), + ("AzureOpenAIModel", "azure.ai.openai"), + ("AmazonBedrockModel", "aws.bedrock"), + ("InferenceClientModel", "huggingface"), + ("TransformersModel", "huggingface"), + ("VLLMModel", "vllm"), + ("MLXModel", "mlx"), + ("LiteLLMModel", "unknown"), + ("LiteLLMRouterModel", "unknown"), + ], +) +def test_resolve_provider_covers_every_real_model_class( + model_class: str, expected: str +) -> None: + # The mapping is keyed by class name, so pin it against the real classes + # rather than only against synthetic stand-ins. + import smolagents # noqa: PLC0415 + + instance = object.__new__(getattr(smolagents, model_class)) + assert resolve_provider(instance) == expected + + +@pytest.mark.parametrize( + "attrs, expected", + [ + ({"api_base": "https://api.openai.com/v1"}, ("api.openai.com", None)), + # The default HTTPS port is omitted per the semconv server.port guidance. + ( + {"api_base": "https://api.openai.com:443/v1"}, + ("api.openai.com", None), + ), + ({"api_base": "http://localhost:11434/v1"}, ("localhost", 11434)), + ( + { + "client_kwargs": { + "azure_endpoint": "https://x.openai.azure.com" + } + }, + ("x.openai.azure.com", None), + ), + ({}, (None, None)), + ], +) +def test_resolve_server_address_port( + attrs: dict[str, Any], expected: tuple[str | None, int | None] +) -> None: + assert resolve_server_address_port(_fake_model("M", **attrs)) == expected + + +def test_server_address_falls_back_to_the_sdk_client() -> None: + # The common configuration: no api_base, so the URL is only known to the + # client the model built for itself. + model = OpenAIModel(model_id="gpt-4o", api_key="test_openai_api_key") + assert resolve_server_address_port(model) == ("api.openai.com", None) + + +def test_request_parameters_recorded( + instrument_with_content, span_exporter +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model( + model_id="broken-model", + temperature=0.5, + top_p=0.9, + top_k=40, + frequency_penalty=0.25, + presence_penalty=1, + max_tokens=256, + seed=7, + ) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + stop_sequences=[""], + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.5 + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.9 + # top_k is a float attribute in the spec even though callers pass an int. + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_K) == 40.0 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_TOP_K), float) + assert attr(span, GenAI.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.25 + assert attr(span, GenAI.GEN_AI_REQUEST_PRESENCE_PENALTY) == 1.0 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == 256 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS), int) + assert attr(span, GenAI.GEN_AI_REQUEST_SEED) == 7 + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == ("",) + + +def test_model_kwargs_win_over_call_kwargs( + instrument_with_content, span_exporter +) -> None: + # _prepare_completion_kwargs applies the call kwargs first and the model + # kwargs on top, and drops any key whose model-level value is the + # REMOVE_PARAMETER sentinel. + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model = Model( + model_id="broken-model", + temperature=0.1, + max_tokens=REMOVE_PARAMETER, + ) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + temperature=0.9, + max_tokens=512, + top_p=0.5, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.1 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) is None + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.5 + + +@pytest.mark.parametrize( + "model_id, model_kwargs, expected", + [ + # gpt-5 doesn't accept `stop`, so smolagents truncates the generated + # text locally instead of sending the sequences. + ("gpt-5", {}, None), + ("gpt-4o", {}, ("",)), + # An explicit `stop` overrides the stop_sequences argument. + ("gpt-4o", {"stop": ["STOP"]}, ("STOP",)), + # The model-level sentinel pops the `stop` that + # _prepare_completion_kwargs seeded from stop_sequences, leaving the + # request with none. + ("gpt-4o", {"stop": "REMOVE"}, None), + ], +) +def test_stop_sequences_follow_what_is_sent( + instrument_with_content, + span_exporter, + model_id: str, + model_kwargs: dict[str, Any], + expected: tuple[str, ...] | None, +) -> None: + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + model = Model(model_id=model_id, **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + stop_sequences=[""], + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == expected + + +def _bedrock_model(response: dict[str, Any]) -> Any: + from smolagents import AmazonBedrockModel # noqa: PLC0415 + + # A caller-supplied client keeps boto3 out of the test. + return AmazonBedrockModel( + model_id="us.amazon.nova-pro-v1:0", + client=SimpleNamespace(converse=lambda **_: response), + ) + + +BEDROCK_RESPONSE: dict[str, Any] = { + "output": { + "message": { + "role": "assistant", + "content": [{"text": "done"}], + "tool_calls": None, + } + }, + "usage": {"inputTokens": 3, "outputTokens": 2}, + "stopReason": "end_turn", +} + + +def test_bedrock_stop_sequences_are_not_recorded( + instrument_with_content, span_exporter +) -> None: + # supports_stop_parameter says yes, but the prepared request carries no + # stop sequences, so the span must not claim any either. + model = _bedrock_model(BEDROCK_RESPONSE) + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + request = model._prepare_completion_kwargs( # noqa: SLF001 + messages=messages, stop_sequences=[""] + ) + assert model.supports_stop_parameter is True + assert "stop" not in request + + model.generate(messages=messages, stop_sequences=[""]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) is None + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "aws.bedrock" + # Bedrock's "end_turn" is normalized to the semconv "stop". + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + + +@pytest.mark.parametrize( + "model_kwargs, call_kwargs, expected", + [ + ({"max_tokens": 256}, {}, 256), + ({}, {"max_tokens": 256}, 256), + # max_new_tokens is the TransformersModel spelling of the same limit, + # and max_tokens wins when a caller sets both. + ({"max_new_tokens": 4096}, {}, 4096), + ({"max_new_tokens": 4096, "max_tokens": 256}, {}, 256), + ({}, {}, None), + ], +) +def test_max_tokens_covers_both_spellings( + instrument_with_content, + span_exporter, + model_kwargs: dict[str, Any], + call_kwargs: dict[str, Any], + expected: int | None, +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model(model_id="broken-model", **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], **call_kwargs + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == expected + + +@pytest.mark.parametrize( + "response_format, model_kwargs, expected", + [ + ({"type": "json_object"}, {}, "json"), + ({"type": "json_schema", "json_schema": {}}, {}, "json"), + ({"type": "text"}, {}, "text"), + (None, {}, None), + # smolagents forwards response_format unchanged, so its type is + # whatever the provider accepts. An unknown one is dropped rather than + # recorded on an enum attribute. + ({"type": "xml"}, {}, None), + ({}, {}, None), + # The model-level kwargs win over the argument, and the sentinel drops + # the key from the request, the same as for every other parameter. + ( + {"type": "json_object"}, + {"response_format": {"type": "text"}}, + "text", + ), + ({"type": "json_object"}, {"response_format": "REMOVE"}, None), + (None, {"response_format": {"type": "json_object"}}, "json"), + ], +) +def test_output_type_follows_the_response_format( + instrument_with_content, + span_exporter, + response_format: dict[str, Any] | None, + model_kwargs: dict[str, Any], + expected: str | None, +) -> None: + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + model = Model(model_id="broken-model", **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + response_format=response_format, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_OUTPUT_TYPE) == expected + + +def test_user_subclass_of_a_patched_model_keeps_its_provider( + instrument_with_content, span_exporter, metric_reader +) -> None: + # A subclass inherits the patched generate, so it is instrumented; resolving + # the provider by exact class name would report "unknown" on both the span + # and the metrics. + class TenantOpenAIModel(OpenAIModel): + pass + + model = TenantOpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="http://localhost:11434/v1", + ) + model.client = stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + # A non-default port is part of the endpoint, unlike the HTTPS default. + assert attr(span, server_attributes.SERVER_ADDRESS) == "localhost" + assert attr(span, server_attributes.SERVER_PORT) == 11434 + duration = metrics_by_name(metric_reader)[ + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + ] + assert data_point_attributes(duration)[0][GenAI.GEN_AI_PROVIDER_NAME] == ( + "openai" + ) + + +def test_provider_error_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_openai_client( + "", error=ConnectionError("connection reset") + ) + + with pytest.raises(ConnectionError, match="connection reset"): + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def _drain_stream(model: OpenAIModel, **kwargs: Any) -> list[Any]: + return list( + model.generate_stream( + messages=[{"role": "user", "content": "Hi"}], **kwargs + ) + ) + + +def test_generate_stream_is_lazy_and_records_the_drained_response( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] + ) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + # A streamed response isn't finished until the caller drains it. + assert spans_by_operation(span_exporter.get_finished_spans(), "chat") == [] + + deltas = list(stream) + assert "".join(delta.content or "" for delta in deltas) == "Bonjour" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" + assert attr(span, GenAI.GEN_AI_REQUEST_STREAM) is True + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "Bonjour"}] + # Deltas carry no finish reason, and "stop" would hide a truncation. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def test_generate_stream_stays_a_generator( + instrument_with_content, span_exporter +) -> None: + # Instrumentation observes; it must not change what generate_stream returns. + model = openai_model() + model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + assert isinstance(stream, Generator) + assert inspect.isgenerator(stream) + list(stream) + + +def test_generate_stream_accumulates_tool_calls( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [ + tool_call_chunk(0, call_id="call_1", name="get_weather"), + tool_call_chunk(0, arguments='{"location":'), + tool_call_chunk(0, arguments='"Paris"}'), + ] + ) + + _drain_stream(model, tools_to_call_from=[GetWeatherTool()]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + (part,) = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES)[0]["parts"] + assert part == { + "type": "tool_call", + "id": "call_1", + "name": "get_weather", + "arguments": '{"location":"Paris"}', + } + # Tool calls are the only evidence of why a streamed generation stopped. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) + + +def test_generate_stream_error_mid_iteration_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon")], error=ConnectionError("stream died") + ) + + with pytest.raises(ConnectionError, match="stream died"): + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" + # What was streamed before the failure is still recorded. + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "Bon"}] + + +def test_generate_stream_close_before_drain_finalizes_once( + instrument_with_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + stream.close() + stream.close() # idempotent + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_generate_stream_records_chunk_metrics( + instrument_with_content, span_exporter, metric_reader +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] + ) + + _drain_stream(model) + + metrics = metrics_by_name(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metrics + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE in metrics + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK in metrics + ) + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_PER_OUTPUT_CHUNK in metrics + ) + + +def test_generate_stream_no_content( + instrument_no_content, span_exporter +) -> None: + model = openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bonjour"), usage_chunk(3, 2)] + ) + + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + + +def test_event_only_content_capture( + instrument_event_only, span_exporter, log_exporter +) -> None: + model = openai_model() + model.client = stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + # Metadata still goes on the span. + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.event_name == "gen_ai.client.inference.operation.details" + # Event attributes carry the messages as structured values, not JSON text. + attributes = record.attributes or {} + inputs = attributes[GenAI.GEN_AI_INPUT_MESSAGES] + outputs = attributes[GenAI.GEN_AI_OUTPUT_MESSAGES] + assert inputs[0]["parts"][0]["content"] == "Hi" + assert outputs[0]["parts"][0]["content"] == "Bonjour" + assert outputs[0]["finish_reason"] == "stop" + + +def test_to_tool_definitions_uses_json_schema() -> None: + (definition,) = to_tool_definitions([GetWeatherTool()]) + assert definition.name == "get_weather" + assert definition.description == "Get the weather for a given city" + # smolagents' raw ``inputs`` map is not a JSON Schema on its own. + assert definition.parameters == { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + } + + +def test_to_input_messages_maps_smolagents_only_roles() -> None: + # smolagents converts these roles itself inside generate(), after the + # wrapper has already read the messages. + messages = to_input_messages( + [ + {"role": MessageRole.TOOL_CALL, "content": "call"}, + {"role": MessageRole.TOOL_RESPONSE, "content": "response"}, + {"role": MessageRole.SYSTEM, "content": "sys"}, + ] + ) + assert [message.role for message in messages] == [ + "assistant", + "user", + "system", + ] + + +def test_to_input_messages_dict_and_chatmessage() -> None: + messages = to_input_messages( + [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + ChatMessage(role=MessageRole.ASSISTANT, content="Hi there"), + ] + ) + assert messages[0].role == "user" + assert messages[0].parts == [Text(content="Hello")] + assert messages[1].role == "assistant" + assert messages[1].parts == [Text(content="Hi there")] + + +def test_to_input_messages_image_and_base64() -> None: + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image", "image": "aVZCT1J3MEtHZ28="}, + ], + } + ] + ) + parts = messages[0].parts + assert parts[0] == Uri(mime_type=None, modality="image", uri=IMAGE_URL) + assert isinstance(parts[1], Blob) + assert parts[1].modality == "image" + assert parts[1].mime_type == "image/png" + assert isinstance(parts[1].content, bytes) + + +def test_to_input_messages_data_url_keeps_media_type() -> None: + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + { + "type": "image", + "image": "data:image/jpeg;base64,aVZCT1J3MEtHZ28=", + } + ], + } + ] + ) + (part,) = messages[0].parts + assert isinstance(part, Blob) + assert part.mime_type == "image/jpeg" + + +@pytest.mark.parametrize( + "image", + [ + "not base64!!!!", + # A path is not base64, but a non-validating decode silently turns this + # one into 15 bytes of garbage after dropping the invalid characters. + "/tmp/photos/my-cat.png", + ], +) +def test_to_input_messages_drops_malformed_base64_image(image: str) -> None: + messages = to_input_messages( + [{"role": "user", "content": [{"type": "image", "image": image}]}] + ) + assert messages[0].parts == [] + + +def test_to_output_message_text_reasoning_and_tool_calls() -> None: + class _Msg: + role = MessageRole.ASSISTANT + content = "The answer" + tool_calls = [ + ChatMessageToolCall( + id="call_1", + type="function", + function=ChatMessageToolCallFunction( + name="get_weather", arguments='{"location": "Paris"}' + ), + ) + ] + + class raw: # noqa: N801 + class _Choice: + class message: # noqa: N801 + reasoning_content = "thinking about it" + + choices = [_Choice()] + + output = to_output_message(_Msg()) + assert output.role == "assistant" + assert Text(content="The answer") in output.parts + assert Reasoning(content="thinking about it") in output.parts + tool_calls = [p for p in output.parts if isinstance(p, ToolCallRequest)] + assert tool_calls[0].name == "get_weather" + assert tool_calls[0].id == "call_1" + assert output.finish_reason == "tool_calls" + + +def test_to_output_message_from_a_dict_raw_response() -> None: + # AmazonBedrockModel, TransformersModel, VLLMModel and MLXModel put a plain + # dict on ChatMessage.raw rather than an SDK object. + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="done", + raw={"stopReason": "max_tokens"}, + ) + output = to_output_message(message) + assert output.parts == [Text(content="done")] + # "max_tokens" is Bedrock's stopReason spelling for a length cutoff. + assert output.finish_reason == "length" + assert response_id(message) is None + assert response_model_name(message) is None + + +def test_to_output_message_unwraps_the_role_enum() -> None: + output = to_output_message( + ChatMessage(role=MessageRole.ASSISTANT, content="done") + ) + assert output.role == "assistant" + + +def test_to_output_message_maps_image_content() -> None: + # A response whose content is a list carries the same element shapes as a + # request, so an image in it maps the same way. + output = to_output_message( + ChatMessage( + role=MessageRole.ASSISTANT, + content=[ + {"type": "text", "text": "Here it is"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image", "image": "aVZCT1J3MEtHZ28="}, + ], + ) + ) + assert output.parts[0] == Text(content="Here it is") + assert output.parts[1] == Uri( + mime_type=None, modality="image", uri=IMAGE_URL + ) + blob = output.parts[2] + assert isinstance(blob, Blob) + assert blob.mime_type == "image/png" + assert blob.modality == "image" + + +LOCAL_RUNTIME_RAW = { + "out": "done", + "completion_kwargs": {"max_new_tokens": 4096}, +} + + +@pytest.mark.parametrize( + "raw, tool_calls, expected", + [ + # API-backed models: the provider's own value, whatever it is. + ( + SimpleNamespace(choices=[SimpleNamespace(finish_reason="stop")]), + [], + "stop", + ), + ( + SimpleNamespace(choices=[SimpleNamespace(finish_reason="length")]), + [], + "length", + ), + # Bedrock's stopReason values map onto the semconv vocabulary. + ({"stopReason": "max_tokens"}, [], "length"), + ({"stopReason": "tool_use"}, [], "tool_calls"), + # An unmapped value passes through rather than being guessed at. + ({"stopReason": "guardrail_intervened"}, [], "guardrail_intervened"), + # The local runtimes report no reason, so none is recorded and + # util-genai omits the empty value. + (LOCAL_RUNTIME_RAW, [], ""), + (None, [], ""), + # Tool calls in the response give the reason without guessing. + ( + LOCAL_RUNTIME_RAW, + [ + ChatMessageToolCall( + id="call_1", + type="function", + function=ChatMessageToolCallFunction( + name="get_weather", arguments="{}" + ), + ) + ], + "tool_calls", + ), + ], +) +def test_finish_reason_follows_the_provider_response( + raw: Any, tool_calls: list[ChatMessageToolCall], expected: str +) -> None: + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="done", + tool_calls=tool_calls or None, + raw=raw, + ) + assert to_output_message(message).finish_reason == expected + + +def test_model_reporting_no_finish_reason_omits_the_attribute( + instrument_with_content, span_exporter +) -> None: + # InferenceClientModel is a patched class whose provider response can come + # back without a finish reason; the span must then carry none. + from smolagents import InferenceClientModel # noqa: PLC0415 + + model = InferenceClientModel( + model_id="Qwen/Qwen2.5-Coder-32B-Instruct", token="hf_test" + ) + model.client = SimpleNamespace( + chat_completion=lambda **_: SimpleNamespace( + id="hf-1", + model="Qwen/Qwen2.5-Coder-32B-Instruct", + choices=[ + SimpleNamespace( + message=SimpleNamespace( + role="assistant", content="ok", tool_calls=None + ) + ) + ], + usage=SimpleNamespace(prompt_tokens=5, completion_tokens=2), + ) + ) + + model.generate(messages=[{"role": "user", "content": "hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["finish_reason"] == "" + + +# The local runtimes flatten a message's content as text, so the content has to +# be a list of parts rather than a bare string. +LOCAL_RUNTIME_MESSAGES: list[dict[str, Any]] = [ + { + "role": "user", + "content": [{"type": "text", "text": "Where is the Louvre?"}], + } +] + + +def _mlx_model(monkeypatch: pytest.MonkeyPatch) -> Any: + """An ``MLXModel`` whose ``mlx_lm`` pieces are stubbed. + + ``MLXModel.generate`` imports nothing itself; it drives ``stream_generate`` + over ``self.model`` and ``self.tokenizer``, which ``__init__`` loads from + ``mlx_lm``. Bypassing ``__init__`` is therefore enough to run the real + ``generate``, and the runtime doesn't have to be installed. ``monkeypatch`` + is unused here; it keeps the factory signature uniform with the vllm one, + which does have modules to stub. + """ + from smolagents.models import MLXModel # noqa: PLC0415 + + model = object.__new__(MLXModel) + model.model_id = "mlx-community/Qwen2.5-0.5B-Instruct-4bit" + model.kwargs = {} + model.flatten_messages_as_text = True + model.apply_chat_template_kwargs = {} + model.model = object() + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, tools=None, **_: [1, 2, 3] + ) + model.stream_generate = lambda *_, **__: iter( + [SimpleNamespace(text="In "), SimpleNamespace(text="Paris")] + ) + return model + + +def _vllm_model(monkeypatch: pytest.MonkeyPatch) -> Any: + """A ``VLLMModel`` with ``vllm`` itself stubbed. + + ``VLLMModel.generate`` imports ``SamplingParams`` and + ``StructuredOutputsParams`` from ``vllm`` when it runs, and neither test env + installs vllm, so both modules are faked for the duration of the test. + Everything the wrapper reads still comes from the real ``generate``. + """ + from smolagents.models import VLLMModel # noqa: PLC0415 + + def fake_params(**kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(**kwargs) + + vllm = ModuleType("vllm") + sampling_params = ModuleType("vllm.sampling_params") + setattr(vllm, "SamplingParams", fake_params) + setattr(sampling_params, "StructuredOutputsParams", fake_params) + setattr(vllm, "sampling_params", sampling_params) + monkeypatch.setitem(sys.modules, "vllm", vllm) + monkeypatch.setitem(sys.modules, "vllm.sampling_params", sampling_params) + + completion = SimpleNamespace( + prompt_token_ids=[1, 2, 3, 4], + outputs=[SimpleNamespace(text="In Paris", token_ids=[5, 6])], + ) + model = object.__new__(VLLMModel) + model.model_id = "Qwen/Qwen2.5-0.5B-Instruct" + model.kwargs = {} + model.flatten_messages_as_text = True + model._is_vlm = False + model.apply_chat_template_kwargs = {} + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, **_: "prompt" + ) + model.model = SimpleNamespace(generate=lambda *_, **__: [completion]) + return model + + +@pytest.mark.parametrize( + "model_factory, provider, request_model, input_tokens, output_tokens", + [ + ( + _mlx_model, + "mlx", + "mlx-community/Qwen2.5-0.5B-Instruct-4bit", + 3, + 2, + ), + (_vllm_model, "vllm", "Qwen/Qwen2.5-0.5B-Instruct", 4, 2), + ], + ids=["mlx", "vllm"], +) +def test_local_runtime_response_is_recorded( + instrument_with_content, + span_exporter, + monkeypatch: pytest.MonkeyPatch, + model_factory: Any, + provider: str, + request_model: str, + input_tokens: int, + output_tokens: int, +) -> None: + output = model_factory(monkeypatch).generate( + messages=LOCAL_RUNTIME_MESSAGES + ) + assert output.content == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == provider + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == request_model + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == input_tokens + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == output_tokens + # A local runtime returns no provider response envelope and listens on no + # socket, so it reports no finish reason, id or response model, and there is + # no endpoint to derive server.address from. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_ID) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) is None + assert attr(span, server_attributes.SERVER_ADDRESS) is None + assert attr(span, server_attributes.SERVER_PORT) is None + assert span.status.status_code == StatusCode.UNSET + + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0]["parts"] == [ + {"type": "text", "content": "Where is the Louvre?"} + ] + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["role"] == "assistant" + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] + assert outputs[0]["finish_reason"] == "" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py new file mode 100644 index 000000000..eeb8d78e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py @@ -0,0 +1,222 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers, tools, and model stubs for smolagents instrumentation tests.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import Any + +from smolagents import OpenAIModel, Tool + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + + +class GetWeatherTool(Tool): + name = "get_weather" + description = "Get the weather for a given city" + inputs = { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + } + output_type = "string" + + def forward(self, location: str) -> str: + return "sunny" + + +def openai_model() -> OpenAIModel: + return OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + + +def stub_openai_client( + content: str, finish_reason: str = "stop", error: Exception | None = None +) -> Any: + """An object shaped like the bits of ``openai.OpenAI`` that ``generate`` uses. + + Building a real ``ChatCompletion`` keeps the response shape honest (the + wrapper reads ``raw.model``, ``raw.id``, and ``raw.choices[0].finish_reason``) + without needing a cassette for a deployment we can't record against. + """ + from openai.types.chat import ( # noqa: PLC0415 + ChatCompletion, + ChatCompletionMessage, + ) + from openai.types.chat.chat_completion import Choice # noqa: PLC0415 + from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 + + completion = ChatCompletion( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion", + created=0, + choices=[ + Choice( + index=0, + finish_reason=finish_reason, + message=ChatCompletionMessage( + role="assistant", content=content + ), + ) + ], + usage=CompletionUsage( + prompt_tokens=3, completion_tokens=1, total_tokens=4 + ), + ) + + def create(**_: Any) -> ChatCompletion: + if error is not None: + raise error + return completion + + return SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + +def stub_streaming_openai_client( + chunks: list[Any], error: Exception | None = None +) -> Any: + """An ``openai.OpenAI`` stand-in whose ``create`` returns a chunk stream. + + ``OpenAIModel.generate_stream`` reads ``event.usage`` and + ``event.choices[0].delta``, so the chunks are real ``ChatCompletionChunk`` + objects. ``error`` is raised after the chunks are yielded, which is how a + provider failure part-way through a stream reaches the caller. + """ + + def create(**_: Any) -> Any: + def stream() -> Any: + yield from chunks + if error is not None: + raise error + + return stream() + + return SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + +def text_chunk(content: str) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 + Choice, + ChoiceDelta, + ) + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[Choice(index=0, delta=ChoiceDelta(content=content))], + ) + + +def tool_call_chunk( + index: int, + call_id: str | None = None, + name: str | None = None, + arguments: str | None = None, +) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 + Choice, + ChoiceDelta, + ChoiceDeltaToolCall, + ChoiceDeltaToolCallFunction, + ) + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[ + Choice( + index=0, + delta=ChoiceDelta( + tool_calls=[ + ChoiceDeltaToolCall( + index=index, + id=call_id, + type="function", + function=ChoiceDeltaToolCallFunction( + name=name, arguments=arguments + ), + ) + ] + ), + ) + ], + ) + + +def usage_chunk(prompt_tokens: int, completion_tokens: int) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[], + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +def spans_by_operation( + spans: list[ReadableSpan], operation: str +) -> list[ReadableSpan]: + return [ + span + for span in spans + if (span.attributes or {}).get(GenAI.GEN_AI_OPERATION_NAME) + == operation + ] + + +def attr(span: ReadableSpan, name: str) -> Any: + return (span.attributes or {}).get(name) + + +def parse_messages(span: ReadableSpan, name: str) -> list[dict[str, Any]]: + raw = attr(span, name) + return json.loads(raw) if isinstance(raw, str) else [] + + +def part_types(messages: list[dict[str, Any]]) -> list[str]: + return [part["type"] for message in messages for part in message["parts"]] + + +def metrics_by_name(metric_reader: Any) -> dict[str, Any]: + data = metric_reader.get_metrics_data() + if data is None: + return {} + return { + metric.name: metric + for resource_metric in data.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + } + + +def data_point_attributes(metric: Any) -> list[dict[str, Any]]: + return [dict(point.attributes) for point in metric.data.data_points] diff --git a/tox.ini b/tox.ini index ce3694aa0..b202ba514 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,7 @@ envlist = ; instrumentation-genai-smolagents py3{10,11,12,13,14}-test-instrumentation-genai-smolagents-latest py310-test-instrumentation-genai-smolagents-oldest + py314-test-instrumentation-genai-smolagents-conformance lint-instrumentation-genai-smolagents ; instrumentation-genai-anthropic @@ -157,6 +158,8 @@ deps = smolagents-latest: {[testenv]test_deps} smolagents-latest: {[testenv]pytest_deps} smolagents-latest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt + smolagents-conformance: {[testenv]pytest_deps} + smolagents-conformance: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt anthropic-oldest: {[testenv]pytest_deps} anthropic-oldest: -e {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic[instruments] @@ -257,6 +260,7 @@ commands = lint-instrumentation-genai-agno: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-agno" test-instrumentation-genai-smolagents-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests --vcr-record=none {posargs} + test-instrumentation-genai-smolagents-conformance: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py --vcr-record=none {posargs} lint-instrumentation-genai-smolagents: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-smolagents" test-instrumentation-genai-anthropic-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests --vcr-record=none {posargs}