From 67581e20d9acce84d6488516c0946221d2534a0f Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Fri, 31 Jul 2026 02:33:56 +0200 Subject: [PATCH] Add `opentelemetry-instrumentation-genai-smolagents` instrumentation package Port openinference-instrumentation-smolagents into a new opentelemetry-instrumentation-genai-smolagents package that emits telemetry only through the public opentelemetry-util-genai API. Instruments MultiStepAgent.run (invoke_agent, sync and streaming), the model classes that define generate or generate_stream (chat), and the tool classes that define __call__ (execute_tool). A streamed chat span stays open until the caller drains the deltas. ToolCallingAgent.process_tool_calls is wrapped without a span of its own: it carries the provider's tool call id down to the execute_tool spans of the step. Also patches local_python_executor.timeout so tool spans started from agent-generated code keep the agent span as their parent instead of becoming root spans. Tests run against smolagents 1.24.0, the declared floor, and the latest release. Known gaps: - No span for an agent step. Neither the semantic conventions nor util-genai have a step or chain concept, so the OpenInference "Step N" CHAIN spans are dropped and tool and inference spans nest directly under invoke_agent. Tracked in open-telemetry/semantic-conventions-genai#81. - A streamed chat span reports no gen_ai.response.id and no gen_ai.response.model, because a smolagents stream delta carries neither. It reports a finish reason only when the model requested tool calls. - gen_ai.tool.call.id is omitted for a CodeAgent, whose model writes code instead of tool calls, and when one step calls the same tool twice. Tool.__call__ receives only the argument values, so the id is matched by tool name, and two calls to one tool have no unambiguous match. - A user-defined Model subclass that overrides generate shadows the patched base method and emits no chat span. --- instrumentation/README.md | 2 +- .../.changelog/340.added | 1 + .../README.rst | 63 +- .../examples/manual/.env | 24 + .../examples/manual/README.rst | 58 + .../examples/manual/custom_hook.py | 94 ++ .../examples/manual/main.py | 72 + .../examples/manual/requirements.txt | 9 + .../examples/zero-code/.env | 27 + .../examples/zero-code/README.rst | 44 + .../examples/zero-code/main.py | 24 + .../examples/zero-code/requirements.txt | 10 + .../pyproject.toml | 2 +- .../genai/smolagents/__init__.py | 217 ++- .../genai/smolagents/_messages.py | 387 +++++ .../genai/smolagents/package.py | 2 + .../instrumentation/genai/smolagents/patch.py | 840 ++++++++++ .../genai/smolagents/provider.py | 151 ++ .../tests/cassettes/agent_with_image.yaml | 75 + .../tests/cassettes/litellm_reasoning.yaml | 20 + .../tests/cassettes/openai_model_basic.yaml | 25 + .../cassettes/openai_model_image_url.yaml | 26 + .../tests/cassettes/openai_model_tool.yaml | 32 + .../tests/conformance/__init__.py | 0 .../tests/conformance/_helpers.py | 34 + .../tests/conformance/agent.py | 84 + .../tests/conformance/inference.py | 122 ++ .../tests/conformance/multimodal.py | 163 ++ .../tests/conftest.py | 85 +- .../tests/fixtures/img.png | Bin 0 -> 3594 bytes .../tests/requirements.latest.txt | 5 +- .../tests/requirements.oldest.txt | 14 +- .../tests/test_agents.py | 704 +++++++++ .../tests/test_conformance.py | 48 + .../tests/test_instrumentor.py | 305 +++- .../tests/test_models.py | 1369 +++++++++++++++++ .../tests/test_tools.py | 559 +++++++ .../tests/test_utils.py | 279 ++++ tox.ini | 4 + 39 files changed, 5934 insertions(+), 46 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/340.added create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/.env create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/README.rst create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/custom_hook.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/main.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/requirements.txt create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/.env create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/README.rst create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/main.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/requirements.txt create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/agent_with_image.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/fixtures/img.png create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_tools.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py diff --git a/instrumentation/README.md b/instrumentation/README.md index 800e69732..83cf165a8 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -9,6 +9,6 @@ | [opentelemetry-instrumentation-genai-openai](./opentelemetry-instrumentation-genai-openai) | openai >= 1.26.0 | Yes | development | [opentelemetry-instrumentation-genai-openai-agents](./opentelemetry-instrumentation-genai-openai-agents) | openai-agents >= 0.3.3 | No | development | [opentelemetry-instrumentation-genai-qwen-agent](./opentelemetry-instrumentation-genai-qwen-agent) | qwen-agent >= 0.0.20 | No | development -| [opentelemetry-instrumentation-genai-smolagents](./opentelemetry-instrumentation-genai-smolagents) | smolagents >= 1.24.0 | No | development +| [opentelemetry-instrumentation-genai-smolagents](./opentelemetry-instrumentation-genai-smolagents) | smolagents >= 1.24.0 | Yes | development | [opentelemetry-instrumentation-genai-weaviate-client](./opentelemetry-instrumentation-genai-weaviate-client) | weaviate-client >= 3.0.0,<5.0.0 | No | development | [opentelemetry-instrumentation-google-genai](./opentelemetry-instrumentation-google-genai) | google-genai >= 1.32.0, <3 | No | development diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/340.added b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/340.added new file mode 100644 index 000000000..c2369a705 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/340.added @@ -0,0 +1 @@ +Add smolagents instrumentation package (``opentelemetry-instrumentation-genai-smolagents``). diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst index 932ad2c12..a7189e6cb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst @@ -7,7 +7,53 @@ OpenTelemetry smolagents Instrumentation :target: https://pypi.org/project/opentelemetry-instrumentation-genai-smolagents/ This library provides OpenTelemetry instrumentation for `smolagents -`_. +`_. It emits GenAI semantic-convention +telemetry (``invoke_agent``, ``chat``, and ``execute_tool`` spans plus the +associated metrics) through ``opentelemetry-util-genai``. + +Token usage is reported on the ``chat`` span of the model call that produced it, +not summed onto the ``invoke_agent`` span. + +A streamed model call, whether it comes from ``stream_outputs=True`` on an agent +or from calling ``Model.generate_stream`` directly, gets a ``chat`` span that +stays open until the caller drains the deltas. The span carries +``gen_ai.request.stream``, and the call also records the +``gen_ai.client.operation.time_to_first_chunk`` and +``gen_ai.client.operation.time_per_output_chunk`` metrics. + +Known gaps: + +* There is no span for an individual agent step, so ``chat`` and + ``execute_tool`` spans nest directly under ``invoke_agent``. The GenAI + semantic conventions define no operation for one iteration of a + reason-and-act loop. ``invoke_workflow`` is the closest name, but the + OpenAI Agents and LangChain instrumentations both use ``invoke_workflow`` + for the outermost orchestration, above ``invoke_agent``. A dedicated span is + proposed in + `semantic-conventions-genai#81 + `_. +* The instrumentation patches ``generate`` and ``generate_stream`` on the model + classes that smolagents ships. A subclass that inherits either method is + instrumented. A subclass that overrides one is not: the override shadows the + patched method, so the call produces no ``chat`` span. +* A streamed ``chat`` span reports no ``gen_ai.response.id`` and no + ``gen_ai.response.model``, because a smolagents stream delta carries neither. + The span reports ``gen_ai.response.finish_reasons`` only when the model + requested tool calls, which is the one stop reason the deltas make visible. +* The agent's step budget (``max_steps``) is not recorded. The GenAI semantic + conventions define no attribute for it, and a run that uses the budget up + already reports ``gen_ai.response.finish_reasons`` as ``length`` on the + ``invoke_agent`` span. +* ``gen_ai.tool.call.id`` is omitted when a single step calls the *same* tool + more than once. ``Tool.__call__`` receives only the argument values, so this + instrumentation matches a call to its id by tool name, and two calls to one + tool in the same step have no unambiguous match. A ``CodeAgent`` has no tool + call ids at all: its model writes code rather than tool calls. + +A ``CodeAgent`` with ``executor_type="local"`` runs generated code through +smolagents' ``local_python_executor.timeout()``, which this instrumentation +wraps so tool spans started in its worker thread stay nested under the agent +span. Installation ------------ @@ -24,10 +70,13 @@ Usage from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import CodeAgent, InferenceClientModel - # Instrument smolagents SmolagentsInstrumentor().instrument() + agent = CodeAgent(tools=[], model=InferenceClientModel()) + agent.run("How many seconds are in a week?") + Configuration ------------- @@ -71,6 +120,16 @@ environment variable: SmolagentsInstrumentor().instrument(completion_hook=my_hook) +See ``examples/manual/`` for a runnable example with the SDK configured in +code (including ``custom_hook.py`` for the completion hook) and +``examples/zero-code/`` for the ``opentelemetry-instrument`` variant. + +Conformance +----------- + +The scenarios that check this package against the GenAI semantic conventions +live under ``tests/conformance/``. + References ---------- diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/.env b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/.env new file mode 100644 index 000000000..ba122eb01 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/.env @@ -0,0 +1,24 @@ +# Update this with your real Hugging Face token +HF_TOKEN=hf_YOUR_TOKEN + +# Uncomment to run a different model through Hugging Face Inference Providers +# CHAT_MODEL=meta-llama/Llama-3.3-70B-Instruct + +# Uncomment and change to your OTLP endpoint +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +# OTEL_EXPORTER_OTLP_PROTOCOL=grpc + +OTEL_SERVICE_NAME=opentelemetry-python-smolagents + +# Remove to hide prompt and completion content +# Possible values (case insensitive): +# - `span_only` - record content on span attributes +# - `event_only` - record content on event attributes +# - `span_and_event` - record content on both span and event attributes +# - everything else - don't record content on any signal +OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=span_only + +# Uncomment to upload prompts and responses to an fsspec-compatible +# destination instead of or in addition to recording them inline on spans/events. +# OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK=upload +# OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH=/path/to/prompts diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/README.rst new file mode 100644 index 000000000..2f299a91e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/README.rst @@ -0,0 +1,58 @@ +OpenTelemetry smolagents Instrumentation Example +================================================ + +This is an example of how to instrument smolagents agent runs when configuring +the OpenTelemetry SDK and instrumentations manually. + +When `main.py `_ is run, it exports traces, metrics, and logs to an +OTLP compatible endpoint. The agent run produces an ``invoke_agent`` span with +``chat`` and ``execute_tool`` spans nested under it, so you can see which model +calls and tools each step of the run made. + +Note: `.env <.env>`_ file configures additional environment variables: + +- ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=span_only`` configures the smolagents instrumentation to capture prompt and completion contents on *span* attributes. +- ``OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK`` (commented out) - uncomment along with ``OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH`` to upload prompts and completions to an ``fsspec``-compatible destination instead of recording them inline. Also uncomment the ``opentelemetry-util-genai[upload]`` line in `requirements.txt `_ and reinstall. + +Setup +----- + +Minimally, update the `.env <.env>`_ file with your ``HF_TOKEN``. An OTLP +compatible endpoint should be listening for traces, metrics, and logs on +http://localhost:4317. If not, update ``OTEL_EXPORTER_OTLP_ENDPOINT`` as well. + +Next, set up a virtual environment like this: + +:: + + python3 -m venv .venv + source .venv/bin/activate + pip install "python-dotenv[cli]" + pip install -r requirements.txt + +Run +--- + +Run the example like this: + +:: + + dotenv run -- python main.py + +You should see the agent's answer while traces, metrics, and logs export to +your configured observability tool. + +Custom completion hook +---------------------- + +`custom_hook.py `_ is a variant of ``main.py`` that passes a +custom ``CompletionHook`` implementation programmatically via +``SmolagentsInstrumentor().instrument(completion_hook=...)``. The example hook +prints prompts and completions to stdout; real hooks typically forward content +to external storage and record reference URIs on the span/log_record. + +Run it the same way: + +:: + + dotenv run -- python custom_hook.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/custom_hook.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/custom_hook.py new file mode 100644 index 000000000..a7f6e6f0f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/custom_hook.py @@ -0,0 +1,94 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: skip-file +"""Instrument smolagents with a custom CompletionHook that prints prompts and +completions to stdout. + +Run with: dotenv run -- python custom_hook.py +""" + +import os + +from smolagents import CodeAgent, InferenceClientModel + +from opentelemetry import _logs, metrics, trace +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.util.genai.completion_hook import CompletionHook +from opentelemetry.util.genai.types import ( + InputMessage, + MessagePart, + OutputMessage, + ToolDefinition, +) + + +class PrintCompletionHook(CompletionHook): + """Minimal CompletionHook that prints inputs/outputs to stdout. + + Real hooks typically forward content to external storage (object store, + database, etc.) and record reference URIs on the span/log_record. + """ + + def on_completion( + self, + *, + inputs: list[InputMessage], + outputs: list[OutputMessage], + system_instruction: list[MessagePart], + tool_definitions: list[ToolDefinition] | None = None, + span=None, + log_record=None, + ) -> None: + print(f"[hook] inputs: {inputs}") + print(f"[hook] outputs: {outputs}") + + +trace.set_tracer_provider(TracerProvider()) +trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(OTLPSpanExporter()) +) + +_logs.set_logger_provider(LoggerProvider()) +_logs.get_logger_provider().add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter()) +) + +metrics.set_meter_provider( + MeterProvider( + metric_readers=[ + PeriodicExportingMetricReader(OTLPMetricExporter()), + ] + ) +) + +SmolagentsInstrumentor().instrument(completion_hook=PrintCompletionHook()) + + +def main(): + agent = CodeAgent(tools=[], model=InferenceClientModel()) + result = agent.run( + os.getenv("SMOLAGENTS_TASK", "How many seconds are in a week?") + ) + print(result) + + +if __name__ == "__main__": + main() diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/main.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/main.py new file mode 100644 index 000000000..ac91180f4 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/main.py @@ -0,0 +1,72 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: skip-file +import os + +from smolagents import CodeAgent, InferenceClientModel + +# NOTE: OpenTelemetry Python Logs API is in beta +from opentelemetry import _logs, metrics, trace +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +# configure tracing +trace.set_tracer_provider(TracerProvider()) +trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(OTLPSpanExporter()) +) + +# configure logging +_logs.set_logger_provider(LoggerProvider()) +_logs.get_logger_provider().add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter()) +) + +# configure metrics +metrics.set_meter_provider( + MeterProvider( + metric_readers=[ + PeriodicExportingMetricReader( + OTLPMetricExporter(), + ), + ] + ) +) + +# instrument smolagents +SmolagentsInstrumentor().instrument() + + +def main(): + agent = CodeAgent( + tools=[], + model=InferenceClientModel( + model_id=os.getenv("CHAT_MODEL", "Qwen/Qwen2.5-Coder-32B-Instruct") + ), + ) + print( + agent.run( + os.getenv("SMOLAGENTS_TASK", "How many seconds are in a week?") + ) + ) + + +if __name__ == "__main__": + main() diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/requirements.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/requirements.txt new file mode 100644 index 000000000..ec5123fa2 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/manual/requirements.txt @@ -0,0 +1,9 @@ +smolagents>=1.24.0 + +opentelemetry-sdk~=1.43.0 +opentelemetry-exporter-otlp-proto-grpc~=1.43.0 +opentelemetry-instrumentation-genai-smolagents~=1.1b0 + +# Uncomment to enable the upload completion hook (pulls in fsspec). +# Required when OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK=upload. +# opentelemetry-util-genai[upload] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/.env b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/.env new file mode 100644 index 000000000..8eafd97f2 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/.env @@ -0,0 +1,27 @@ +# Update this with your real Hugging Face token +HF_TOKEN=hf_YOUR_TOKEN + +# Uncomment to run a different model through Hugging Face Inference Providers +# CHAT_MODEL=meta-llama/Llama-3.3-70B-Instruct + +# Uncomment and change to your OTLP endpoint +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +# OTEL_EXPORTER_OTLP_PROTOCOL=grpc + +OTEL_SERVICE_NAME=opentelemetry-python-smolagents + +# Remove to hide prompt and completion content +# Possible values (case insensitive): +# - `span_only` - record content on span attributes +# - `event_only` - record content on event attributes +# - `span_and_event` - record content on both span and event attributes +# - everything else - don't record content on any signal +OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=span_only + +# Uncomment to upload prompts and responses to an fsspec-compatible +# destination instead of or in addition to recording them inline on spans/events. +# OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK=upload +# OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH=/path/to/prompts + +# Uncomment if your OTLP endpoint doesn't support logs +# OTEL_LOGS_EXPORTER=console diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/README.rst new file mode 100644 index 000000000..fb9144560 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/README.rst @@ -0,0 +1,44 @@ +OpenTelemetry smolagents Zero-Code Instrumentation Example +========================================================== + +This is an example of how to instrument smolagents agent runs with zero code +changes, using `opentelemetry-instrument`. + +When `main.py `_ is run, it exports traces, metrics, and logs to an +OTLP compatible endpoint. The agent run produces an ``invoke_agent`` span with +``chat`` and ``execute_tool`` spans nested under it, so you can see which model +calls and tools each step of the run made. + +Note: `.env <.env>`_ file configures additional environment variables: + +- ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=span_only`` configures the smolagents instrumentation to capture prompt and completion contents on *span* attributes. +- ``OTEL_LOGS_EXPORTER=otlp`` to specify exporter type. +- ``OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK`` (commented out) - uncomment along with ``OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH`` to upload prompts and completions to an ``fsspec``-compatible destination instead of recording them inline. Also uncomment the ``opentelemetry-util-genai[upload]`` line in `requirements.txt `_ and reinstall. + +Setup +----- + +Minimally, update the `.env <.env>`_ file with your ``HF_TOKEN``. An OTLP +compatible endpoint should be listening for traces, metrics, and logs on +http://localhost:4317. If not, update ``OTEL_EXPORTER_OTLP_ENDPOINT`` as well. + +Next, set up a virtual environment like this: + +:: + + python3 -m venv .venv + source .venv/bin/activate + pip install "python-dotenv[cli]" + pip install -r requirements.txt + +Run +--- + +Run the example like this: + +:: + + dotenv run -- opentelemetry-instrument python main.py + +You should see the agent's answer while traces, metrics, and logs export to +your configured observability tool. diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/main.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/main.py new file mode 100644 index 000000000..2239c7d79 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/main.py @@ -0,0 +1,24 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import os + +from smolagents import CodeAgent, InferenceClientModel + + +def main(): + agent = CodeAgent( + tools=[], + model=InferenceClientModel( + model_id=os.getenv("CHAT_MODEL", "Qwen/Qwen2.5-Coder-32B-Instruct") + ), + ) + print( + agent.run( + os.getenv("SMOLAGENTS_TASK", "How many seconds are in a week?") + ) + ) + + +if __name__ == "__main__": + main() diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/requirements.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/requirements.txt new file mode 100644 index 000000000..d7760df8f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/examples/zero-code/requirements.txt @@ -0,0 +1,10 @@ +smolagents>=1.24.0 + +opentelemetry-sdk~=1.43.0 +opentelemetry-exporter-otlp-proto-grpc~=1.43.0 +opentelemetry-distro~=0.64b0 +opentelemetry-instrumentation-genai-smolagents~=1.1b0 + +# Uncomment to enable the upload completion hook (pulls in fsspec). +# Required when OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK=upload. +# opentelemetry-util-genai[upload] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml index 9f7ed96ed..67140011a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "opentelemetry-api ~= 1.43", "opentelemetry-instrumentation >= 0.64b0, <1", "opentelemetry-semantic-conventions >= 0.64b0, <1", - "opentelemetry-util-genai >= 1.0b0, <2", + "opentelemetry-util-genai >= 1.1b0.dev, <2", ] [project.optional-dependencies] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index b3cffd043..7ade84336 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -15,10 +15,13 @@ from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) + from smolagents import CodeAgent, InferenceClientModel - # Enable instrumentation SmolagentsInstrumentor().instrument() + agent = CodeAgent(tools=[], model=InferenceClientModel()) + agent.run("How many seconds are in a week?") + Configuration ------------- @@ -38,20 +41,140 @@ from __future__ import annotations -from typing import Any, Collection +from contextvars import copy_context +from functools import wraps +from types import ModuleType +from typing import Any, Callable, Collection + +from wrapt import wrap_function_wrapper from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.utils import unwrap from opentelemetry.util.genai.completion_hook import load_completion_hook from opentelemetry.util.genai.handler import TelemetryHandler from .package import _instruments +from .patch import ( + agent_run, + agent_tool_calls, + model_generate, + model_generate_stream, + tool_call, +) __all__ = ["SmolagentsInstrumentor"] +_LOCAL_EXECUTOR_MODULE = "smolagents.local_python_executor" + + +def _model_classes_defining(smolagents: ModuleType, method: str) -> list[type]: + """The exported model classes whose ``method`` gets wrapped. + + Only classes that define ``method`` in their own ``__dict__`` are patched, + so a class that inherits it (``AzureOpenAIModel``, ``LiteLLMRouterModel``) + isn't wrapped a second time and can't produce duplicate ``chat`` spans. + A user-defined subclass that overrides the method shadows the patched base + method and emits no ``chat`` span; that limitation is documented in + ``README.rst``. + + Deduplicated by class object, because smolagents exports some classes under + two names (``OpenAIServerModel`` is ``OpenAIModel``) and wrapping the same + class twice would double every ``chat`` span. + """ + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + Model, + ) + + classes: dict[type, None] = {} + for obj in vars(smolagents).values(): + if ( + isinstance(obj, type) + and issubclass(obj, Model) + and method in obj.__dict__ + ): + classes.setdefault(obj, None) + return list(classes) + + +def _tool_classes_defining_call( + smolagents: ModuleType, +) -> list[type]: + """The tool classes whose ``__call__`` gets wrapped. + + ``PipelineTool`` overrides ``Tool.__call__`` without delegating to it, so + patching ``Tool`` alone would miss it and the shipped ``SpeechToTextTool``. + Only classes that define ``__call__`` are patched, so a call emits exactly + one span. smolagents doesn't export ``PipelineTool``, hence the MRO walk. + """ + from smolagents.tools import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + Tool, + ) + + classes: dict[type, None] = {} + for obj in vars(smolagents).values(): + if not isinstance(obj, type) or not issubclass(obj, Tool): + continue + for cls in obj.__mro__: + if issubclass(cls, Tool) and "__call__" in cls.__dict__: + classes.setdefault(cls, None) + return list(classes) + + +def _context_preserving_timeout( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Run code decorated with ``timeout()`` in a copy of the caller's context. + + ``local_python_executor.timeout()`` runs the decorated function in a + ``ThreadPoolExecutor`` worker and submits it without propagating + ``contextvars``, so a span started while executing agent-generated code (a + tool call) loses the active OTel context and becomes a root span instead of + a child of the agent span. smolagents copies the context for its parallel + tool calls (``agents.py``: ``ctx = copy_context(); executor.submit(ctx.run, + ...)``) but not here. + + A ``CodeAgent`` with the default ``executor_type="local"`` goes through this + on every step; the remote executors do not. + """ + decorator = wrapped(*args, **kwargs) + + def context_preserving_decorator( + func: Callable[..., Any], + ) -> Callable[..., Any]: + @wraps(func) + def run_with_caller_context( + *call_args: Any, **call_kwargs: Any + ) -> Any: + context = copy_context() + + def in_caller_context( + *inner_args: Any, **inner_kwargs: Any + ) -> Any: + return context.run(func, *inner_args, **inner_kwargs) + + return decorator(in_caller_context)(*call_args, **call_kwargs) + + return run_with_caller_context + + return context_preserving_decorator + class SmolagentsInstrumentor(BaseInstrumentor): """An instrumentor for smolagents.""" + # ``BaseInstrumentor.__new__`` returns a per-class singleton, but Python + # still runs ``__init__`` on every construction. Initializing this state in + # ``__init__`` would let the documented ``SmolagentsInstrumentor() + # .uninstrument()`` form wipe the live instance's bookkeeping and leave + # smolagents permanently patched, so these are class-level defaults that + # only ``_instrument`` / ``_uninstrument`` rebind. + _wrapped_generate_classes: list[type] = [] + _wrapped_generate_stream_classes: list[type] = [] + _wrapped_tool_classes: list[type] = [] + def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -65,15 +188,99 @@ def _instrument(self, **kwargs: Any) -> None: - logger_provider: LoggerProvider instance - completion_hook: CompletionHook instance """ - TelemetryHandler( + import smolagents # pylint: disable=import-outside-toplevel # noqa: PLC0415 + + handler = TelemetryHandler( tracer_provider=kwargs.get("tracer_provider"), meter_provider=kwargs.get("meter_provider"), logger_provider=kwargs.get("logger_provider"), completion_hook=kwargs.get("completion_hook") or load_completion_hook(), ) - # Patching will be added in follow-up PRs. + + wrapped_generate_classes: list[type] = [] + self._wrapped_generate_classes = wrapped_generate_classes + wrapped_generate_stream_classes: list[type] = [] + self._wrapped_generate_stream_classes = wrapped_generate_stream_classes + wrapped_tool_classes: list[type] = [] + self._wrapped_tool_classes = wrapped_tool_classes + try: + wrap_function_wrapper( + "smolagents", + "MultiStepAgent.run", + agent_run(handler), + ) + # Emits no span of its own; it carries the provider's tool call id + # down to the execute_tool spans of the step. + wrap_function_wrapper( + "smolagents", + "ToolCallingAgent.process_tool_calls", + agent_tool_calls, + ) + + for tool_cls in _tool_classes_defining_call(smolagents): + wrap_function_wrapper( + tool_cls, + "__call__", + tool_call(handler), + ) + wrapped_tool_classes.append(tool_cls) + + for model_cls in _model_classes_defining(smolagents, "generate"): + wrap_function_wrapper( + model_cls, + "generate", + model_generate(handler), + ) + wrapped_generate_classes.append(model_cls) + + for model_cls in _model_classes_defining( + smolagents, "generate_stream" + ): + wrap_function_wrapper( + model_cls, + "generate_stream", + model_generate_stream(handler), + ) + wrapped_generate_stream_classes.append(model_cls) + + # TODO: no span per agent step. OpenInference wraps _step_stream to + # emit one CHAIN span per ReAct iteration, using its own span-kind + # vocabulary. The semconv defines no operation for an iteration, and + # invoke_workflow is taken: openai-agents and langchain both put it + # above invoke_agent, so one per step would invert that nesting. + # Until a span is specified, chat and execute_tool nest directly + # under invoke_agent. + # https://github.com/open-telemetry/semantic-conventions-genai/issues/81 + wrap_function_wrapper( + _LOCAL_EXECUTOR_MODULE, + "timeout", + _context_preserving_timeout, + ) + except Exception: + # BaseInstrumentor.instrument() doesn't mark the instrumentor as + # instrumented when _instrument raises, so uninstrument() would + # refuse to run and leave the patches applied with no way to undo. + self._uninstrument() + raise def _uninstrument(self, **kwargs: Any) -> None: """Disable smolagents instrumentation and restore patched originals.""" - # Unpatching will be added in follow-up PRs. + import smolagents # pylint: disable=import-outside-toplevel # noqa: PLC0415 + + unwrap(smolagents.MultiStepAgent, "run") + unwrap(smolagents.ToolCallingAgent, "process_tool_calls") + + for tool_cls in self._wrapped_tool_classes: + unwrap(tool_cls, "__call__") + self._wrapped_tool_classes = [] + + for model_cls in self._wrapped_generate_classes: + unwrap(model_cls, "generate") + self._wrapped_generate_classes = [] + + for model_cls in self._wrapped_generate_stream_classes: + unwrap(model_cls, "generate_stream") + self._wrapped_generate_stream_classes = [] + + unwrap(_LOCAL_EXECUTOR_MODULE, "timeout") diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py new file mode 100644 index 000000000..e4257ec33 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -0,0 +1,387 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Convert smolagents message and tool shapes into util-genai GenAI types. + +smolagents passes ``generate(messages=...)`` a list of ``ChatMessage`` objects +or plain dicts, and returns a ``ChatMessage``. This module maps those, and the +``tools_to_call_from`` tool objects, onto the types in +``opentelemetry.util.genai.types``. util-genai then serializes them into +``gen_ai.input.messages``, ``gen_ai.output.messages``, and +``gen_ai.tool.definitions``. +""" + +from __future__ import annotations + +import base64 +import binascii +import logging +from collections.abc import Mapping +from enum import Enum +from typing import Any + +from opentelemetry.util.genai.types import ( + Blob, + FunctionToolDefinition, + InputMessage, + OutputMessage, + Reasoning, + Text, + ToolCallRequest, + ToolDefinition, + Uri, +) +from opentelemetry.util.genai.utils import gen_ai_json_dumps + +_logger = logging.getLogger(__name__) + +_DEFAULT_IMAGE_MIME_TYPE = "image/png" +_DATA_URL_PREFIX = "data:" + +# smolagents-internal roles -> semconv ``gen_ai`` message roles. smolagents +# applies the same mapping (``models.tool_role_conversions``) inside +# ``generate``, after the wrapper has already read ``messages``, so the wrapper +# has to apply it too. Without this map the input messages would carry roles +# the spec doesn't define. A model configured with ``custom_role_conversions`` +# overrides the mapping and can send different roles. Roles that are already +# spec values pass through unchanged. +_ROLE_MAP: dict[str, str] = { + "tool-call": "assistant", + "tool-response": "user", +} + +# Amazon Bedrock reports the stop reason as ``stopReason`` using Anthropic's +# vocabulary. Normalize it the way the anthropic instrumentation does +# (``anthropic/utils.py``); anything unmapped passes through. +_STOP_REASON_MAP: dict[str, str] = { + "end_turn": "stop", + "stop_sequence": "stop", + "max_tokens": "length", + "tool_use": "tool_calls", +} + + +def _unwrap_role(role: Any) -> str | None: + if role is None: + return None + if isinstance(role, Enum): + role = role.value + name = str(role) + return _ROLE_MAP.get(name, name) + + +def _decode_base64_image(image: str) -> tuple[bytes, str] | None: + """Decode a base64 payload or data URL into ``(bytes, mime_type)``.""" + mime_type = _DEFAULT_IMAGE_MIME_TYPE + if image.startswith(_DATA_URL_PREFIX): + header, _, image = image.partition(",") + media_type = header[len(_DATA_URL_PREFIX) :].split(";")[0] + if media_type: + mime_type = media_type + try: + return base64.b64decode(image, validate=True), mime_type + except (binascii.Error, ValueError): + _logger.debug("Failed to decode a base64 image", exc_info=True) + return None + + +def _encode_image_base64(image: Any) -> str | None: + try: + from smolagents.utils import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + encode_image_base64, + ) + except ImportError: + _logger.debug("smolagents.utils.encode_image_base64 is unavailable") + return None + try: + encoded = encode_image_base64(image) + except Exception: # pylint: disable=broad-except + _logger.debug( + "Failed to encode image of type %s, dropping it from telemetry", + type(image).__name__, + exc_info=True, + ) + return None + return encoded if isinstance(encoded, str) else None + + +def _image_blob(image: Any) -> Blob | None: + """Build a ``Blob`` part from a base64 string, data URL, or PIL image.""" + if isinstance(image, str): + decoded = _decode_base64_image(image) + else: + encoded = _encode_image_base64(image) + decoded = ( + _decode_base64_image(encoded) if encoded is not None else None + ) + if decoded is None: + return None + content, mime_type = decoded + return Blob(mime_type=mime_type, modality="image", content=content) + + +def _image_part_from_element(element: dict[str, Any]) -> Uri | Blob | None: + content_type = element.get("type") + if content_type == "image_url": + image_url = element.get("image_url") + url = image_url.get("url") if isinstance(image_url, dict) else None + if isinstance(url, str) and url: + return Uri(mime_type=None, modality="image", uri=url) + return None + if content_type == "image": + image = element.get("image") + if image is not None: + return _image_blob(image) + return None + + +def _parts_from_content(content: Any) -> list[Any]: + parts: list[Any] = [] + if isinstance(content, str): + parts.append(Text(content=content)) + return parts + if isinstance(content, list): + for element in content: + if not isinstance(element, dict): + _logger.debug( + "Unknown message content dropped from telemetry: %s", + type(element).__name__, + ) + continue + if element.get("type") == "text" and (text := element.get("text")): + parts.append(Text(content=text)) + continue + image_part = _image_part_from_element(element) + if image_part is not None: + parts.append(image_part) + else: + _logger.debug( + "Unknown message part dropped from telemetry: %s", + element.get("type"), + ) + return parts + + +def _get_role_and_content(message: Any) -> tuple[Any, Any]: + if isinstance(message, dict): + return message.get("role"), message.get("content") + return getattr(message, "role", None), getattr(message, "content", None) + + +def to_input_messages(messages: Any) -> list[InputMessage]: + """Map smolagents ``generate`` input messages to ``InputMessage`` objects.""" + result: list[InputMessage] = [] + if not isinstance(messages, list): + return result + for message in messages: + raw_role, content = _get_role_and_content(message) + role = _unwrap_role(raw_role) + if not role: + continue + result.append( + InputMessage(role=role, parts=_parts_from_content(content)) + ) + return result + + +def _raw_value(raw: Any, key: str) -> Any: + """Read ``key`` off a provider response that may be an object or a dict. + + ``ChatMessage.raw`` is whatever the provider handed back: an OpenAI-shaped + object for the API-backed models, and a dict for ``AmazonBedrockModel`` + (the boto3 ``converse`` response) and the local runtimes. + """ + if isinstance(raw, Mapping): + return raw.get(key) + return getattr(raw, key, None) + + +def _first_choice(output_message: Any) -> Any: + choices = _raw_value(getattr(output_message, "raw", None), "choices") + if isinstance(choices, list) and choices: + return choices[0] + return None + + +def _reasoning_from_raw(output_message: Any) -> str | None: + message = _raw_value(_first_choice(output_message), "message") + reasoning = _raw_value(message, "reasoning_content") + return reasoning if isinstance(reasoning, str) and reasoning else None + + +def _tool_call_requests(output_message: Any) -> list[ToolCallRequest]: + # ChatMessage.__post_init__ coerces every entry into a + # ChatMessageToolCall, so the id/function/name/arguments are all present. + tool_calls = getattr(output_message, "tool_calls", None) or [] + return [ + ToolCallRequest( + name=tool_call.function.name, + id=tool_call.id, + arguments=tool_call.function.arguments, + ) + for tool_call in tool_calls + ] + + +def _finish_reason(output_message: Any, has_tool_calls: bool) -> str | None: + """Why the provider stopped generating, or ``None`` if it didn't say. + + The local runtimes (``TransformersModel``, ``VLLMModel``, ``MLXModel``) put + ``{"out": ..., "completion_kwargs": ...}`` on ``raw`` and report no finish + reason at all. Defaulting those to ``"stop"`` would make a generation cut + short by ``max_new_tokens`` look like a natural stop. A response carrying + tool calls is the one case where the reason follows without guessing. + """ + reason = _raw_value(_first_choice(output_message), "finish_reason") + if isinstance(reason, str) and reason: + return reason + raw = getattr(output_message, "raw", None) + stop_reason = _raw_value(raw, "stopReason") + if isinstance(stop_reason, str) and stop_reason: + return _STOP_REASON_MAP.get(stop_reason, stop_reason) + return "tool_calls" if has_tool_calls else None + + +def to_output_message(output_message: Any) -> OutputMessage: + """Map a smolagents ``ChatMessage`` response to an ``OutputMessage``.""" + role = _unwrap_role(getattr(output_message, "role", None)) or "assistant" + parts: list[Any] = _parts_from_content( + getattr(output_message, "content", None) + ) + if reasoning := _reasoning_from_raw(output_message): + parts.append(Reasoning(content=reasoning)) + tool_call_requests = _tool_call_requests(output_message) + parts.extend(tool_call_requests) + # OutputMessage requires the field; util-genai drops an empty value when it + # emits gen_ai.response.finish_reasons. + reason = _finish_reason(output_message, bool(tool_call_requests)) + return OutputMessage(role=role, parts=parts, finish_reason=reason or "") + + +def response_id(output_message: Any) -> str | None: + """Extract ``gen_ai.response.id`` from the provider response on ``.raw``.""" + value = _raw_value(getattr(output_message, "raw", None), "id") + return value if isinstance(value, str) and value else None + + +def response_model_name(output_message: Any) -> str | None: + """Extract ``gen_ai.response.model`` from the provider response on ``.raw``.""" + value = _raw_value(getattr(output_message, "raw", None), "model") + return value if isinstance(value, str) and value else None + + +def _tool_parameters(tool: Any) -> dict[str, Any] | None: + """Return the JSON Schema ``parameters`` object for a smolagents tool. + + A tool's ``inputs`` map is not a JSON Schema on its own: smolagents wraps it + in an object schema, derives ``required`` from ``nullable``, and rewrites its + non-JSON-Schema ``"any"`` type. ``get_tool_json_schema`` builds exactly the + schema the provider receives. + """ + try: + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + get_tool_json_schema, + ) + except ImportError: + _logger.debug("smolagents.models.get_tool_json_schema is unavailable") + return None + try: + schema = get_tool_json_schema(tool) + parameters = schema["function"]["parameters"] + except Exception: # pylint: disable=broad-except + _logger.debug( + "Failed to build a JSON Schema for tool %s", + getattr(tool, "name", None), + exc_info=True, + ) + return None + return parameters if isinstance(parameters, dict) else None + + +def to_tool_definitions(tools: Any) -> list[ToolDefinition] | None: + """Map smolagents tool objects to function tool definitions.""" + if not isinstance(tools, list) or not tools: + return None + definitions: list[ToolDefinition] = [] + for tool in tools: + name = getattr(tool, "name", None) + if not name: + continue + definitions.append( + FunctionToolDefinition( + name=name, + description=getattr(tool, "description", None), + parameters=_tool_parameters(tool), + ) + ) + return definitions or None + + +def _is_pil_image(value: Any) -> bool: + """Whether ``value`` is a PIL image, which ``AgentImage`` subclasses.""" + try: + from PIL.Image import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + Image, + ) + except ImportError: + _logger.debug("PIL is unavailable") + return False + return isinstance(value, Image) + + +def to_text(value: Any) -> str: + """Stringify a value without running smolagents' side-effecting ``__str__``. + + ``AgentText`` and ``AgentAudio`` subclass ``str`` but override ``__str__`` + (``agent_types.py``), and ``AgentAudio.to_string()`` writes a ``.wav`` file + to a temp directory. Reading the underlying ``str`` avoids that. + """ + if isinstance(value, str): + return str.__str__(value) + return str(value) + + +def to_content_value(value: Any) -> Any: + """Return a value safe to hand to util-genai as span content. + + util-genai serializes tool results as JSON and falls back to ``str(value)``. + For an image tool that fallback records a heap address, and for an + ``AgentImage`` it saves a PNG into a fresh temp directory and mutates the + object the caller still holds. Values JSON can't represent are recorded as + their type name instead. + """ + if value is None or isinstance(value, (bool, int, float, bytes)): + return value + if isinstance(value, str): + return to_text(value) + try: + return gen_ai_json_dumps(value) + except (TypeError, ValueError): + return type(value).__name__ + + +def final_answer_parts(output: Any) -> list[Any]: + """Map an agent's final answer onto output message parts. + + ``MultiStepAgent`` wraps the answer in ``handle_agent_output_types``, so an + image-producing run returns an ``AgentImage``. Recording it as a ``Blob`` + keeps the content and matches how input images are recorded; stringifying + it would write the image to disk and record the path instead. + """ + if _is_pil_image(output): + blob = _image_blob(output) + return [blob] if blob is not None else [] + return [Text(content=to_text(output))] + + +def task_to_input_message(task: Any, images: Any) -> InputMessage: + """Build the agent-run input message from the task string and images.""" + parts: list[Any] = [] + if isinstance(task, str) and task: + parts.append(Text(content=task)) + if isinstance(images, list): + for image in images: + blob = _image_blob(image) + if blob is not None: + parts.append(blob) + return InputMessage(role="user", parts=parts) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py index b56883424..fa7223304 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/package.py @@ -2,3 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 _instruments = ("smolagents >= 1.24.0",) + +_supports_metrics = True diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py new file mode 100644 index 000000000..69f0fb751 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -0,0 +1,840 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""wrapt wrapper factories for smolagents instrumentation. + +Each factory takes the shared :class:`TelemetryHandler` and returns a wrapper +suitable for :func:`wrapt.wrap_function_wrapper`: + +- :func:`agent_run` wraps ``MultiStepAgent.run`` -> ``invoke_agent`` span. +- :func:`model_generate` wraps each defining ``Model.generate`` -> ``chat`` span. +- :func:`model_generate_stream` wraps each defining ``Model.generate_stream`` + -> ``chat`` span, held open until the stream is drained. +- :func:`tool_call` wraps ``Tool.__call__`` -> ``execute_tool`` span. +- :func:`agent_tool_calls` wraps ``ToolCallingAgent.process_tool_calls``. No + span; it publishes the step's tool call ids for the ``execute_tool`` spans. + +Original library exceptions are always re-raised unmodified; telemetry is +finalized via ``invocation.stop()`` / ``invocation.fail(exc)``. The one +exception is documented on :func:`_tolerant_run_generator`. +""" + +from __future__ import annotations + +import logging +from collections.abc import Generator, Mapping +from contextvars import ContextVar +from dataclasses import dataclass +from inspect import GEN_SUSPENDED, Parameter, getgeneratorstate, signature +from typing import Any, Callable + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import ( + AgentInvocation, + InferenceInvocation, +) +from opentelemetry.util.genai.stream import SyncStreamWrapper +from opentelemetry.util.genai.types import OutputMessage, Text, ToolCallRequest + +from ._messages import ( + final_answer_parts, + response_id, + response_model_name, + task_to_input_message, + to_content_value, + to_input_messages, + to_output_message, + to_tool_definitions, +) +from .provider import resolve_provider, resolve_server_address_port + +_logger = logging.getLogger(__name__) + +_Wrapper = Callable[ + [Callable[..., Any], Any, tuple[Any, ...], dict[str, Any]], Any +] + + +def _bind_arguments( + wrapped: Callable[..., Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> dict[str, Any]: + """Bind call args to the wrapped callable's signature, applying defaults. + + smolagents passes the interesting arguments positionally (``run(task)``, + ``model.generate(input_messages)``), so binding is what makes them + readable by name. On a binding failure the keyword arguments are returned + on their own, without the positional ones and without the defaults. + """ + try: + bound = signature(wrapped).bind(*args, **kwargs) + bound.apply_defaults() + return dict(bound.arguments) + except (TypeError, ValueError): + _logger.debug( + "Failed to bind arguments of %s; falling back to keyword arguments", + getattr(wrapped, "__qualname__", wrapped), + exc_info=True, + ) + return dict(kwargs) + + +def _coerce_float(value: Any) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return None + + +def _coerce_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + return None + + +def _remove_parameter_sentinel() -> Any: + """Return smolagents' ``REMOVE_PARAMETER`` sentinel, or ``None`` if absent.""" + try: + from smolagents.models import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + REMOVE_PARAMETER, + ) + except ImportError: + _logger.debug("smolagents.models.REMOVE_PARAMETER is unavailable") + return None + return REMOVE_PARAMETER + + +# Model classes that never forward ``stop_sequences``, whatever +# ``supports_stop_parameter`` answers. ``AmazonBedrockModel`` overrides +# ``_prepare_completion_kwargs`` and calls the base with a hardcoded +# ``stop_sequences=None`` (``models.py``), so its ``converse`` request carries +# no stop sequences at all. ``_forwards_stop_sequences`` matches these names +# along the MRO, so subclasses are covered too. +_MODELS_DROPPING_STOP_SEQUENCES = frozenset({"AmazonBedrockModel"}) + + +def _forwards_stop_sequences(instance: Any) -> bool: + if any( + cls.__name__ in _MODELS_DROPPING_STOP_SEQUENCES + for cls in type(instance).__mro__ + ): + return False + return bool(getattr(instance, "supports_stop_parameter", False)) + + +def _merged_request_kwargs( + instance: Any, bound: dict[str, Any] +) -> dict[str, Any]: + """Rebuild the request keyword arguments smolagents will send. + + ``Model._prepare_completion_kwargs`` seeds ``stop`` from the + ``stop_sequences`` argument and ``response_format`` from its own argument, + applies the per-call ``**kwargs`` on top and the model-level ``self.kwargs`` + last, so a key set at two levels reaches the provider with the model-level + value and a model-level ``REMOVE_PARAMETER`` drops it from the request + entirely. Following that order here is what keeps a removed key off the span + as well. + """ + merged: dict[str, Any] = {} + stop_sequences = bound.get("stop_sequences") + if stop_sequences is not None and _forwards_stop_sequences(instance): + merged["stop"] = stop_sequences + response_format = bound.get("response_format") + if response_format is not None: + merged["response_format"] = response_format + call_kwargs = bound.get("kwargs") + if isinstance(call_kwargs, dict): + merged.update(call_kwargs) + model_kwargs = getattr(instance, "kwargs", None) + if not isinstance(model_kwargs, dict): + return merged + remove = _remove_parameter_sentinel() + for name, value in model_kwargs.items(): + if remove is not None and value is remove: + merged.pop(name, None) + else: + merged[name] = value + return merged + + +def _stop_sequences(merged: dict[str, Any]) -> list[str] | None: + """Return the stop sequences the request carries, if any.""" + stop = merged.get("stop", merged.get("stop_sequences")) + if isinstance(stop, str): + return [stop] + if isinstance(stop, list): + return [str(item) for item in stop] + return None + + +# smolagents' ``response_format`` type -> ``gen_ai.output.type`` value, for the +# types whose provider spelling differs from the semconv one. The openai +# instrumentation maps the same two. +_OUTPUT_TYPE_MAP: dict[str, str] = { + "json_object": GenAI.GenAiOutputTypeValues.JSON.value, + "json_schema": GenAI.GenAiOutputTypeValues.JSON.value, +} + +_OUTPUT_TYPE_VALUES = frozenset( + value.value for value in GenAI.GenAiOutputTypeValues +) + + +def _output_type(merged: dict[str, Any]) -> str | None: + """Map the request's ``response_format`` to ``gen_ai.output.type``. + + smolagents forwards ``response_format`` to the provider unchanged, so its + ``type`` is whatever the provider accepts. Only the values the semconv + defines are recorded; a provider-specific one is dropped rather than put on + an enum attribute. + """ + response_format = merged.get("response_format") + if not isinstance(response_format, Mapping): + return None + format_type = response_format.get("type") + if not isinstance(format_type, str): + return None + output_type = _OUTPUT_TYPE_MAP.get(format_type, format_type) + if output_type not in _OUTPUT_TYPE_VALUES: + _logger.debug( + "No gen_ai.output.type value for response_format type %r", + format_type, + ) + return None + return output_type + + +def _apply_request_parameters( + invocation: InferenceInvocation, instance: Any, bound: dict[str, Any] +) -> None: + """Copy the request parameters smolagents will send onto the span.""" + merged = _merged_request_kwargs(instance, bound) + + invocation.temperature = _coerce_float(merged.get("temperature")) + invocation.top_p = _coerce_float(merged.get("top_p")) + invocation.top_k = _coerce_float(merged.get("top_k")) + invocation.frequency_penalty = _coerce_float( + merged.get("frequency_penalty") + ) + invocation.presence_penalty = _coerce_float(merged.get("presence_penalty")) + # TransformersModel takes the generation limit as max_new_tokens (which its + # constructor defaults to 4096) and treats max_tokens as an alias for it. + invocation.max_tokens = _coerce_int( + merged.get("max_tokens", merged.get("max_new_tokens")) + ) + invocation.seed = _coerce_int(merged.get("seed")) + invocation.stop_sequences = _stop_sequences(merged) + invocation.output_type = _output_type(merged) + + +def _apply_token_usage( + invocation: InferenceInvocation, output_message: Any +) -> None: + # ChatMessage.token_usage is the only source: the per-model + # last_input_token_count / last_output_token_count counters were removed + # before the oldest supported smolagents. It is None for the local runtimes + # (TransformersModel, VLLMModel, MLXModel), which report no usage. + token_usage = getattr(output_message, "token_usage", None) + if token_usage is None: + return + invocation.input_tokens = token_usage.input_tokens + invocation.output_tokens = token_usage.output_tokens + + +def _start_inference( + handler: TelemetryHandler, + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> InferenceInvocation: + """Start the ``chat`` span and record the request. + + ``generate`` and ``generate_stream`` take the same parameters. + """ + provider = resolve_provider(instance) + server_address, server_port = resolve_server_address_port(instance) + invocation = handler.inference( + provider, + request_model=getattr(instance, "model_id", None), + server_address=server_address, + server_port=server_port, + ) + bound = _bind_arguments(wrapped, args, kwargs) + _apply_request_parameters(invocation, instance, bound) + invocation.tool_definitions = to_tool_definitions( + bound.get("tools_to_call_from") + ) + if handler.should_capture_content(): + invocation.input_messages = to_input_messages(bound.get("messages")) + return invocation + + +def model_generate(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate`` to emit a ``chat`` span.""" + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + + try: + output_message = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + + _apply_token_usage(invocation, output_message) + invocation.response_model_name = response_model_name(output_message) + invocation.response_id = response_id(output_message) + output = to_output_message(output_message) + # to_output_message leaves finish_reason empty when the provider + # reported none, and an empty value is dropped rather than guessed at. + if output.finish_reason: + invocation.finish_reasons = [output.finish_reason] + if handler.should_capture_content(): + invocation.output_messages = [output] + invocation.stop() + return output_message + + return wrapper + + +@dataclass +class _StreamedToolCall: + """A tool call assembled from stream deltas.""" + + id: str | None = None + name: str = "" + arguments: str = "" + + +class _ModelStreamWrapper(SyncStreamWrapper[Any]): + """Keep the ``chat`` span open until the delta stream is drained. + + Passing the invocation to ``super().__init__()`` turns on + ``gen_ai.request.stream`` and the per-chunk timing metrics. + """ + + def __init__( + self, + stream: Generator[Any, Any, Any], + invocation: InferenceInvocation, + handler: TelemetryHandler, + ) -> None: + super().__init__(stream, invocation=invocation) + self._self_inference = invocation + self._self_handler = handler + self._self_content: list[str] = [] + self._self_tool_calls: dict[int, _StreamedToolCall] = {} + self._self_input_tokens = 0 + self._self_output_tokens = 0 + self._self_saw_token_usage = False + + def _accumulate_tool_call(self, delta: Any) -> None: + index = getattr(delta, "index", None) + if not isinstance(index, int): + # agglomerate_stream_deltas raises here; telemetry must not. + _logger.debug("Dropping a tool call delta that carries no index") + return + tool_call = self._self_tool_calls.setdefault( + index, _StreamedToolCall() + ) + if delta.id: + tool_call.id = delta.id + function = getattr(delta, "function", None) + if function is None: + return + if function.name: + tool_call.name = function.name + if function.arguments: + tool_call.arguments += function.arguments + + def _process_chunk(self, chunk: Any) -> None: + content = getattr(chunk, "content", None) + if content: + self._self_content.append(content) + token_usage = getattr(chunk, "token_usage", None) + if token_usage is not None: + # Summed like agglomerate_stream_deltas, so the span agrees with the + # totals the agent's monitor reports. + self._self_saw_token_usage = True + self._self_input_tokens += token_usage.input_tokens + self._self_output_tokens += token_usage.output_tokens + for delta in getattr(chunk, "tool_calls", None) or []: + self._accumulate_tool_call(delta) + + def _output_message(self) -> OutputMessage | None: + parts: list[Any] = [] + content = "".join(self._self_content) + if content: + parts.append(Text(content=content)) + parts.extend( + ToolCallRequest( + name=tool_call.name, + id=tool_call.id, + arguments=tool_call.arguments or None, + ) + for tool_call in self._self_tool_calls.values() + ) + if not parts: + # Closed before it was drained, so there is no response to report. + return None + # Deltas carry no finish reason, so tool calls are the only evidence. + # Defaulting to "stop" would hide a generation cut short by a token + # limit. + finish_reason = "tool_calls" if self._self_tool_calls else "" + return OutputMessage( + role="assistant", parts=parts, finish_reason=finish_reason + ) + + def _finalize(self, error: BaseException | None = None) -> None: + invocation = self._self_inference + if self._self_saw_token_usage: + invocation.input_tokens = self._self_input_tokens + invocation.output_tokens = self._self_output_tokens + output = self._output_message() + if output is not None: + if output.finish_reason: + invocation.finish_reasons = [output.finish_reason] + if self._self_handler.should_capture_content(): + invocation.output_messages = [output] + if error is not None: + invocation.fail(error) + else: + invocation.stop() + + def _on_stream_end(self) -> None: + self._finalize() + + def _on_stream_error(self, error: BaseException) -> None: + # Records what was streamed before the failure. + self._finalize(error) + + +def model_generate_stream(handler: TelemetryHandler) -> _Wrapper: + """Wrap a defining ``Model.generate_stream`` to emit a ``chat`` span. + + ``stream_outputs=True`` routes an agent's model calls here. The span stays + open until the caller drains the deltas. + """ + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + invocation = _start_inference(handler, wrapped, instance, args, kwargs) + + try: + stream = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + + return _ModelStreamWrapper(stream, invocation, handler) + + return wrapper + + +def _positional_parameter_names(instance: Any) -> list[str]: + """Names ``Tool.forward`` gives its positional parameters, in order. + + ``Tool.__call__`` forwards positional arguments straight to ``forward``, so + the signature is what names them. The declared ``inputs`` mapping is not: + ``Tool.validate_arguments`` only checks that the two agree as *sets*. A tool + whose hand-written ``inputs`` is ordered differently would record every + argument under the wrong name. ``@tool`` builds ``SimpleTool`` with an + unbound ``forward``, hence the ``self`` filter. + + ``skip_forward_signature_validation`` opts out of that check, so ``forward`` + says nothing about the declared inputs and those are used instead. A + ``PipelineTool`` is why this matters: its ``forward(inputs)`` names the + *encoded* inputs, not the caller's argument. + """ + if getattr(instance, "skip_forward_signature_validation", False) is True: + return list(instance.inputs) + try: + parameters = list(signature(instance.forward).parameters.values()) + except (TypeError, ValueError): + _logger.debug( + "Failed to read the signature of %s.forward", + type(instance).__name__, + exc_info=True, + ) + parameters = [] + names = [ + parameter.name + for parameter in parameters + if parameter.kind + in (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD) + and parameter.name != "self" + ] + return names or list(instance.inputs) + + +def _tool_arguments( + instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any] +) -> Any: + """Map a ``Tool.__call__`` invocation onto ``{input name: value}``.""" + call_kwargs = { + key: value + for key, value in kwargs.items() + if key != "sanitize_inputs_outputs" + } + if not args: + return call_kwargs + names = _positional_parameter_names(instance) + # A lone dict whose keys are all declared inputs is forwarded as kwargs by + # Tool.__call__ itself, so record it the same way. + if ( + not call_kwargs + and len(args) == 1 + and isinstance(args[0], dict) + and all(key in names for key in args[0]) + ): + return args[0] + # zip truncates: positional arguments beyond the declared inputs are dropped + # rather than recorded under a wrong name. + return {**dict(zip(names, args)), **call_kwargs} + + +# The tool calls a ToolCallingAgent step is about to run, published by +# agent_tool_calls for the execute_tool spans of that step. It is a ContextVar +# because smolagents runs parallel tool calls in worker threads, and each thread +# copies the caller's context. +_PENDING_TOOL_CALLS: ContextVar[tuple[Any, ...]] = ContextVar( + "otel_smolagents_pending_tool_calls", default=() +) + + +def _claim_tool_call_id(tool_name: str) -> str | None: + """The id of the pending tool call for ``tool_name``, if there is just one. + + ``Tool.__call__`` gets only the argument values, so the id comes from the + ``ToolCall`` objects the step yielded. The match is by name, because + ``execute_tool_call`` rewrites state-variable arguments first. Two calls to + one tool in a step have no unambiguous match, so the id is left off. + """ + matches = [ + pending + for pending in _PENDING_TOOL_CALLS.get() + if getattr(pending, "name", None) == tool_name + ] + if len(matches) != 1: + return None + call_id = getattr(matches[0], "id", None) + return call_id if isinstance(call_id, str) and call_id else None + + +def _tool_call_class() -> type | None: + try: + from smolagents.memory import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + ToolCall, + ) + except ImportError: + _logger.debug("smolagents.memory.ToolCall is unavailable") + return None + return ToolCall + + +def agent_tool_calls( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> Generator[Any, Any, Any]: + """Publish a ``ToolCallingAgent`` step's tool calls. Emits no span. + + ``process_tool_calls`` yields every ``ToolCall`` before it runs any of them, + so the published tuple is complete by the time a tool executes. Parallel + calls run in worker threads that copy the context, so they see it too. + """ + tool_call_cls = _tool_call_class() + pending: list[Any] = [] + token = _PENDING_TOOL_CALLS.set(()) + try: + for item in wrapped(*args, **kwargs): + if tool_call_cls is not None and isinstance(item, tool_call_cls): + pending.append(item) + _PENDING_TOOL_CALLS.set(tuple(pending)) + yield item + finally: + _reset_pending_tool_calls(token) + + +def _reset_pending_tool_calls(token: Any) -> None: + try: + _PENDING_TOOL_CALLS.reset(token) + except ValueError: + # Token from another context: a generator closed on a different thread. + # Telemetry must not raise over it. + _logger.debug("Failed to reset the pending tool calls", exc_info=True) + + +def tool_call(handler: TelemetryHandler) -> _Wrapper: + """Wrap ``Tool.__call__`` to emit an ``execute_tool`` span.""" + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + # Tool.validate_arguments() runs after every Tool.__init__ and rejects + # a tool without a str name and description. + invocation = handler.tool( + name=instance.name, + tool_call_id=_claim_tool_call_id(instance.name), + tool_type="function", + tool_description=instance.description, + ) + if invocation.should_capture_content_on_span: + invocation.arguments = _tool_arguments(instance, args, kwargs) + + try: + result = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + + if invocation.should_capture_content_on_span: + invocation.tool_result = to_content_value(result) + invocation.stop() + return result + + return wrapper + + +def _final_answer_step_class() -> type | None: + try: + from smolagents.memory import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + FinalAnswerStep, + ) + except ImportError: + _logger.debug("smolagents.memory.FinalAnswerStep is unavailable") + return None + return FinalAnswerStep + + +def _max_steps_error_class() -> type[Exception] | None: + try: + from smolagents import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + AgentMaxStepsError, + ) + except ImportError: + _logger.debug("smolagents.AgentMaxStepsError is unavailable") + return None + return AgentMaxStepsError + + +def _run_finish_reason(agent: Any) -> str: + """Why the run stopped: ``"stop"`` for an answer, ``"length"`` for a budget. + + A run that uses up ``max_steps`` does not raise. smolagents asks the model + for a closing answer, appends an ``ActionStep`` carrying + ``AgentMaxStepsError`` and returns normally, so without this check a run that + gave up is indistinguishable from one that answered. + ``MultiStepAgent.run`` detects it the same way to set its ``RunResult`` state + to ``"max_steps_error"`` (``agents.py``). + """ + max_steps_error = _max_steps_error_class() + steps = getattr(getattr(agent, "memory", None), "steps", None) or [] + if ( + max_steps_error is not None + and steps + and isinstance(getattr(steps[-1], "error", None), max_steps_error) + ): + return "length" + return "stop" + + +def _run_output(result: Any) -> Any: + # ``run()`` returns a RunResult when ``return_full_result`` is true, which + # defaults to the agent-level ``self.return_full_result``, so a plain + # ``run(task)`` can return one too. Read ``.output`` when present rather + # than importing RunResult. + return getattr(result, "output", result) + + +def _assistant_message(output: Any, finish_reason: str) -> OutputMessage: + return OutputMessage( + role="assistant", + parts=final_answer_parts(output), + finish_reason=finish_reason, + ) + + +def _tolerant_run_generator( + generator: Generator[Any, Any, Any], +) -> Generator[Any, Any, Any]: + """Delegate to the run generator, tolerating its refusal to shut down. + + ``MultiStepAgent._run_stream`` yields from inside a ``finally`` block + (``agents.py``), so closing it mid-run makes CPython raise ``RuntimeError: + generator ignored GeneratorExit`` and leaves the generator suspended. That is + the generator declining to stop, not a failed run: the steps that did run + succeeded. Returning instead keeps ``with agent.run(..., stream=True)`` from + reporting an error the caller never wrote, and leaves ``close()``, + ``__exit__`` and finalization to :class:`SyncStreamWrapper`. + + A generator that fails mid-iteration, or whose cleanup code raises, ends up + *closed*; that is how a real failure is told apart, and it propagates and + fails the span. + + This is a generator function rather than an adapter class so that the object + :class:`SyncStreamWrapper` proxies is a real generator. A proxy forwards + ``__class__``, so ``isinstance(stream, Generator)`` and + ``inspect.isgenerator(stream)`` keep answering what they answer for an + uninstrumented run. + """ + try: + yield from generator + except RuntimeError: + if getgeneratorstate(generator) != GEN_SUSPENDED: + raise + _logger.debug( + "The smolagents run generator ignored GeneratorExit on close", + exc_info=True, + ) + + +class _AgentRunStreamWrapper(SyncStreamWrapper[Any]): + """Keep the ``invoke_agent`` span open until the run generator is drained. + + The public :class:`SyncStreamWrapper` owns iteration, ``close()``, error + handling, and exactly-once finalization; this subclass only accumulates + per-chunk state and finalizes the invocation. + """ + + def __init__( + self, + stream: Generator[Any, Any, Any], + invocation: AgentInvocation, + handler: TelemetryHandler, + agent: Any, + ) -> None: + # The invocation is deliberately not passed to super().__init__(). That + # is what turns on the base class's per-chunk timing, which records + # gen_ai.client.operation.time_to_first_chunk and + # time_per_output_chunk. Both metrics describe the response stream of a + # generation call and require gen_ai.provider.name, while a chunk of an + # agent run is a step object, so this invocation keeps its own + # attribute. + super().__init__(_tolerant_run_generator(stream)) + self._self_agent_invocation = invocation + self._self_handler = handler + self._self_agent = agent + self._self_final_output: Any = None + self._self_saw_final = False + self._self_final_step_cls = _final_answer_step_class() + + def _process_chunk(self, chunk: Any) -> None: + final_cls = self._self_final_step_cls + if final_cls is not None and isinstance(chunk, final_cls): + self._self_final_output = chunk.output + self._self_saw_final = True + + def _on_stream_end(self) -> None: + invocation = self._self_agent_invocation + if self._self_saw_final: + finish_reason = _run_finish_reason(self._self_agent) + invocation.finish_reasons = [finish_reason] + if self._self_handler.should_capture_content(): + invocation.output_messages = [ + _assistant_message(self._self_final_output, finish_reason) + ] + invocation.stop() + + def _on_stream_error(self, error: BaseException) -> None: + self._self_agent_invocation.fail(error) + + +def agent_run(handler: TelemetryHandler) -> _Wrapper: + """Wrap ``MultiStepAgent.run`` to emit an ``invoke_agent`` span.""" + + def wrapper( + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + agent = instance + # An agent only has a name when it was given one, which is required of + # managed agents and optional everywhere else. + agent_name = agent.name or agent.__class__.__name__ + model_id = getattr(agent.model, "model_id", None) + invocation = handler.invoke_local_agent( + agent_name=agent_name, + # An agent runs exactly one model, which is the condition semconv + # puts on gen_ai.request.model for an agent span. + request_model=model_id if isinstance(model_id, str) else None, + ) + invocation.agent_description = agent.description + # Managed agents are callable by the model just like tools are: + # ToolCallingAgent hands them to it as tools_to_call_from and a CodeAgent + # calls them from generated code. Leaving them out would make the run + # advertise fewer tools than the chat spans under it. + invocation.tool_definitions = to_tool_definitions( + [*agent.tools.values(), *agent.managed_agents.values()] + ) + # gen_ai.client.operation.duration requires gen_ai.provider.name, and + # invoke_local_agent() exposes no provider argument. Resolve it from the + # agent's model and set it on the metric attributes, the way the + # openai-agents package does for tool spans. The local agent span + # deliberately carries no provider, so this stays off the span. + invocation.metric_attributes[GenAI.GEN_AI_PROVIDER_NAME] = ( + resolve_provider(agent.model) + ) + + bound = _bind_arguments(wrapped, args, kwargs) + if handler.should_capture_content(): + invocation.input_messages = [ + task_to_input_message(bound.get("task"), bound.get("images")) + ] + + # A managed agent runs inside the manager's step; its tools must not + # claim the manager's call ids. + token = _PENDING_TOOL_CALLS.set(()) + try: + result = wrapped(*args, **kwargs) + except Exception as error: # pylint: disable=broad-except + invocation.fail(error) + raise + finally: + _reset_pending_tool_calls(token) + + if handler.should_capture_content() and bound.get("additional_args"): + # run() appends the additional arguments to the task, so the + # effective task is only readable after the call. + invocation.input_messages = [ + task_to_input_message( + getattr(agent, "task", None) or bound.get("task"), + bound.get("images"), + ) + ] + + if bound.get("stream") and isinstance(result, Generator): + return _AgentRunStreamWrapper(result, invocation, handler, agent) + + finish_reason = _run_finish_reason(agent) + invocation.finish_reasons = [finish_reason] + if handler.should_capture_content(): + invocation.output_messages = [ + _assistant_message(_run_output(result), finish_reason) + ] + invocation.stop() + return result + + return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py new file mode 100644 index 000000000..f531606c1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/provider.py @@ -0,0 +1,151 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve a smolagents model instance to a ``gen_ai.provider.name`` value. + +Resolution order: + +1. For the LiteLLM model classes, the ``model_id`` vendor prefix + (``anthropic/claude-...`` -> ``anthropic``). +2. The model class name (e.g. ``OpenAIModel`` -> ``openai``), looked up along the + class hierarchy so a user subclass resolves to the provider of the base class + whose ``generate`` it inherits. +3. ``unknown``. + +``gen_ai.provider.name`` is a metric attribute as well as a span attribute, so +every value has to stay low cardinality: deployment-specific detail is reported +as ``server.address`` instead. ``TelemetryHandler.inference`` requires +``provider`` as a string, so this always returns a value rather than ``None``. +""" + +from __future__ import annotations + +import logging +from typing import Any +from urllib.parse import urlparse + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + +_logger = logging.getLogger(__name__) + +_PROVIDER = GenAI.GenAiProviderNameValues + +_UNKNOWN_PROVIDER = "unknown" + +# Model class name -> provider value. The GenAI registry has no value for the +# Hugging Face, vLLM, and MLX runtimes, so those use the product name; a class +# name would look like a provider without being one. +_CLASS_NAME_TO_PROVIDER: dict[str, str] = { + "OpenAIModel": _PROVIDER.OPENAI.value, + "AzureOpenAIModel": _PROVIDER.AZURE_AI_OPENAI.value, + "AmazonBedrockModel": _PROVIDER.AWS_BEDROCK.value, + "InferenceClientModel": "huggingface", + "TransformersModel": "huggingface", + "VLLMModel": "vllm", + "MLXModel": "mlx", +} + +# LiteLLM model_id prefix -> semconv provider value, for the prefixes whose +# LiteLLM vendor slug differs from the semconv value. Every other prefix is +# passed through as-is (``ollama/llama3`` -> ``ollama``). LiteLLM's slugs are a +# closed vocabulary, which keeps the cardinality bounded. +_LITELLM_PREFIX_TO_PROVIDER: dict[str, str] = { + "azure": _PROVIDER.AZURE_AI_OPENAI.value, + "azure_ai": _PROVIDER.AZURE_AI_INFERENCE.value, + "bedrock": _PROVIDER.AWS_BEDROCK.value, + "gemini": _PROVIDER.GCP_GEMINI.value, + "mistral": _PROVIDER.MISTRAL_AI.value, + "vertex_ai": _PROVIDER.GCP_VERTEX_AI.value, + "watsonx": _PROVIDER.IBM_WATSONX_AI.value, + "xai": _PROVIDER.X_AI.value, +} + +_LITELLM_CLASS_NAMES = frozenset({"LiteLLMModel", "LiteLLMRouterModel"}) + + +def _class_names(instance: Any) -> list[str]: + """The instance's class names, most derived first. + + Only the classes that define ``generate`` are patched, so an instrumented + model can be a user subclass of one of them. Matching the exact class name + alone would report ``unknown`` for every such subclass. + """ + return [cls.__name__ for cls in type(instance).__mro__] + + +def _provider_from_litellm(instance: Any) -> str | None: + model_id = getattr(instance, "model_id", None) + if not isinstance(model_id, str) or "/" not in model_id: + return None + prefix = model_id.split("/", 1)[0].lower() + return _LITELLM_PREFIX_TO_PROVIDER.get(prefix, prefix) + + +def _endpoint(instance: Any) -> str | None: + """The endpoint URL the model calls, or ``None`` if it exposes none. + + Three places can hold it, checked in this order: + + 1. ``api_base`` on the instance (``LiteLLMModel``). + 2. ``base_url`` or ``azure_endpoint`` in the SDK client kwargs + (``OpenAIModel``, ``AzureOpenAIModel``, ``InferenceClientModel``). + 3. ``base_url`` on the client the model constructed (e.g. openai's httpx + URL), which holds the effective URL when the caller left it at the + provider default. + """ + api_base = getattr(instance, "api_base", None) + if api_base: + return str(api_base) + client_kwargs = getattr(instance, "client_kwargs", None) + if isinstance(client_kwargs, dict): + for key in ("base_url", "azure_endpoint"): + value = client_kwargs.get(key) + if value: + return str(value) + client_base_url = getattr( + getattr(instance, "client", None), "base_url", None + ) + if client_base_url: + return str(client_base_url) + return None + + +def resolve_server_address_port( + instance: Any, +) -> tuple[str | None, int | None]: + """Return ``(server.address, server.port)`` from the model's endpoint URL. + + Models that don't expose an ``api_base`` / ``base_url`` / ``azure_endpoint`` + (e.g. ``LiteLLMModel`` resolving the host internally, local runtimes) + yield ``(None, None)`` and the caller omits the attributes. + """ + endpoint = _endpoint(instance) + if endpoint is None: + return None, None + parsed = urlparse(endpoint) + port = parsed.port + if port == 443: + port = None + return parsed.hostname or None, port + + +def resolve_provider(instance: Any) -> str: + """Return the ``gen_ai.provider.name`` value for a smolagents model instance.""" + class_names = _class_names(instance) + + if not _LITELLM_CLASS_NAMES.isdisjoint(class_names): + provider = _provider_from_litellm(instance) + if provider is not None: + return provider + + for class_name in class_names: + provider = _CLASS_NAME_TO_PROVIDER.get(class_name) + if provider is not None: + return provider + + _logger.debug( + "No known gen_ai.provider.name for model class %s", class_names[0] + ) + return _UNKNOWN_PROVIDER diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/agent_with_image.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/agent_with_image.yaml new file mode 100644 index 000000000..bc7672dcd --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/agent_with_image.yaml @@ -0,0 +1,75 @@ +interactions: +- request: + body: '{"messages":[{"role":"system","content":[{"type":"text","text":"You are + an expert assistant who can solve any task using tool calls. You will be given + a task to solve as best you can.\nTo do so, you have been given access to some + tools.\n\nThe tool call you write is an action: after the tool is executed, + you will get the result of the tool call as an \"observation\".\nThis Action/Observation + can repeat N times, you should take several steps when needed.\n\nYou can use + the result of the previous action as input for the next action.\nThe observation + will always be a string: it can represent a file, like \"image_1.jpg\".\nThen + you can use it as input for the next action. You can do it for instance as follows:\n\nObservation: + \"image_1.jpg\"\n\nAction:\n{\n \"name\": \"image_transformer\",\n \"arguments\": + {\"image\": \"image_1.jpg\"}\n}\n\nTo provide the final answer to the task, + use an action blob with \"name\": \"final_answer\" tool. It is the only way + to complete the task, else you will be stuck on a loop. So your final output + should look like this:\nAction:\n{\n \"name\": \"final_answer\",\n \"arguments\": + {\"answer\": \"insert your final answer here\"}\n}\n\n\nHere are a few examples + using notional tools:\n---\nTask: \"Generate an image of the oldest person in + this document.\"\n\nAction:\n{\n \"name\": \"document_qa\",\n \"arguments\": + {\"document\": \"document.pdf\", \"question\": \"Who is the oldest person mentioned?\"}\n}\nObservation: + \"The oldest person in the document is John Doe, a 55 year old lumberjack living + in Newfoundland.\"\n\nAction:\n{\n \"name\": \"image_generator\",\n \"arguments\": + {\"prompt\": \"A portrait of John Doe, a 55-year-old man living in Canada.\"}\n}\nObservation: + \"image.png\"\n\nAction:\n{\n \"name\": \"final_answer\",\n \"arguments\": + \"image.png\"\n}\n\n---\nTask: \"What is the result of the following operation: + 5 + 3 + 1294.678?\"\n\nAction:\n{\n \"name\": \"python_interpreter\",\n \"arguments\": + {\"code\": \"5 + 3 + 1294.678\"}\n}\nObservation: 1302.678\n\nAction:\n{\n \"name\": + \"final_answer\",\n \"arguments\": \"1302.678\"\n}\n\n---\nTask: \"Which city + has the highest population , Guangzhou or Shanghai?\"\n\nAction:\n{\n \"name\": + \"web_search\",\n \"arguments\": \"Population Guangzhou\"\n}\nObservation: + [''Guangzhou has a population of 15 million inhabitants as of 2021.'']\n\n\nAction:\n{\n \"name\": + \"web_search\",\n \"arguments\": \"Population Shanghai\"\n}\nObservation: + ''26 million (2019)''\n\nAction:\n{\n \"name\": \"final_answer\",\n \"arguments\": + \"Shanghai\"\n}\n\nAbove example were using notional tools that might not exist + for you. You only have access to these tools:\n- final_answer: Provides a final + answer to the given problem.\n Takes inputs: {''answer'': {''type'': ''any'', + ''description'': ''The final answer to the problem''}}\n Returns an output + of type: any\n\nHere are the rules you should always follow to solve your task:\n1. + ALWAYS provide a tool call, else you will fail.\n2. Always use the right arguments + for the tools. Never use variable names as the action arguments, use the value + instead.\n3. Call a tool only when needed: do not call the search agent if you + do not need information, try to solve the task yourself. If no tool call is + needed, use final_answer tool to return your answer.\n4. Never re-do a tool + call that you previously did with the exact same parameters.\n\nNow Begin!"}]},{"role":"user","content":[{"type":"text","text":"New + task:\nDescribe what you see in this image briefly."},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAYAAADDPmHLAAAQfElEQVR4nO2deXRUVZ7Hv7/3KpXKDoEEApFAcGMJ0LKL7AKiIoOSqI1IQMXT2tN9TvfMdKNIpZpuenB6PGdae47SLMHdFIILuIyySNhEgkAQcAGMQshCCmP2qnr3N+fFIBAIZKl6775Qn3+gUsu9Vff77rv3d38LoZ3hdDqV/O/QUxVKH1JwPZh7AOjOQHcAHQCOAxALUMx5b/MBXAmmKhAqAZQAfIqJisH0DQS+oTDxdd9kHHO5XALtCILFufthZ7LqpzEADwNhGBiDAIoISmNcL459rGAvGDsJvMWd7SqChbGcAKb+69/Do8rPTIAi7iDQRAA3mtylIwT+iKG8iyrxidvt8sJCWEIA45xOW2KBMlWwuJ+I7vhpCpeSHwn8rhDKK6d7iY+2uFx+SI7UArgnc/F1KmmPMuMBAF1gLUoAXq2w7YU3Vj91FJIiowBo5pysSUT4LYDbACiwNgLEHxLwTM4q18eQDJkEQBmZWdMZvBCgwWiPMPYw8V/XZLvW1T+SACkEMHNO1mQiLAX0FfzVAOcBygJ3tvOjq1oA9z6U1V9o/DeApuCqhD/UCI+tXeU6dlUJID39mQiKKv8DgxYAsOOqhmsIeJqrsMSMLaThAsiYkzWGCasB9DS6bck5AMGPuF907W6XAtD38p0LsIiAJwBSjWrXYgiAn62M6fSH95/9TV27EUD6bGcPKPQKCLcY0V47YC80dab7paeOB7uhoO+xZ2Zm3QcVB0KD3yJugur/LGOua4qVZwDKyHQ6GeQMYhvtHWbw0/1S8ESwTiGDIoD0dKedI2kFUb0JN0TbeRdVsfe63b+rgewCuGve0phwUZPTYMYNETi22n087ZVXXD9KK4D75jm7aYI2XD0WPaPhPJudbnttWdZp6QTQMPi5AFID9ZkhLgHxYWhhk9wvLjwJWQRw//yszn4vPgHQF/LhDbMp38XGRp9O6BxXFxMdJRwOGzkc4WpMTFS9PaKiokqrra3Tamv9XFFZpZSeLg8v/7Eqwe/XrpHUUnlMs/HYtctdJ0wXwKxZzlhvGDZJdIJXHdch+uAvBvSuGTG0X3KXLh1TVEWxteaDNCH8Rac8BTv3HDqx/8DXUeXl1brAIyEHB+w+Ht3WNUGbBDBtflakw8sfADQaJmO3hx2eNGHw6Ynjbhpqs6mOYLTh82k1H2/Oy9u4JS/B6/XfALNhfNwxPOn2Zcse9RkuAN379lABvQNAd9EyDbvd9uXsX06pHtg/9RdGtvv5/qOfv/z6h9E+n3YdTIQIL+asyspsrX9BqwWQPjfLBcYimAbXjBqZtmvmjLGjWzvFtxVNE7431mzaseuzw8MBBGXWaQ4MXrwm27XIMAHMfNA1lRReb5a7Fil0+rH500/ccO01Umw3Dx/57ovnV7yTxMzxZvWBQRlrsp3uoAvgvkxnTw2UB8CUL6sqyqkF/z7Ll5jQQQ/4kIaikjMFf/2vlyOYkWhOD/gMNAxyv+T6riXvatEVnJnpdGhMusrMUTpz5W8eu6dMtsHX6ZrYMeW3j99zhkBVMAXqqJ+4pqfnqEETQDXwNAhDYBLT7hi1r1fPrv0hKak9u90wdcqwfaZ1gHALRx1eGBQB3JvpHMGgx2ESMTGReyZNGCy9P8GUW4eOio2J+Nys9ol5Yfo85y0BFcD8+S+ECWCZiT76PH/unVGwAESEeQ/eYV5fCTYIytZD6Jrz8mYNqMdX9G8ApcEkYmMi81J6dOkDi5DaK+l6M2cBAL2jKs/8LiACSH/IeS2xeAomMmH8YEP84wLJ2NGDAn523+Jbge6K1+YZQKN/BC3cunl4Rw3vOxAWY9TNA9LAMDM4NJJULGmTABp80ibDRBwO+1fh4fZoWIxIhz3G7ggzNSiUQb/U3fBbKwBi5isqKNh0TexYBouS2CkuYI4brYQY+O/LGfyaFEDGHNfd9d6pJpOQGG/ZlCyJCfHm5wcgDEmf47q9xQJg8B8hARGOcCkCWFtDZKQkfaemF/GXFEB6pmuSmRa/83GE2yybHyA8wi6HAEDD0+dmTbjUM038uNysPaQR1Nb6LHsLqKvxSpEDoAE94caVBXDvnMW9zV75n4/fb/5ttLUIoUEaGHc2jO3lBcCK9qt2kJYlxMUoGvkfueiPjSN4GxIyhWiHEGh24+PiCwSQWFA/9VstG1eI5tONog/f2qQAGDSrBR8WwoII5vsvKYCfjg95mim9CmEYBEzXg3cvEkBM5ZlxjRIoh2ifdKBojLlIACz4TtO6FMJYuD7dbqM1AMmz9w8RXBisJ9k+J4C7H/hLEoDrg9xuCGmg/g1j/pMAFJtvrNldCmEopNj8t5y7BRBGGtt+CNNhHvazAIhZltDuC1BV5bupU4abXRCi1dw2ecQNqqp8DwkhYj2eEUqDaVCKGLtLuIJ74mKjusKidIiL6vJw5h0/yJIZ/AKY9GhqUkTUl6kASedzHx3t2NfnxhQZhdki+vXpmRYV4TgA2SBEp892XqOo8Jsa394UA/qllqOdkJaWqs8C0kE2pY8CpmshIV26dpIxN0+rSOoaL+V3EYzrFSaWMqtXuM26rmCNsdvtUn4XhThZYaJ6g0CIqw9mJCvEsOwqO0Rb4e761GRSRosQEtBBkbgIY4igQ3GKRIkPQxgOhwRwlaNvTzhUv+eqhVQFIImiF85R47VuRFBj/F6/fGcBP6HoM4CUAjh1ymOpMuyXo7C4TNIMJ6zpi0ApY68OfnHUtKybgSb/4NEOkJNKXQABLUESKKpr6gYcOvKtfKdoLeSLw9/mV1bVDpRZAB5IyrKVGzqV/1hdAovyww8VJcuzN3SEvMgtACFEd+efV/qLS34ogMXIP3R8f9aS1X5NE8mQFirRj4NLITFCcLf1H+wIegXNQCEEa09mLf/8nyvXD9T7DrkpVBj4FpLz1ZffJ8Ai7Nv/zb6KyhpDi1e0gZOKApJeADV13j4VVTVmZ9xqFh9s/KwWFoGYTlhiBtANFltz9x+C5NTWeauKik5bxo9RAEcUTRGHYQFytx2QeTVdz/r3d+bJ6GDbFDZVHFbWrnIdB1h6B8zq2rq0wlNlpmbevNLib/uOg71gGbj89ZWuQn0byADthwVwr9vS5kKJwWLHzi/2aELohSatAdFe/Z8GZ0U2M7V5szl67OSQiqpq6VLHMrN4671tspp7Lwkz7z4vNEzRa/5aAIrKefMT6czDG7fk7fLW+cwvJNkyzgnAJ1gXgKxHlhewP//okIqKGmnMw36/8G54/9MUWAsOs9m3/yyAdS9l6T+oJXYDYI5Z+eJ70vT1FfdHuzQhusNa7H9t+ZPF+n9+Dlgg4P9gEY4eL7y5oKD4S7P7cbKw7Hhe3lf1UbZWgujcWJ+LWBH1dYCtQthzL6zT/EK0umhyIBZ+zz7/ZqUexASLwcCHFwmgpJfIlflksDF1Xl/fHPemHWa1/1rOxm3V1XWmFdJqPVxa2oO3XiSALS6Xnwh6PWDLsOuzQ8NOnjx9zOh28w8e37dr9+GbYUEYeEsf67OPGwUt8kuwFBTxzHNuoU/HRrVYU1X7w/LVG7rX1+ezIATlggLTFwigTw9sAljKlCZN4fP5r9XYOMfWM+VVHma2zPH0BTBOoKrPpiYF4HK5BDFeNrxjIYxBwUq3O+OCi+WiuHW/LewFWV3FQ7QJoSm2lY3/eJEA1q5YWADQ221rK4R0EL/909heyCUzVxDjfwzpVAjDIKHq9QPRLAHkrM7aCtSfD4RoH+zOWb2o3vbfmMvkrlEWB7FDIQyEwK6mnmtSAO5s50cAX1I1kiFUIsOSMDGxJLUAmwljW062672mnr7sD8dQF0ByFKJTRGRYiHtURLhlfP50mJQnLvf8ZQWwJntRLgEXWI5kIzo28pSR7cXFRScQ6AysAPE6fQwv95IrTp0K2X6v+2RCUm4de5OhJ4JERKm9uuVDergG/vqxQ5sE8Pqqhd+DeSkkRLWpx0ePGmB4jeOM9PEpYDnD6s/CUJa6X3rqiiF1zVs8VeM/wTgIiSBQ1e9/naGpqhJmdNtJiR1Txo4ZZNpRdDM4Eg3RrIu2WQJwu11eoSgPy2IiVhTl5K8emf5tcnJn0/Ic3zN99Jh+fVO3QDpYE6RkZme7mhWi1qItTcYc5xImMm9nwPCnpHTZ+dhDd6VFRDmkcMPenXdkzxvuzfE+v1+KnMsMXrom2/XH5r6+RWfaJT2xKKEAen0hw50hYqIj985/aFpMyjWJoyERwwbfOGToTTeI3O35O99ev72LqUJg7KFqLGrJW1ps1EifvbgXVP/nepZJGIDDYT8454HJ/n439pI+6FIIFpu37v/svQ92JhgvBC7XCDetXeVqkYdUq6xaM+c67ySuPzEMmgXOpioF/zJtdOHoUWkj9K0XLIQwXgjM4Jlrsl1rW/rGVv+wGZnOJxj0FwQYIniGD+mTnzFz/AibqlrO4/Z8dFe13O35nwb91sDsdK92/ak1b23LlUUZmVmvMnAfAgJXpfVP3fXA/VOGRoSHtasE1iKIMwITctasyrqvtZFdbXFs5IqY+MzoCo/uHzex9Z8Cf/I1idvmz72zT4e4qNZ/jsQoCikTxw0aPn7MQF0InwZOCJxLlXGZbQnra/O99a55S2PCRfVmgAa3YWUvZeEquW8NnB/mdYx99dUFbTqXCMjiasbsrESbgo0g9G9vK3s5bw38NYAx7mxXUVv7ELDVdfrcJQlg78d6xbemXqOqyrczpo0+ZcWVvTwzAn8JETbR/eLCk4FoO6CDUC8C1H3YUJXyXCNEZePHDPpi2u03jzTDdm8VhGBt89a9uzZ8sDvJf0khcL7NZp90NrI3EAT8Kpw1yxnrtdGbINyqP+7WrfO2Xz86o290lKPdJH8ONpoQ/nXv5O7I3Z4/kJkbDG6cG+Z1TG/rPb8xQZmG09OddiVa/eeUSUNTp04eVl+mPETLKSsrL3z6mdfP1NZ5D0WCH2zuAU9LCNp9mJmVIo9nIRjOYFoM2zns8/n/vOL5/83So7aC0UDQF2KnSkvHg5TX9GqwwW6rnVEGQQ8mJcY36dAZCAxZiZ8oK0tWGTkARhrRXjsgj2xqetcOHYKeJNuQqTm5U6cTZZ3ixwH1rmVSu1KZjAbC38o6xd9sxODrGL4XL/Z4BgjBKwAY7ssnNYwDIH4kqXPn+vRtRmGKMYaZbcUez+PM0E8TLeVnHwRqwVha1jl+SX8iwwtlmWqNKykpuU5T1CW6i53ZfTEBPcToLUVo/5GYmPgNTEKKH73I40ljwU/pJgRcBRBju2Ba0C0x3vQAXCkEcJai06cnNDiZjED7ZCdYPJmUkLAZkiCVAM5SWFo6WFGU+cyYDSAC1sYL4G0CL+vaubN+WCYVUgrgLMXFxV2EGpYJ8GMAesBaFAG8WiN6Tt8GQ1KkFsBZmFktLisby1DuBniGfsYEOSkE+C0C1nbp1OkTIpLe5mEJATQ+YyjxeIYLfefAmAygn4lnDQLAIRDeV4B1ifHxnxKRpYpeW04AjfF4PHG1zCNJYCQIowDoyZujg9RcBQOfEmMHK9jpINoZHx8vfbmddi2AS1FaWtrNq6qpihC9wZRKxL0ZlKSH9zcYnvSFZex5QtGTPlc0hMFXAfQjQZxkpmMgPsZCORZG2tGEhIRCk79awPl/+oMNTPkSoNoAAAAASUVORK5CYII="}}]}],"model":"gpt-4o","stop":["Observation:","Calling + tools:"],"tool_choice":"required","tools":[{"type":"function","function":{"name":"final_answer","description":"Provides + a final answer to the given problem.","parameters":{"type":"object","properties":{"answer":{"type":"string","description":"The + final answer to the problem"}},"required":["answer"]}}}]}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-DVJPdiwp16bKXEYMV5LTiliNkq9Rf\",\n \"object\": + \"chat.completion\",\n \"created\": 1776355161,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_UemzWkeWxboyk7wCbBNRfUZp\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"final_answer\",\n + \ \"arguments\": \"{\\\"answer\\\":\\\"The image is a generic + user avatar icon, typically used as a placeholder for a profile picture. It + features a silhouette of a person against a circular background.\\\"}\"\n + \ }\n }\n ],\n \"refusal\": null,\n \"annotations\": + []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 1102,\n \"completion_tokens\": + 44,\n \"total_tokens\": 1146,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_07a5e8f420\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml new file mode 100644 index 000000000..6177ff059 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/litellm_reasoning.yaml @@ -0,0 +1,20 @@ +interactions: +- request: + body: '{"model": "claude-3-7-sonnet-20250219", "messages": [{"role": "user", "content": + [{"type": "text", "text": "Who won the World Cup in 2018? Answer in one word + with no punctuation."}]}], "thinking": {"type": "enabled", "budget_tokens": + 4000}, "max_tokens": 8096}' + headers: {} + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"id":"msg_011KX7d4TtALbugymC3Kb4oE","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[{"type":"thinking","thinking":"The + World Cup in 2018 was won by France. They defeated Croatia 4-2 in the final + match in Moscow, Russia.\n\nI need to answer in one word with no punctuation, + so my answer should simply be:\nFrance","signature":"ErUBCkYIBBgCIkAOLwDXty2UNgzsRPd4O0tNxqhaxPqqLw9isc2bDqyVS7Y87Cefkib8FVE9iTTjTSynEjrrHb+vei4K5pQrVOUrEgzCGfpi49gURMlFYTwaDGTEJyP22/PoNkje2CIwyiqcZqRj6rxDwzG0ayl3JQ2mi8x3iDIEuyP92Pw19EPRhATbpYiORzbaVoPUBuPEKh3gUYNSZwcjUAYNO01Qqo7GYFF08xO0MuULheotLRgC"},{"type":"text","text":"France"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":54,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":65,"service_tier":"standard"}}' + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml new file mode 100644 index 000000000..44f70cc65 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_basic.yaml @@ -0,0 +1,25 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"Who won the + World Cup in 2018? Answer in one word with no punctuation."}]}],"model":"gpt-4o","max_tokens":4096}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb\",\n \"object\": + \"chat.completion\",\n \"created\": 1738649686,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"France\",\n \"refusal\": null\n + \ },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n + \ ],\n \"usage\": {\n \"prompt_tokens\": 25,\n \"completion_tokens\": + 2,\n \"total_tokens\": 27,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_4691090a87\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml new file mode 100644 index 000000000..9fde9a56e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_image_url.yaml @@ -0,0 +1,26 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What breed + is this dog?"},{"type":"image_url","image_url":{"url":"https://fastly.picsum.photos/id/237/200/300.jpg?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U"}}]}],"model":"gpt-4o"}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-DVJBfdzKCrLtPYM9ui8SZgIEQv857\",\n \"object\": + \"chat.completion\",\n \"created\": 1776354295,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"This looks like a Labrador Retriever + puppy. They are known for their friendly and outgoing nature.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 268,\n \"completion_tokens\": 18,\n \"total_tokens\": 286,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_07a5e8f420\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml new file mode 100644 index 000000000..a5a57c069 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/openai_model_tool.yaml @@ -0,0 +1,32 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What is the + weather in Paris?"}]}],"model":"gpt-4o","max_tokens":4096,"tool_choice":"required","tools": + [{"type":"function","function":{"name":"get_weather","description":"Get the weather for a + given city","parameters":{"type":"object","properties":{"location":{"type":"string","description": + "The city to get the weather for"}},"required":["location"]}}}]}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-Ax7BZfEQe2evzgqbTVXWo0ZqoMIUf\",\n \"object\": + \"chat.completion\",\n \"created\": 1738652337,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_EUuviydGIG5Jau3DLw4v4cue\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n + \ \"arguments\": \"{\\\"location\\\":\\\"Paris\\\"}\"\n }\n + \ }\n ],\n \"refusal\": null\n },\n \"logprobs\": + null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": + {\n \"prompt_tokens\": 61,\n \"completion_tokens\": 15,\n \"total_tokens\": + 76,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n + \ \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_50cad350e4\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py new file mode 100644 index 000000000..be0d0071e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/_helpers.py @@ -0,0 +1,34 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for smolagents conformance scenarios.""" + +from __future__ import annotations + +import json +from typing import Any + + +def attr(span: dict[str, Any], name: str) -> Any: + for entry in span["attributes"]: + if entry["name"] == name: + return entry["value"] + return None + + +def chat_spans(report: Any) -> list[dict[str, Any]]: + return [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + + +def part_fields(messages_json: str | None) -> list[tuple[str, str | None]]: + messages = json.loads(messages_json) if messages_json else [] + return [ + (part["type"], part.get("modality")) + for message in messages + for part in message["parts"] + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py new file mode 100644 index 000000000..93ec7a241 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py @@ -0,0 +1,84 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario for the agent operations (``invoke_agent`` with a nested +``execute_tool`` and ``chat``).""" + +from __future__ import annotations + +import pathlib +from typing import Any + +from smolagents import OpenAIModel, ToolCallingAgent + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class AgentScenario(Scenario): + # ToolCallingAgent runs one step: one chat call whose final_answer tool call + # is executed as one execute_tool span, all under one invoke_agent span. + expected_spans = {"invoke_agent": 1, "chat": 1, "execute_tool": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + from PIL import Image # noqa: PLC0415 + + image_path = ( + pathlib.Path(__file__).parent.parent / "fixtures" / "img.png" + ) + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("agent_with_image.yaml"): + model = OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + agent = ToolCallingAgent(tools=[], model=model, max_steps=3) + agent.run( + "Describe what you see in this image briefly.", + images=[Image.open(image_path)], + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + agent_names = { + attr["value"] + for entry in report["samples"] + if "span" in entry + for attr in entry["span"]["attributes"] + if attr["name"] == "gen_ai.agent.name" + } + assert "ToolCallingAgent" in agent_names, ( + f"expected the agent name on the invoke_agent span, saw {agent_names}" + ) + tool_call_ids = { + attr["value"] + for entry in report["samples"] + if "span" in entry + for attr in entry["span"]["attributes"] + if attr["name"] == "gen_ai.tool.call.id" + } + assert tool_call_ids, ( + "expected the provider's tool call id on the execute_tool span" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py new file mode 100644 index 000000000..b0b3abd4f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/inference.py @@ -0,0 +1,122 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios for the ``chat`` operation (plain and tool-calling).""" + +from __future__ import annotations + +from typing import Any + +from smolagents import OpenAIModel +from smolagents.models import ChatMessage, MessageRole + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +from ..test_utils import GetWeatherTool # noqa: TID252 +from ._helpers import attr, chat_spans, part_fields + + +def _openai_model() -> OpenAIModel: + return OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + + +class ChatScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_basic.yaml"): + _openai_model().generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? Answer " + "in one word with no punctuation." + ), + } + ], + } + ] + ) + + +class ToolCallingScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_tool.yaml"): + _openai_model().generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What is the weather in Paris?", + } + ], + ) + ], + tools_to_call_from=[GetWeatherTool()], + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_part_types = { + part_type + for span in chat_spans(report) + for part_type, _ in part_fields( + attr(span, "gen_ai.output.messages") + ) + } + assert "tool_call" in output_part_types, ( + f"expected a tool_call output part, saw {output_part_types}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py new file mode 100644 index 000000000..5e8025c1b --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/multimodal.py @@ -0,0 +1,163 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios for non-text message parts: an image ``uri`` on a chat +input, and a ``reasoning`` part on a chat output.""" + +from __future__ import annotations + +import os +from typing import Any +from unittest import mock + +from smolagents import LiteLLMModel, OpenAIModel +from smolagents.models import ChatMessage, MessageRole + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) +from opentelemetry.test_util_genai.instrumentor import instrument + +from ._helpers import attr, chat_spans, part_fields + +_IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +class MultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("openai_model_image_url.yaml"): + model = OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What breed is this dog?", + }, + { + "type": "image_url", + "image_url": {"url": _IMAGE_URL}, + }, + ], + ) + ] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + input_parts = { + fields + for span in chat_spans(report) + for fields in part_fields(attr(span, "gen_ai.input.messages")) + } + assert ("uri", "image") in input_parts, ( + f"expected an image uri input part, saw {input_parts}" + ) + + +class ReasoningScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + expected_violations = ( + # LiteLLM routes to the provider host internally and a LiteLLMModel + # built without an explicit api_base exposes no endpoint URL, so there + # is nothing to derive server.address from on the chat span. + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + env = {"LITELLM_LOCAL_MODEL_COST_MAP": "True"} + with ( + mock.patch.dict(os.environ, env), + mock.patch("tiktoken.get_encoding") as get_encoding, + ): + get_encoding.return_value = mock.MagicMock( + encode=lambda *_: [1, 2, 3] + ) + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("litellm_reasoning.yaml"): + model = LiteLLMModel( + model_id="anthropic/claude-3-7-sonnet-20250219", + api_key="test_anthropic_api_key", + thinking={"type": "enabled", "budget_tokens": 4000}, + ) + model.generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? " + "Answer in one word with no " + "punctuation." + ), + } + ], + } + ] + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_parts = { + part_type + for span in chat_spans(report) + for part_type, _ in part_fields( + attr(span, "gen_ai.output.messages") + ) + } + assert "reasoning" in output_parts, ( + f"expected a reasoning output part, saw {output_parts}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py index da97f2987..baebcbf9c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py @@ -5,22 +5,103 @@ from __future__ import annotations +import os +from unittest.mock import MagicMock, patch + import pytest from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) from opentelemetry.test_util_genai.instrumentor import instrument +from opentelemetry.test_util_genai.vcr import ( + scrub_response_headers_overwrite, +) + +pytest_plugins = [ + "opentelemetry.test_util_genai.fixtures", + "opentelemetry.test_util_genai.vcr", +] + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_openai_api_key"), + ("x-api-key", "test_anthropic_api_key"), + ("openai-organization", "test_openai_org_id"), + ("openai-project", "test_openai_project_id"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers_overwrite( + { + "openai-organization": "test_openai_org_id", + "openai-project": "test_openai_project_id", + "Set-Cookie": "test_set_cookie", + } + ), + } + + +@pytest.fixture +def litellm_local_cost_map(): + """Use LiteLLM's bundled model-cost map so it doesn't fetch prices over the + network during cassette playback.""" + previous = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + try: + yield + finally: + if previous is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = previous + + +@pytest.fixture +def patch_tiktoken_encoding(): + """Patch ``tiktoken.get_encoding`` so LiteLLM doesn't download an encoding.""" + with patch("tiktoken.get_encoding") as mock_get_encoding: + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + mock_get_encoding.return_value = mock_encoding + yield + + +@pytest.fixture +def instrument_no_content(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="NO_CONTENT", + ) as instrumentor: + yield instrumentor -pytest_plugins = ["opentelemetry.test_util_genai.fixtures"] + +@pytest.fixture +def instrument_with_content(tracer_provider, logger_provider, meter_provider): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ) as instrumentor: + yield instrumentor @pytest.fixture -def instrument_smolagents(tracer_provider, logger_provider, meter_provider): +def instrument_event_only(tracer_provider, logger_provider, meter_provider): with instrument( SmolagentsInstrumentor(), tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, + content_capture="EVENT_ONLY", + emit_event=True, ) as instrumentor: yield instrumentor diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/fixtures/img.png b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/fixtures/img.png new file mode 100644 index 0000000000000000000000000000000000000000..d412166079109b2796fd9ccd719263a974087cc3 GIT binary patch literal 3594 zcmV+l4)yVgP)8*ZnHXbN#mejsK~x`h%IdRDNml~HXImAJiph?I6bx^tcMn^`!YO{4vA$6!%wT$QO45bl-N2!QXdoqb}de?tuW|U zj9ufD)JGg5%i#w9RL2Qa_>>1jHUuc78Rx^+6!n;tc9&CDR&&Gl=op(iUjB*5h!fMk zkMm_~e9du)%CETr-{bV_7}jd^^RmhNPMJM`GiqzJ=J3+`ANW(_)-hsqASzY7>=Fxk z#hIVg8hPStGzN=WAjs7Ru^nce;*ymgaAWM(8fdn__CuQmwoBAY5Y}AJOUnv3!g;Mx zW*e+;$QHAu3h5M;U-FEf&>CXaQzy)4cVl}+ji6r43qUR&@t1TmZ!=6kv}i(O7q*|J zCCp>rfhJU)4BO~aEzC>19ox~88fh~>t&>TcabW?cto{bu(@L5%Qqa@MoXsQ|;GfME zmg|Lt5-qRWZ`R3>&Gh&W)SR%LN$Sz0l{fGN@6*YI&CDoCOk_7=dy`Up61zp1RV{i( z!`yRIm9JyFlhBCsTeXR{2`e&d6y~SjitTTzAS#8S;Z`~cHlOneDX!xS{<`MFe=mEz zIAh(m!gU*W7H!=0PU+U&g%tq&4gU|0!8vg5+?2Ey_^rgz4XTJ;_8OxmQJ7lq9+g>oDT3?cipj;%-44pr!2~-f@|yC&n)UpFB3eHs(ghz zzf>o2hLf1OwDGNN`%61|eCMzu!^J6&ukVeVePxNM=|}PXO+i!+rlxNA684KBg1RYB z#JieA84NgMe{t1e?5s%he!+haCp?kXjMHN@PFZ<7#*Fa=f~=@GF~-9BUf8g+a3-(# znbPU?UwHnd?ZrGj=d|fF4(MaS{)woD#>tyFT+5lH`im=TYp_GiqOiQ?h*MUBIGsk~ zkW>NtDI$uO*!TDiP0MV2lB>I}skI$D!E9<8TTb%Vu@|S;I9Lce-LSvHDoJA3!MHZ< z%?(0iXAikQZotN;LWR5Gs8dpVHR(1A=y4Ttuf~20n}#Pq=-k*KHsu~D!$zmZn_Ksl z<8&Jd{}eFnx3DNqVm}p3{O%_$%&5Y~rbxFxFqph>fHDSMzSTZpd1^RI-7Dofa&O(yco!5VW}1H3w@w-aU(f@vdeQ88<;w+ot`s8-(zil z3nv0o0M%MO1Y*?$Xd~FW&6KL{eIHLM6_V0u;wZ-oW zGGE(MgbiyIUfi&&1n0*bSS9xFxEfmSL~{{ycy;r;CD^c5;gwDAkQgg5gsj|${XC`w zLZf;!U)^1V4QmzXya5ylfW6qyx`HK51QVNlF>BZFD8h!d3TxNr(g^_W(@8B?GqNTR z{;_G;0T4c5mJ#;%D8UR~f!Kmfb^wHxu5r7Iof7+<&W}vR4uJ47?u1i*!WR4kv#|pp zyf`G~E$sIpLHD7cT|?J&>;MQmpXe?}L!sSY4?6(DkM5dqh-Xlgyx!}D+o=ZglcblCYq?$!Js=FMgq zn}Nde!(`_FO_%(#Y8XyGpp!py5tXwsFj5FMWzZZXfc(q~!>4>l3hAj8wK#tyfKkH~ z+y4Rn{Vc+Uv?|ZPoJ(V97-Fbjz$q$!#s?^nv%8=I8`7%eVDLQz0J)LCfcVt%I&4U*B0o*E zGe}}S2!KxQ0U8vPS6e;^M!JJFd7}Yqg8(3#z_=t==h3zfFE*-GXlgm>jZb!U;C#`6 z)k6Rf{XPv7m!x*pHnyI`#-+}IQgxL|obHaImhU&|vowqh(` zuSXaeK1Z;~?%uEs95y-?pzs%;M0E~;9_;iL6GLc@3c@UQZ6v#D=#IT5R6Bh1k(I(L z0)XbgP%W&txxFKRjc+#H{y;y0b%CeEs0E{_EDJjs(${2@TRL|04%3&0pE5iEVlM~4 zALl9G6evz#(nD6yG@s}s(>&bK<3G>8{bc)7#$aG>%F>%I2LMQ*yZN(*#VVb$z>3Tq zOO&4wHKUvlf@MQ=PNd3(%Ie0W*cnFSu5WJRY580yeWV$h5Q62LuilGKq|D=~OF!=L z`7dCn7zGsk#6K-3+mkvjdJ9L7tP*P4j-b;4_U*-vF$#WfE57;%VGs!T8vB1puoJcR z2b5ub?z&^G*g28rK(V{V?EQeI3ord|#D2dlSU(Q`fI10Jq#uXZF?Ld;RY;r|9}Q z_Vc*np8+~MUQK`<02A^N_V0wEv~}D>*a0v;uo4{h@1!IUT|pGl-|Y`v!j6g*?uKK~ z{*Ti@7f~t0ex0-&lIs_m3lbwvRy(niA_>g)!D*lj_z?SbN^(j{F1{!wk{7M2!cK}L z{B1Mfl++MT2T7(<@*hj_a{9OV#Xi$qoCsU<3Jzf>MViIxs}|@8VtvBAwA-;irzJsT zOSCsIpZJutwob2c1;8&j+}LuGdS{SBQW^H+v<16N@eDWMo0tiI4fj^n9y^YmV-#7h z@1gt_Xp$s$9rojl^qB1W36&;dm^NNiQEU2qK_z2T5l)-=uuI@8?7y(c^o}C_+Wp&- z^`+1yi=AN>dq1eCN=Qj3(>5^S{=hTQ$fzMxhCnAhlTlq+)6|R|Kmiz!uzgR_A=3jv zRH}40g$$>lWNGA7vUv-ZHRPAOEwOV*moLzty|bXwoIqfwVV)=US48BH+3)#H`?ZM( zA9IskC*i4rdN(#Sbxa6^fl(7ZjF3}e&*>y&#yDjq2!lO4yuQz|VXcN?p9Wq{$Nn)w zbDWa0Rg;)$!+Kq--_CFA>iHBK(&|L=jR5;SfG@Cf4AJn8<4QhG)Rf>Q`)B|4auIB? zh7A{Ogg+2isUWPm9{a^mJ({%g2A+^a>adKk><~6ILzRN0O<%5!g{-e)_H0{v`}v!1_1$mYC6TV6WNXYb8zsBL@#|y&_MjpTaUY?Ptuj5C6fX5e_-e_KG>I?>N;~6r1 z=$3_X_nbZ*p5a_nbYyVwT2CObf@kicPKIrUxx_!4tH4&+eo-TDDD)$qQA0=aS^|Ht zjz1uGV|zuVcmeU6o?vhh&v>uK5VMbS-1$mWKlNkD=kwjg4X}eV8(RhkdIG^bZi73q zU1ptnLZKgWgXHQMDJtpl2g|#Cp_$FtiAMDV0{3wPMa_<(Ivj#(G0wLwq;9|e$GnWN zfg7w($B0(rJSUe2_X9YewkD9*>%GYn2>pW_uwBQn)bT*I7urB@zHCiVyw-5_@aG|v z#SPn~V{9t$v;|(`X}uq6IyfJ;!XW5l@CJiF^8|ydxxrx@88iDD;NA^9Eg6s|NE=SC ztq3a=3VpRZ82l}-ORL-&`JA&ljtUoNx&y(3a4)zg+#9FI76JMF{#&|2p?mo2SOjPB zS8_IYUIE9=Y2v_6q#+IrPyyg?C}+Sieom;6|7P=WPz6M&vcWa+eWF4C12a215GluW Q+5i9m07*qoM6N<$f?RLq=>Px# literal 0 HcmV?d00001 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt index d833435b7..649ffc343 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt @@ -26,7 +26,10 @@ # This variant of the requirements aims to test the system using the newest # supported version of external dependencies. -smolagents +# openai/litellm are the model backends exercised by the VCR model tests; they +# are test-only (not declared in pyproject.toml) so they live here. +smolagents[openai,litellm] +wrapt>=2.2.2 -e util/opentelemetry-util-genai -e instrumentation/opentelemetry-instrumentation-genai-smolagents diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt index 1d9b353f3..afb682e96 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt @@ -21,5 +21,15 @@ # pyproject.toml is the single source of truth. The OpenTelemetry SDK and test utilities # come transitively from opentelemetry-test-util-genai. # -# There is nothing to pin yet: the lifecycle tests need no model backend. The pins for -# the backends the model tests drive arrive with those tests. +# openai and litellm are the model backends the VCR model tests instantiate. They are +# test-only (not declared in pyproject.toml) and smolagents does not pull them in through +# the instruments extra, so pin them here to the versions smolagents 1.24.0 declares in its +# own openai/litellm extras. litellm resolves to its floor; openai resolves to 1.61.0 +# rather than 1.58.1 because litellm 1.60.2 itself requires openai>=1.61.0. +openai>=1.58.1 +litellm>=1.60.2 + +# The declared opentelemetry-util-genai floor carries the streaming metrics and the +# gen_ai.request.stream attribute, which are not in a published release yet. +# Drop this once opentelemetry-util-genai 1.1b0 is published. +-e util/opentelemetry-util-genai diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py new file mode 100644 index 000000000..4e54d39e8 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py @@ -0,0 +1,704 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Agent (``invoke_agent``) instrumentation tests: a VCR-backed run, streaming +and non-streaming runs against fake models, managed-agent nesting, executor +context propagation, run outcomes, and the failure / early-close / recovery +lifecycle paths.""" + +from __future__ import annotations + +import inspect +import json +import pathlib +import tempfile +from collections.abc import Generator +from types import GeneratorType, SimpleNamespace +from typing import Any + +import pytest +from smolagents import CodeAgent, OpenAIModel, ToolCallingAgent +from smolagents.models import ( + ChatMessage, + ChatMessageToolCall, + ChatMessageToolCallFunction, +) +from smolagents.monitoring import TokenUsage + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import error_attributes +from opentelemetry.trace import StatusCode + +from .test_utils import ( + CODE_FINAL_ANSWER, + BrokenTool, + FakeCodeModel, + FakeStreamingCodeModel, + ImageTool, + NeverFinishingCodeModel, + attr, + data_point_attributes, + metrics_by_name, + parse_messages, + part_types, + spans_by_operation, + stub_streaming_openai_client, + text_chunk, + usage_chunk, +) + + +def _operations(spans: list[Any]) -> list[str]: + return [ + (span.attributes or {}).get(GenAI.GEN_AI_OPERATION_NAME) + for span in spans + ] + + +def test_tool_calling_agent_with_image( + instrument_with_content, span_exporter, vcr +) -> None: + from PIL import Image # noqa: PLC0415 + + model = OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + image_path = pathlib.Path(__file__).parent / "fixtures" / "img.png" + agent = ToolCallingAgent(tools=[], model=model, max_steps=3) + with vcr.use_cassette("agent_with_image.yaml"): + agent.run( + "Describe what you see in this image briefly.", + images=[Image.open(image_path)], + ) + + spans = span_exporter.get_finished_spans() + (agent_span,) = spans_by_operation(spans, "invoke_agent") + (chat_span,) = spans_by_operation(spans, "chat") + (tool_span,) = spans_by_operation(spans, "execute_tool") + + # No step/chain span is emitted; the three GenAI operations nest directly. + assert sorted(filter(None, _operations(spans))) == [ + "chat", + "execute_tool", + "invoke_agent", + ] + assert agent_span.name == "invoke_agent ToolCallingAgent" + assert chat_span.parent.span_id == agent_span.context.span_id + assert tool_span.parent.span_id == agent_span.context.span_id + assert attr(chat_span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + # Token usage belongs to the model call. Repeating the run total on the + # parent span would report the same tokens twice on + # gen_ai.client.token.usage, which the util emits from these fields. + assert attr(chat_span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 1102 + assert attr(chat_span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 44 + assert attr(agent_span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) is None + assert attr(agent_span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) is None + + # The agent passes both the task and the model messages positionally. + agent_inputs = parse_messages(agent_span, GenAI.GEN_AI_INPUT_MESSAGES) + assert agent_inputs[0]["role"] == "user" + assert agent_inputs[0]["parts"][0] == { + "type": "text", + "content": "Describe what you see in this image briefly.", + } + assert agent_inputs[0]["parts"][1]["type"] == "blob" + assert agent_inputs[0]["parts"][1]["modality"] == "image" + chat_inputs = parse_messages(chat_span, GenAI.GEN_AI_INPUT_MESSAGES) + assert [message["role"] for message in chat_inputs] == ["system", "user"] + assert part_types(chat_inputs[1:]) == ["text", "blob"] + + +def test_code_agent_non_streaming( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + result = agent.run("Test question") + assert result == "Test result from CodeAgent" + + spans = span_exporter.get_finished_spans() + (agent_span,) = spans_by_operation(spans, "invoke_agent") + (tool_span,) = spans_by_operation(spans, "execute_tool") + + # FakeCodeModel is not a wrapped model class, so there is no chat span. + assert "chat" not in _operations(spans) + # Executor context propagation: final_answer nests under the agent span. + assert tool_span.parent.span_id == agent_span.context.span_id + assert attr(agent_span, GenAI.GEN_AI_REQUEST_MODEL) == "fake-model" + assert attr(agent_span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert json.loads(attr(agent_span, GenAI.GEN_AI_TOOL_DEFINITIONS)) == [ + { + "type": "function", + "name": "final_answer", + "description": "Provides a final answer to the given problem.", + "parameters": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The final answer to the problem", + } + }, + "required": ["answer"], + }, + } + ] + # run(task) is positional, so the task is only recorded if the wrapper + # binds positional arguments to the signature. + inputs = parse_messages(agent_span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0] == { + "role": "user", + "parts": [{"type": "text", "content": "Test question"}], + } + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"][0]["content"] == "Test result from CodeAgent" + + +def test_code_agent_no_content(instrument_no_content, span_exporter) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + # Metadata stays; the task and the final answer are omitted. + assert attr(agent_span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert attr(agent_span, GenAI.GEN_AI_TOOL_DEFINITIONS) is not None + assert attr(agent_span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_additional_args_reach_the_recorded_task( + instrument_with_content, span_exporter +) -> None: + # run() appends the additional arguments to the task, so the effective task + # is only readable after the call. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question", additional_args={"city": "Paris"}) + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + (part,) = parse_messages(agent_span, GenAI.GEN_AI_INPUT_MESSAGES)[0][ + "parts" + ] + assert part["content"].startswith("Test question") + assert "city" in part["content"] + assert "Paris" in part["content"] + + +def test_run_returning_full_result_records_the_final_answer( + instrument_with_content, span_exporter +) -> None: + # run(return_full_result=True) returns a RunResult wrapping the answer. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + result = agent.run("Test question", return_full_result=True) + assert result.output == "Test result from CodeAgent" + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [ + {"type": "text", "content": "Test result from CodeAgent"} + ] + + +IMAGE_FINAL_ANSWER = """ +Thought: Return the image. +Code: +```py +final_answer(make_image()) +``` +""" + + +class _ImageAnswerModel(FakeCodeModel): + def generate(self, messages: list[Any], **kwargs: Any) -> ChatMessage: + return ChatMessage( + role="assistant", + content=IMAGE_FINAL_ANSWER, + token_usage=TokenUsage(input_tokens=1, output_tokens=1), + ) + + +def test_image_final_answer_is_recorded_as_a_blob( + instrument_with_content, span_exporter, monkeypatch +) -> None: + # smolagents wraps an image answer in AgentImage, whose __str__ saves a PNG + # into a new temp directory and mutates the object the caller still holds. + # Telemetry must record the image itself and touch no disk. + def _no_temp_dirs(*args: Any, **kwargs: Any) -> str: + raise AssertionError("telemetry wrote an image to disk") + + monkeypatch.setattr(tempfile, "mkdtemp", _no_temp_dirs) + + agent = CodeAgent( + tools=[ImageTool()], model=_ImageAnswerModel(), max_steps=2 + ) + agent.run("Make an image") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + (part,) = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES)[0][ + "parts" + ] + assert part["type"] == "blob" + assert part["modality"] == "image" + assert part["mime_type"] == "image/png" + + +def test_code_agent_streaming_is_lazy_until_drained( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + stream = agent.run("Test question", stream=True) + + # Nothing has finished before the caller drains the stream. + assert ( + spans_by_operation(span_exporter.get_finished_spans(), "invoke_agent") + == [] + ) + + list(stream) + + spans = span_exporter.get_finished_spans() + (agent_span,) = spans_by_operation(spans, "invoke_agent") + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"][0]["content"] == "Test result from CodeAgent" + # The agent span is still open while the caller drains the stream, so the + # tool call the run makes on the way stays nested under it instead of + # becoming a root span. + (tool_span,) = spans_by_operation(spans, "execute_tool") + assert tool_span.parent is not None + assert tool_span.parent.span_id == agent_span.context.span_id + + +def test_streaming_run_stays_a_generator( + instrument_with_content, span_exporter +) -> None: + # Instrumentation observes, it doesn't change what run() returns. Callers + # branch on these checks, so a wrapper that fails them changes behaviour. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + stream = agent.run("Test question", stream=True) + + assert isinstance(stream, Generator) + assert inspect.isgenerator(stream) + assert stream.__class__ is GeneratorType + list(stream) + + +def test_streaming_run_records_no_chunk_metrics( + instrument_with_content, span_exporter, metric_reader +) -> None: + # gen_ai.client.operation.time_to_first_chunk and time_per_output_chunk + # describe the response stream of a generation call. A chunk of an agent + # run is a step object, so the run reports neither. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + list(agent.run("Test question", stream=True)) + + metrics = metrics_by_name(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metrics + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK + not in metrics + ) + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_PER_OUTPUT_CHUNK + not in metrics + ) + + +def test_streaming_run_with_stream_outputs_records_the_model_call( + instrument_with_content, span_exporter, metric_reader +) -> None: + # stream_outputs=True routes model calls through generate_stream. The chat + # span finishes when the agent drains the deltas, so it nests under the + # still-open agent span. + model = OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + model.client = stub_streaming_openai_client( + [text_chunk(CODE_FINAL_ANSWER), usage_chunk(11, 7)] + ) + agent = CodeAgent(tools=[], model=model, max_steps=3, stream_outputs=True) + + assert ( + list(agent.run("Test question", stream=True))[-1].output + == "Test result from CodeAgent" + ) + + spans = span_exporter.get_finished_spans() + (agent_span,) = spans_by_operation(spans, "invoke_agent") + (chat_span,) = spans_by_operation(spans, "chat") + assert chat_span.parent.span_id == agent_span.context.span_id + assert attr(chat_span, GenAI.GEN_AI_REQUEST_STREAM) is True + assert attr(chat_span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 11 + assert attr(chat_span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 7 + # The tokens belong to the model call, not to the run. + assert attr(agent_span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) is None + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE in metrics_by_name( + metric_reader + ) + + +def test_streaming_run_with_an_unwrapped_streaming_model_has_no_chat_span( + instrument_with_content, span_exporter +) -> None: + # A model class smolagents doesn't ship is not patched. See README.rst. + agent = CodeAgent( + tools=[], + model=FakeStreamingCodeModel(), + max_steps=3, + stream_outputs=True, + ) + list(agent.run("Test question", stream=True)) + + spans = span_exporter.get_finished_spans() + assert spans_by_operation(spans, "invoke_agent") + assert "chat" not in _operations(spans) + + +def test_agent_run_metrics( + instrument_with_content, span_exporter, metric_reader +) -> None: + # gen_ai.provider.name is required on gen_ai.client.operation.duration and + # AgentInvocation carries none by default, so the wrapper resolves it from + # the agent's model. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question") + + metrics = metrics_by_name(metric_reader) + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + assert { + GenAI.GEN_AI_OPERATION_NAME: "invoke_agent", + GenAI.GEN_AI_PROVIDER_NAME: "unknown", + GenAI.GEN_AI_REQUEST_MODEL: "fake-model", + } in data_point_attributes(duration) + # The run reports no tokens of its own. Only model calls feed the histogram, + # and a fake model makes none. + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE not in metrics + + +@pytest.mark.parametrize("stream", [False, True]) +def test_run_that_exhausts_max_steps_is_not_reported_as_a_plain_stop( + instrument_with_content, span_exporter, stream: bool +) -> None: + # smolagents doesn't raise when a run runs out of steps: it synthesizes an + # answer, appends an ActionStep carrying AgentMaxStepsError and returns. + # Reporting "stop" would make giving up indistinguishable from answering. + agent = CodeAgent(tools=[], model=NeverFinishingCodeModel(), max_steps=1) + if stream: + list(agent.run("Test question", stream=True)) + else: + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert attr(agent_span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ( + "length", + ) + # The library returned normally, so the run is not an error. + assert agent_span.status.status_code == StatusCode.UNSET + assert attr(agent_span, error_attributes.ERROR_TYPE) is None + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["finish_reason"] == "length" + + +def test_streaming_run_close_finalizes_once( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + stream = agent.run("Test question", stream=True) + + # Close before draining: no FinalAnswerStep observed, so no output recorded. + stream.close() + stream.close() # idempotent + + agent_spans = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert len(agent_spans) == 1 + assert attr(agent_spans[0], GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +@pytest.mark.parametrize("use_context_manager", [True, False]) +def test_streaming_run_stopped_midway_finalizes_successfully( + instrument_with_content, span_exporter, use_context_manager: bool +) -> None: + # smolagents' _run_stream yields from a finally block, so closing it after + # the first step makes CPython raise "generator ignored GeneratorExit". + # Abandoning a run is not a failed run and must not surface an error the + # caller never wrote. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + stream = agent.run("Test question", stream=True) + if use_context_manager: + with stream: + for _ in stream: + break + else: + for _ in stream: + break + stream.close() + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.UNSET + assert attr(agent_span, error_attributes.ERROR_TYPE) is None + # No FinalAnswerStep was observed, so the run reports no outcome. + assert attr(agent_span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def test_streaming_run_close_reraises_a_real_cleanup_error( + instrument_with_content, span_exporter, monkeypatch +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + def _failing_cleanup(*args: Any, **kwargs: Any): + try: + yield SimpleNamespace(token_usage=None) + finally: + raise RuntimeError("cleanup exploded") + + monkeypatch.setattr(agent, "_run_stream", _failing_cleanup) + stream = agent.run("Test question", stream=True) + for _ in stream: + break + with pytest.raises(RuntimeError, match="cleanup exploded"): + stream.close() + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.ERROR + assert attr(agent_span, error_attributes.ERROR_TYPE) == "RuntimeError" + + +def test_caller_error_inside_stream_context( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + with pytest.raises(RuntimeError, match="caller exploded"): + with agent.run("Test question", stream=True): + raise RuntimeError("caller exploded") + + agent_spans = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert len(agent_spans) == 1 + assert agent_spans[0].status.status_code == StatusCode.ERROR + assert attr(agent_spans[0], error_attributes.ERROR_TYPE) == "RuntimeError" + + +def test_run_failure_before_stream( + instrument_with_content, span_exporter, monkeypatch +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + def _boom(*args: Any, **kwargs: Any): + raise RuntimeError("boom") + + monkeypatch.setattr(agent, "_run_stream", _boom) + with pytest.raises(RuntimeError, match="boom"): + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.ERROR + assert attr(agent_span, error_attributes.ERROR_TYPE) == "RuntimeError" + + +def test_stream_failure_during_iteration( + instrument_with_content, span_exporter, monkeypatch +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + def _bad_stream(*args: Any, **kwargs: Any): + yield SimpleNamespace(token_usage=None) + raise ConnectionError("stream died") + + monkeypatch.setattr(agent, "_run_stream", _bad_stream) + stream = agent.run("Test question", stream=True) + with pytest.raises(ConnectionError, match="stream died"): + list(stream) + + agent_spans = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert len(agent_spans) == 1 + assert agent_spans[0].status.status_code == StatusCode.ERROR + assert ( + attr(agent_spans[0], error_attributes.ERROR_TYPE) == "ConnectionError" + ) + + +class _ManagerModel: + """Manager CodeAgent model: first delegate to the managed agent, then finish.""" + + model_id = "manager-model" + kwargs: dict[str, Any] = {} + + def generate(self, messages, **kwargs) -> ChatMessage: + if len(messages) < 4: + content = ( + "Thought: delegate.\nCode:\n```py\n" + 'search_agent("Who is the president?")\n```' + ) + else: + content = ( + "Thought: finish.\nCode:\n```py\n" + 'final_answer("Final report.")\n```' + ) + return ChatMessage( + role="assistant", + content=content, + token_usage=TokenUsage(input_tokens=5, output_tokens=3), + ) + + def __call__(self, *args, **kwargs) -> ChatMessage: + return self.generate(*args, **kwargs) + + +class _ManagedModel: + """Managed ToolCallingAgent model: immediately return final_answer.""" + + model_id = "managed-model" + kwargs: dict[str, Any] = {} + + def generate(self, messages, **kwargs) -> ChatMessage: + return ChatMessage( + role="assistant", + content="", + tool_calls=[ + ChatMessageToolCall( + id="call_0", + type="function", + function=ChatMessageToolCallFunction( + name="final_answer", + arguments="Report on the president", + ), + ) + ], + token_usage=TokenUsage(input_tokens=4, output_tokens=2), + ) + + def __call__(self, *args, **kwargs) -> ChatMessage: + return self.generate(*args, **kwargs) + + def parse_tool_calls(self, message: ChatMessage) -> ChatMessage: + return message + + +def test_managed_agent_nesting(instrument_with_content, span_exporter) -> None: + managed = ToolCallingAgent( + tools=[], + model=_ManagedModel(), + max_steps=3, + name="search_agent", + description="Runs searches.", + ) + manager = CodeAgent( + tools=[], + model=_ManagerModel(), + managed_agents=[managed], + max_steps=4, + ) + assert manager.run("Fake question.") == "Final report." + + agent_spans = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + by_name = {span.name: span for span in agent_spans} + assert "invoke_agent search_agent" in by_name + manager_span = next( + span for name, span in by_name.items() if "search_agent" not in name + ) + managed_span = by_name["invoke_agent search_agent"] + assert managed_span.parent.span_id == manager_span.context.span_id + assert ( + attr(managed_span, GenAI.GEN_AI_AGENT_DESCRIPTION) == "Runs searches." + ) + assert attr(manager_span, GenAI.GEN_AI_AGENT_DESCRIPTION) is None + # A managed agent is something the manager's model can call, so it belongs + # in the manager's tool definitions next to its own tools. + definitions = json.loads(attr(manager_span, GenAI.GEN_AI_TOOL_DEFINITIONS)) + assert [definition["name"] for definition in definitions] == [ + "final_answer", + "search_agent", + ] + search_agent = definitions[1] + assert search_agent["description"] == "Runs searches." + assert "task" in search_agent["parameters"]["properties"] + + +class _RecoveringModel: + """ToolCallingAgent model: call the broken tool once, then final_answer.""" + + model_id = "recovering-model" + kwargs: dict[str, Any] = {} + + def __init__(self) -> None: + self.calls = 0 + + def generate(self, messages, **kwargs) -> ChatMessage: + self.calls += 1 + if self.calls == 1: + name, arguments = "broken_tool", {"location": "Paris"} + else: + name, arguments = "final_answer", "recovered" + return ChatMessage( + role="assistant", + content="", + tool_calls=[ + ChatMessageToolCall( + id=f"call_{self.calls}", + type="function", + function=ChatMessageToolCallFunction( + name=name, arguments=arguments + ), + ) + ], + token_usage=TokenUsage(input_tokens=3, output_tokens=1), + ) + + def __call__(self, *args, **kwargs) -> ChatMessage: + return self.generate(*args, **kwargs) + + def parse_tool_calls(self, message: ChatMessage) -> ChatMessage: + return message + + +def test_expected_tool_error_is_recorded_and_agent_continues( + instrument_with_content, span_exporter +) -> None: + agent = ToolCallingAgent( + tools=[BrokenTool()], model=_RecoveringModel(), max_steps=4 + ) + assert agent.run("Do the thing") == "recovered" + + tool_spans = spans_by_operation( + span_exporter.get_finished_spans(), "execute_tool" + ) + statuses = { + attr(span, GenAI.GEN_AI_TOOL_NAME): span.status.status_code + for span in tool_spans + } + assert statuses["broken_tool"] == StatusCode.ERROR + assert statuses["final_answer"] == StatusCode.UNSET + broken = next( + span + for span in tool_spans + if attr(span, GenAI.GEN_AI_TOOL_NAME) == "broken_tool" + ) + assert attr(broken, error_attributes.ERROR_TYPE) == "ValueError" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py new file mode 100644 index 000000000..841ac723f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py @@ -0,0 +1,48 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-scenario conformance tests for smolagents.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +# Skip collection when weaver_live_check or OTLP exporters aren't installed +# (non-conformance envs). +pytest.importorskip("opentelemetry.test.weaver_live_check") +pytest.importorskip("opentelemetry.exporter.otlp.proto.grpc") + +from opentelemetry.test.weaver_live_check import WeaverLiveCheck # noqa: E402 +from opentelemetry.test_util_genai.conformance import ( # noqa: E402 + Scenario, + run_conformance, +) + +from .conformance.agent import AgentScenario # noqa: E402 +from .conformance.inference import ( # noqa: E402 + ChatScenario, + ToolCallingScenario, +) +from .conformance.multimodal import ( # noqa: E402 + MultimodalScenario, + ReasoningScenario, +) + + +@pytest.mark.parametrize( + "scenario", + [ + pytest.param(ChatScenario()), + pytest.param(ToolCallingScenario()), + pytest.param(AgentScenario()), + pytest.param(MultimodalScenario()), + pytest.param(ReasoningScenario()), + ], + ids=lambda s: type(s).__name__, +) +def test_conformance( + scenario: Scenario, vcr: Any, weaver_live_check: WeaverLiveCheck +) -> None: + run_conformance(scenario, vcr=vcr, weaver=weaver_live_check) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py index 1f4b69905..9f9fe9a06 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py @@ -1,19 +1,37 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Entry-point and instrument/uninstrument lifecycle tests. - -The instrumentor patches nothing yet, so these cover the parts of its contract -that hold before any patching lands: the entry point resolves, the declared -dependency is reported, and repeated instrument/uninstrument cycles stay quiet. -""" +"""Lifecycle, entry-point, completion-hook, and restoration tests.""" from __future__ import annotations +from typing import Any +from unittest.mock import patch + +import pytest +import smolagents +from smolagents import CodeAgent +from smolagents.tools import PipelineTool +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, + _model_classes_defining, + _tool_classes_defining_call, ) +from opentelemetry.test_util_genai.instrumentor import instrument from opentelemetry.util._importlib_metadata import entry_points +from opentelemetry.util.genai.completion_hook import CompletionHook + +from .test_utils import FakeCodeModel + + +class RecordingHook(CompletionHook): + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def on_completion(self, **kwargs: Any) -> None: + self.calls.append(kwargs) def test_entrypoint_loads_instrumentor() -> None: @@ -29,45 +47,63 @@ def test_instrumentation_dependencies() -> None: assert "smolagents >= 1.24.0" in dependencies -def test_instrument_uninstrument_cycle( +def test_instrument_uninstrument_restores_originals( tracer_provider, logger_provider, meter_provider ) -> None: + original_run = smolagents.MultiStepAgent.run + original_tool_call = smolagents.Tool.__call__ + original_pipeline_call = PipelineTool.__dict__["__call__"] + original_process_tool_calls = smolagents.ToolCallingAgent.__dict__[ + "process_tool_calls" + ] + original_generate = smolagents.OpenAIModel.generate + original_generate_stream = smolagents.OpenAIModel.generate_stream + original_timeout = smolagents.local_python_executor.timeout + instrumentor = SmolagentsInstrumentor() instrumentor.instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, ) - assert instrumentor.is_instrumented_by_opentelemetry - instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry + assert smolagents.MultiStepAgent.run is not original_run + assert PipelineTool.__dict__["__call__"] is not original_pipeline_call + assert ( + smolagents.ToolCallingAgent.__dict__["process_tool_calls"] + is not original_process_tool_calls + ) + assert smolagents.OpenAIModel.generate is not original_generate + assert ( + smolagents.OpenAIModel.generate_stream is not original_generate_stream + ) + assert smolagents.local_python_executor.timeout is not original_timeout + instrumentor.uninstrument() -def test_repeated_instrument_uninstrument( - tracer_provider, logger_provider, meter_provider -) -> None: - # BaseInstrumentor returns a per-class singleton, so the lifecycle has to - # survive being driven more than once. - instrumentor = SmolagentsInstrumentor() - for _ in range(2): - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - assert instrumentor.is_instrumented_by_opentelemetry - instrumentor.uninstrument() - assert not instrumentor.is_instrumented_by_opentelemetry + assert smolagents.MultiStepAgent.run is original_run + assert smolagents.Tool.__call__ is original_tool_call + assert PipelineTool.__dict__["__call__"] is original_pipeline_call + assert ( + smolagents.ToolCallingAgent.__dict__["process_tool_calls"] + is original_process_tool_calls + ) + assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.OpenAIModel.generate_stream is original_generate_stream + assert smolagents.local_python_executor.timeout is original_timeout def test_uninstrument_through_a_new_constructor_call( tracer_provider, logger_provider, meter_provider ) -> None: - # BaseInstrumentor.__new__ returns a per-class singleton but Python still - # runs __init__ on every construction, so the documented + # BaseInstrumentor is a per-class singleton, so the documented # SmolagentsInstrumentor().instrument() / SmolagentsInstrumentor() - # .uninstrument() form has to work on the live instance. + # .uninstrument() form must restore everything even though the second + # constructor call re-runs __init__ on the live instance. + original_run = smolagents.MultiStepAgent.run + original_generate = smolagents.OpenAIModel.generate + original_timeout = smolagents.local_python_executor.timeout + SmolagentsInstrumentor().instrument( tracer_provider=tracer_provider, logger_provider=logger_provider, @@ -75,23 +111,228 @@ def test_uninstrument_through_a_new_constructor_call( ) SmolagentsInstrumentor().uninstrument() - assert not SmolagentsInstrumentor().is_instrumented_by_opentelemetry + assert smolagents.MultiStepAgent.run is original_run + assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.local_python_executor.timeout is original_timeout + + +@pytest.mark.parametrize("method", ["generate", "generate_stream"]) +def test_model_classes_defining(method: str) -> None: + classes = _model_classes_defining(smolagents, method) + + # Each class object appears once, whatever names it is exported under. + assert len(classes) == len(set(classes)) + # Every entry owns the method, so no class is wrapped for one it only + # inherits. + for model_cls in classes: + assert method in model_cls.__dict__ + + # The API-backed model classes define both; the classes that inherit them + # are reached through the base they inherit from. + assert { + smolagents.OpenAIModel, + smolagents.LiteLLMModel, + smolagents.InferenceClientModel, + } <= set(classes) + assert smolagents.AzureOpenAIModel not in classes + assert smolagents.LiteLLMRouterModel not in classes + + +def test_only_generate_covers_the_base_class_and_bedrock() -> None: + # The base Model and AmazonBedrockModel define generate but no + # generate_stream, so streaming is patched on fewer classes. + generate = set(_model_classes_defining(smolagents, "generate")) + generate_stream = set( + _model_classes_defining(smolagents, "generate_stream") + ) + + assert {smolagents.Model, smolagents.AmazonBedrockModel} <= generate + assert generate_stream.isdisjoint( + {smolagents.Model, smolagents.AmazonBedrockModel} + ) + + +def test_tool_classes_defining_call() -> None: + classes = _tool_classes_defining_call(smolagents) + + assert len(classes) == len(set(classes)) + for tool_cls in classes: + assert "__call__" in tool_cls.__dict__ + # PipelineTool overrides Tool.__call__ and calls forward directly instead of + # delegating, so both classes need patching. smolagents doesn't export + # PipelineTool, so it is reached through the MRO of SpeechToTextTool. + assert set(classes) == {smolagents.Tool, PipelineTool} + # A tool that only inherits __call__ is covered by the class it inherits it + # from, and must not be wrapped again. + assert smolagents.FinalAnswerTool not in classes + assert smolagents.SpeechToTextTool not in classes + + +def test_repeated_instrument_uninstrument( + tracer_provider, logger_provider, meter_provider +) -> None: + # BaseInstrumentor returns a per-class singleton, so the wrapped-class + # bookkeeping has to survive being filled and drained more than once. + original_run = smolagents.MultiStepAgent.run + original_generate = smolagents.OpenAIModel.generate + + instrumentor = SmolagentsInstrumentor() + for _ in range(2): + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + assert smolagents.MultiStepAgent.run is not original_run + assert smolagents.OpenAIModel.generate is not original_generate + instrumentor.uninstrument() + assert smolagents.MultiStepAgent.run is original_run + assert smolagents.OpenAIModel.generate is original_generate def test_uninstrument_without_instrument() -> None: # BaseInstrumentor.uninstrument() short-circuits, but _uninstrument() must - # also be a no-op on its own: the rollback path in _instrument() calls it - # after a partial patch. + # also be a no-op on unpatched attributes: the rollback in _instrument() + # calls it after a partial patch. + original_run = smolagents.MultiStepAgent.run + original_tool_call = smolagents.Tool.__call__ + original_generate = smolagents.OpenAIModel.generate + original_timeout = smolagents.local_python_executor.timeout + SmolagentsInstrumentor().uninstrument() SmolagentsInstrumentor()._uninstrument() # noqa: SLF001 + assert smolagents.MultiStepAgent.run is original_run + assert smolagents.Tool.__call__ is original_tool_call + assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.local_python_executor.timeout is original_timeout + def test_instrument_with_no_providers() -> None: # Without providers the handler falls back to the globals; instrumenting # must not require a caller to pass them. + original_run = smolagents.MultiStepAgent.run + instrumentor = SmolagentsInstrumentor() instrumentor.instrument() try: - assert instrumentor.is_instrumented_by_opentelemetry + assert smolagents.MultiStepAgent.run is not original_run finally: instrumentor.uninstrument() + + assert smolagents.MultiStepAgent.run is original_run + + +def test_failed_instrument_rolls_back_partial_patches( + tracer_provider, logger_provider, meter_provider +) -> None: + # A failure part-way through must leave no class patched, because + # uninstrument() cannot clean up after a failed _instrument(). + original_run = smolagents.MultiStepAgent.run + original_tool_call = smolagents.Tool.__call__ + original_pipeline_call = PipelineTool.__dict__["__call__"] + original_process_tool_calls = smolagents.ToolCallingAgent.__dict__[ + "process_tool_calls" + ] + original_generate = smolagents.OpenAIModel.generate + original_generate_stream = smolagents.OpenAIModel.generate_stream + original_timeout = smolagents.local_python_executor.timeout + + real_wrap = wrap_function_wrapper + + def fail_on_timeout(module: Any, name: str, wrapper: Any) -> None: + if name == "timeout": + raise RuntimeError("boom") + real_wrap(module, name, wrapper) + + with patch( + "opentelemetry.instrumentation.genai.smolagents.wrap_function_wrapper", + fail_on_timeout, + ): + with pytest.raises(RuntimeError, match="boom"): + SmolagentsInstrumentor().instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + + assert smolagents.MultiStepAgent.run is original_run + assert smolagents.Tool.__call__ is original_tool_call + assert PipelineTool.__dict__["__call__"] is original_pipeline_call + assert ( + smolagents.ToolCallingAgent.__dict__["process_tool_calls"] + is original_process_tool_calls + ) + assert smolagents.OpenAIModel.generate is original_generate + assert smolagents.OpenAIModel.generate_stream is original_generate_stream + assert smolagents.local_python_executor.timeout is original_timeout + + +def test_inherited_generate_wrapped_only_on_defining_classes( + tracer_provider, logger_provider, meter_provider +) -> None: + # AzureOpenAIModel and LiteLLMRouterModel inherit generate; they must not be + # wrapped separately or they would emit duplicate chat spans. + assert "generate" not in smolagents.AzureOpenAIModel.__dict__ + assert "generate" not in smolagents.LiteLLMRouterModel.__dict__ + + instrumentor = SmolagentsInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + try: + # wrapt returns a fresh bound wrapper per attribute access, so the + # wrapper objects differ; the underlying wrapped function is shared, + # proving AzureOpenAIModel inherits the single wrapped generate. + assert ( + smolagents.AzureOpenAIModel.generate.__wrapped__ + is smolagents.OpenAIModel.generate.__wrapped__ + ) + finally: + instrumentor.uninstrument() + + +def test_explicit_completion_hook_takes_precedence( + tracer_provider, logger_provider, meter_provider, span_exporter +) -> None: + explicit_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook" + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + completion_hook=explicit_hook, + ): + agent = CodeAgent(tools=[], model=FakeCodeModel()) + agent.run("What is 2 + 2?") + + load_hook.assert_not_called() + assert explicit_hook.calls, "explicit completion hook was not invoked" + + +def test_env_completion_hook_used_when_no_explicit_hook( + tracer_provider, logger_provider, meter_provider +) -> None: + env_hook = RecordingHook() + with patch( + "opentelemetry.instrumentation.genai.smolagents.load_completion_hook", + return_value=env_hook, + ) as load_hook: + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + agent = CodeAgent(tools=[], model=FakeCodeModel()) + agent.run("What is 2 + 2?") + + load_hook.assert_called_once() + assert env_hook.calls, "env-resolved completion hook was not invoked" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py new file mode 100644 index 000000000..a7b92d242 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -0,0 +1,1369 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Model (``chat``) instrumentation tests: VCR-backed runs of the real +smolagents model classes, provider/endpoint resolution, request-parameter +mapping, and message conversion. +""" + +from __future__ import annotations + +import inspect +import json +import sys +from collections.abc import Generator +from types import ModuleType, SimpleNamespace +from typing import Any + +import pytest +from smolagents import LiteLLMModel, OpenAIModel +from smolagents.models import ( + ChatMessage, + ChatMessageToolCall, + ChatMessageToolCallFunction, + MessageRole, +) + +from opentelemetry.instrumentation.genai.smolagents._messages import ( + response_id, + response_model_name, + to_input_messages, + to_output_message, + to_tool_definitions, +) +from opentelemetry.instrumentation.genai.smolagents.provider import ( + resolve_provider, + resolve_server_address_port, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import ( + error_attributes, + server_attributes, +) +from opentelemetry.trace import StatusCode +from opentelemetry.util.genai.types import ( + Blob, + Reasoning, + Text, + ToolCallRequest, + Uri, +) + +from .test_utils import ( + GetWeatherTool, + attr, + data_point_attributes, + metrics_by_name, + parse_messages, + part_types, + spans_by_operation, + stub_streaming_openai_client, + text_chunk, + tool_call_chunk, + usage_chunk, +) + +IMAGE_URL = ( + "https://fastly.picsum.photos/id/237/200/300.jpg" + "?hmac=TmmQSbShHz9CdQm0NkEjx1Dyh_Y984R9LpNrpvH2D_U" +) + + +def _openai_model() -> OpenAIModel: + return OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + + +def _stub_openai_client( + content: str, finish_reason: str = "stop", error: Exception | None = None +) -> Any: + """An object shaped like the bits of ``openai.OpenAI`` that ``generate`` uses. + + Building a real ``ChatCompletion`` keeps the response shape honest (the + wrapper reads ``raw.model``, ``raw.id``, and ``raw.choices[0].finish_reason``) + without needing a cassette for a deployment we can't record against. + """ + from openai.types.chat import ( # noqa: PLC0415 + ChatCompletion, + ChatCompletionMessage, + ) + from openai.types.chat.chat_completion import Choice # noqa: PLC0415 + from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 + + completion = ChatCompletion( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion", + created=0, + choices=[ + Choice( + index=0, + finish_reason=finish_reason, + message=ChatCompletionMessage( + role="assistant", content=content + ), + ) + ], + usage=CompletionUsage( + prompt_tokens=3, completion_tokens=1, total_tokens=4 + ), + ) + + def create(**_: Any) -> ChatCompletion: + if error is not None: + raise error + return completion + + return SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + +def test_openai_model_basic( + instrument_with_content, span_exporter, metric_reader, vcr +) -> None: + model = _openai_model() + text = "Who won the World Cup in 2018? Answer in one word with no punctuation." + with vcr.use_cassette("openai_model_basic.yaml"): + output = model.generate( + messages=[ + {"role": "user", "content": [{"type": "text", "text": text}]} + ] + ) + assert output.content == "France" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.name == "chat gpt-4o" + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 25 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + assert ( + attr(span, GenAI.GEN_AI_RESPONSE_ID) + == "chatcmpl-Ax6UoZOGLTVmdQxp0ToJi1tv1FUkb" + ) + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) == "gpt-4o-2024-08-06" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert attr(span, server_attributes.SERVER_ADDRESS) == "api.openai.com" + assert attr(span, server_attributes.SERVER_PORT) is None + + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0]["role"] == "user" + assert inputs[0]["parts"][0] == {"type": "text", "content": text} + + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["role"] == "assistant" + assert outputs[0]["parts"][0] == {"type": "text", "content": "France"} + + metrics = metrics_by_name(metric_reader) + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + assert duration.unit == "s" + assert data_point_attributes(duration) == [ + { + GenAI.GEN_AI_OPERATION_NAME: "chat", + GenAI.GEN_AI_PROVIDER_NAME: "openai", + GenAI.GEN_AI_REQUEST_MODEL: "gpt-4o", + GenAI.GEN_AI_RESPONSE_MODEL: "gpt-4o-2024-08-06", + server_attributes.SERVER_ADDRESS: "api.openai.com", + } + ] + token_usage = metrics[gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE] + assert { + point.attributes[GenAI.GEN_AI_TOKEN_TYPE]: point.sum + for point in token_usage.data.data_points + } == {"input": 25, "output": 2} + + +def test_openai_model_no_content( + instrument_no_content, span_exporter, vcr +) -> None: + model = _openai_model() + with vcr.use_cassette("openai_model_basic.yaml"): + model.generate( + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Hi"}]} + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert isinstance(attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS), int) + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_openai_model_image_url( + instrument_with_content, span_exporter, vcr +) -> None: + model = _openai_model() + with vcr.use_cassette("openai_model_image_url.yaml"): + model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + {"type": "text", "text": "What breed is this dog?"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + ], + ) + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + parts = inputs[0]["parts"] + assert parts[0] == {"type": "text", "content": "What breed is this dog?"} + assert parts[1]["type"] == "uri" + assert parts[1]["modality"] == "image" + assert parts[1]["uri"] == IMAGE_URL + + +def test_openai_model_with_tools( + instrument_with_content, span_exporter, vcr +) -> None: + model = _openai_model() + with vcr.use_cassette("openai_model_tool.yaml"): + output = model.generate( + messages=[ + ChatMessage( + role=MessageRole.USER, + content=[ + { + "type": "text", + "text": "What is the weather in Paris?", + } + ], + ) + ], + tools_to_call_from=[GetWeatherTool()], + ) + assert output.tool_calls[0].function.name == "get_weather" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_DEFINITIONS)) == [ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather for a given city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + }, + } + ] + + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + tool_call_parts = [ + part for part in outputs[0]["parts"] if part["type"] == "tool_call" + ] + assert tool_call_parts[0]["name"] == "get_weather" + # smolagents hands the provider's raw argument payload through unparsed. + assert tool_call_parts[0]["arguments"] == '{"location":"Paris"}' + assert tool_call_parts[0]["id"] == "call_EUuviydGIG5Jau3DLw4v4cue" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) + + +def _litellm_supports_reasoning() -> bool: + """litellm surfaces Anthropic ``reasoning_content`` only from ~1.63 onward. + + The oldest supported litellm (smolagents' 1.60.2 floor) doesn't parse the + Anthropic thinking blocks into ``reasoning_content``, so the reasoning part + can't be mapped there. Gate the reasoning-specific assertion on the version. + """ + from importlib.metadata import version # noqa: PLC0415 + + parts = version("litellm").split(".") + try: + return (int(parts[0]), int(parts[1])) >= (1, 63) + except (IndexError, ValueError): + return True + + +def test_litellm_reasoning( + instrument_with_content, + span_exporter, + litellm_local_cost_map, + patch_tiktoken_encoding, + vcr, +) -> None: + model = LiteLLMModel( + model_id="anthropic/claude-3-7-sonnet-20250219", + api_key="test_anthropic_api_key", + thinking={"type": "enabled", "budget_tokens": 4000}, + ) + with vcr.use_cassette("litellm_reasoning.yaml"): + model.generate( + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Who won the World Cup in 2018? Answer in one " + "word with no punctuation." + ), + } + ], + } + ] + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "anthropic" + assert ( + attr(span, GenAI.GEN_AI_REQUEST_MODEL) + == "anthropic/claude-3-7-sonnet-20250219" + ) + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + if _litellm_supports_reasoning(): + assert "reasoning" in part_types(outputs) + + +def test_model_generate_reraises_and_records_error( + instrument_with_content, span_exporter +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model(model_id="broken-model") + with pytest.raises(NotImplementedError): + model.generate(messages=[{"role": "user", "content": "hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "NotImplementedError" + + +def test_inherited_azure_generate_emits_one_chat_span( + instrument_with_content, span_exporter +) -> None: + # AzureOpenAIModel inherits generate from OpenAIModel, so only the defining + # class is patched. Exercise the inherited method end to end to prove the + # single patch still produces exactly one span with the Azure provider. + from smolagents import AzureOpenAIModel # noqa: PLC0415 + + model = AzureOpenAIModel( + model_id="gpt-4o-deployment", + azure_endpoint="https://example-resource.openai.azure.com", + api_key="test_azure_api_key", + api_version="2024-10-21", + ) + model.client = _stub_openai_client("Bonjour") + + output = model.generate(messages=[{"role": "user", "content": "Hi"}]) + assert output.content == "Bonjour" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "azure.ai.openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o-deployment" + assert ( + attr(span, server_attributes.SERVER_ADDRESS) + == "example-resource.openai.azure.com" + ) + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 1 + + +def _fake_model(class_name: str, **attrs: Any) -> Any: + instance = type(class_name, (), {})() + for key, value in attrs.items(): + setattr(instance, key, value) + return instance + + +@pytest.mark.parametrize( + "class_name, attrs, expected", + [ + ("OpenAIModel", {}, "openai"), + ("AzureOpenAIModel", {}, "azure.ai.openai"), + ("AmazonBedrockModel", {}, "aws.bedrock"), + # No GenAI registry value exists for these runtimes, so the product + # name is used rather than the class name. + ("InferenceClientModel", {}, "huggingface"), + ("TransformersModel", {}, "huggingface"), + ("VLLMModel", {}, "vllm"), + ("MLXModel", {}, "mlx"), + # LiteLLM vendor prefixes: remapped where the slug differs from the + # semconv value, passed through otherwise. + ("LiteLLMModel", {"model_id": "anthropic/claude-3"}, "anthropic"), + ("LiteLLMModel", {"model_id": "mistral/large"}, "mistral_ai"), + ("LiteLLMModel", {"model_id": "xai/grok"}, "x_ai"), + ("LiteLLMModel", {"model_id": "gemini/gemini-2.0"}, "gcp.gemini"), + ( + "LiteLLMModel", + {"model_id": "vertex_ai/gemini-2.0"}, + "gcp.vertex_ai", + ), + ("LiteLLMModel", {"model_id": "watsonx/granite"}, "ibm.watsonx.ai"), + ( + "LiteLLMModel", + {"model_id": "azure_ai/phi-4"}, + "azure.ai.inference", + ), + ("LiteLLMModel", {"model_id": "ollama/llama3"}, "ollama"), + # LiteLLMRouterModel takes a model-group name, not a provider/model + # slug, so there is nothing to resolve. + ("LiteLLMRouterModel", {"model_id": "model-group-1"}, "unknown"), + # gen_ai.provider.name is also a metric attribute. An unmapped model + # must not fall back to the deployment-specific host or a class name. + ( + "CustomModel", + {"api_base": "https://llm.example.com/v1"}, + "unknown", + ), + ("CustomModel", {}, "unknown"), + ], +) +def test_resolve_provider( + class_name: str, attrs: dict[str, Any], expected: str +) -> None: + assert resolve_provider(_fake_model(class_name, **attrs)) == expected + + +@pytest.mark.parametrize( + "model_class, expected", + [ + ("OpenAIModel", "openai"), + ("AzureOpenAIModel", "azure.ai.openai"), + ("AmazonBedrockModel", "aws.bedrock"), + ("InferenceClientModel", "huggingface"), + ("TransformersModel", "huggingface"), + ("VLLMModel", "vllm"), + ("MLXModel", "mlx"), + ("LiteLLMModel", "unknown"), + ("LiteLLMRouterModel", "unknown"), + ], +) +def test_resolve_provider_covers_every_real_model_class( + model_class: str, expected: str +) -> None: + # The mapping is keyed by class name, so pin it against the real classes + # rather than only against synthetic stand-ins. + import smolagents # noqa: PLC0415 + + instance = object.__new__(getattr(smolagents, model_class)) + assert resolve_provider(instance) == expected + + +@pytest.mark.parametrize( + "attrs, expected", + [ + ({"api_base": "https://api.openai.com/v1"}, ("api.openai.com", None)), + # The default HTTPS port is omitted per the semconv server.port guidance. + ( + {"api_base": "https://api.openai.com:443/v1"}, + ("api.openai.com", None), + ), + ({"api_base": "http://localhost:11434/v1"}, ("localhost", 11434)), + ( + { + "client_kwargs": { + "azure_endpoint": "https://x.openai.azure.com" + } + }, + ("x.openai.azure.com", None), + ), + ({}, (None, None)), + ], +) +def test_resolve_server_address_port( + attrs: dict[str, Any], expected: tuple[str | None, int | None] +) -> None: + assert resolve_server_address_port(_fake_model("M", **attrs)) == expected + + +def test_server_address_falls_back_to_the_sdk_client() -> None: + # The common configuration: no api_base, so the URL is only known to the + # client the model built for itself. + model = OpenAIModel(model_id="gpt-4o", api_key="test_openai_api_key") + assert resolve_server_address_port(model) == ("api.openai.com", None) + + +def test_request_parameters_recorded( + instrument_with_content, span_exporter +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model( + model_id="broken-model", + temperature=0.5, + top_p=0.9, + top_k=40, + frequency_penalty=0.25, + presence_penalty=1, + max_tokens=256, + seed=7, + ) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + stop_sequences=[""], + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.5 + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.9 + # top_k is a float attribute in the spec even though callers pass an int. + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_K) == 40.0 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_TOP_K), float) + assert attr(span, GenAI.GEN_AI_REQUEST_FREQUENCY_PENALTY) == 0.25 + assert attr(span, GenAI.GEN_AI_REQUEST_PRESENCE_PENALTY) == 1.0 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == 256 + assert isinstance(attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS), int) + assert attr(span, GenAI.GEN_AI_REQUEST_SEED) == 7 + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == ("",) + + +def test_model_kwargs_win_over_call_kwargs( + instrument_with_content, span_exporter +) -> None: + # _prepare_completion_kwargs applies the call kwargs first and the model + # kwargs on top, and drops any key whose model-level value is the + # REMOVE_PARAMETER sentinel. + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model = Model( + model_id="broken-model", + temperature=0.1, + max_tokens=REMOVE_PARAMETER, + ) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + temperature=0.9, + max_tokens=512, + top_p=0.5, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_TEMPERATURE) == 0.1 + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) is None + assert attr(span, GenAI.GEN_AI_REQUEST_TOP_P) == 0.5 + + +@pytest.mark.parametrize( + "model_id, model_kwargs, expected", + [ + # gpt-5 doesn't accept `stop`, so smolagents truncates the generated + # text locally instead of sending the sequences. + ("gpt-5", {}, None), + ("gpt-4o", {}, ("",)), + # An explicit `stop` overrides the stop_sequences argument. + ("gpt-4o", {"stop": ["STOP"]}, ("STOP",)), + # The model-level sentinel pops the `stop` that + # _prepare_completion_kwargs seeded from stop_sequences, leaving the + # request with none. + ("gpt-4o", {"stop": "REMOVE"}, None), + ], +) +def test_stop_sequences_follow_what_is_sent( + instrument_with_content, + span_exporter, + model_id: str, + model_kwargs: dict[str, Any], + expected: tuple[str, ...] | None, +) -> None: + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + model = Model(model_id=model_id, **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + stop_sequences=[""], + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) == expected + + +def _bedrock_model(response: dict[str, Any]) -> Any: + from smolagents import AmazonBedrockModel # noqa: PLC0415 + + # A caller-supplied client keeps boto3 out of the test. + return AmazonBedrockModel( + model_id="us.amazon.nova-pro-v1:0", + client=SimpleNamespace(converse=lambda **_: response), + ) + + +BEDROCK_RESPONSE: dict[str, Any] = { + "output": { + "message": { + "role": "assistant", + "content": [{"text": "done"}], + "tool_calls": None, + } + }, + "usage": {"inputTokens": 3, "outputTokens": 2}, + "stopReason": "end_turn", +} + + +def test_bedrock_stop_sequences_are_not_recorded( + instrument_with_content, span_exporter +) -> None: + # supports_stop_parameter says yes, but the prepared request carries no + # stop sequences, so the span must not claim any either. + model = _bedrock_model(BEDROCK_RESPONSE) + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + request = model._prepare_completion_kwargs( # noqa: SLF001 + messages=messages, stop_sequences=[""] + ) + assert model.supports_stop_parameter is True + assert "stop" not in request + + model.generate(messages=messages, stop_sequences=[""]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_STOP_SEQUENCES) is None + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "aws.bedrock" + # Bedrock's "end_turn" is normalized to the semconv "stop". + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + + +@pytest.mark.parametrize( + "model_kwargs, call_kwargs, expected", + [ + ({"max_tokens": 256}, {}, 256), + ({}, {"max_tokens": 256}, 256), + # max_new_tokens is the TransformersModel spelling of the same limit, + # and max_tokens wins when a caller sets both. + ({"max_new_tokens": 4096}, {}, 4096), + ({"max_new_tokens": 4096, "max_tokens": 256}, {}, 256), + ({}, {}, None), + ], +) +def test_max_tokens_covers_both_spellings( + instrument_with_content, + span_exporter, + model_kwargs: dict[str, Any], + call_kwargs: dict[str, Any], + expected: int | None, +) -> None: + from smolagents.models import Model # noqa: PLC0415 + + model = Model(model_id="broken-model", **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], **call_kwargs + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_REQUEST_MAX_TOKENS) == expected + + +@pytest.mark.parametrize( + "response_format, model_kwargs, expected", + [ + ({"type": "json_object"}, {}, "json"), + ({"type": "json_schema", "json_schema": {}}, {}, "json"), + ({"type": "text"}, {}, "text"), + (None, {}, None), + # smolagents forwards response_format unchanged, so its type is + # whatever the provider accepts. An unknown one is dropped rather than + # recorded on an enum attribute. + ({"type": "xml"}, {}, None), + ({}, {}, None), + # The model-level kwargs win over the argument, and the sentinel drops + # the key from the request, the same as for every other parameter. + ( + {"type": "json_object"}, + {"response_format": {"type": "text"}}, + "text", + ), + ({"type": "json_object"}, {"response_format": "REMOVE"}, None), + (None, {"response_format": {"type": "json_object"}}, "json"), + ], +) +def test_output_type_follows_the_response_format( + instrument_with_content, + span_exporter, + response_format: dict[str, Any] | None, + model_kwargs: dict[str, Any], + expected: str | None, +) -> None: + from smolagents.models import ( # noqa: PLC0415 + REMOVE_PARAMETER, + Model, + ) + + model_kwargs = { + key: REMOVE_PARAMETER if value == "REMOVE" else value + for key, value in model_kwargs.items() + } + model = Model(model_id="broken-model", **model_kwargs) + with pytest.raises(NotImplementedError): + model.generate( + messages=[{"role": "user", "content": "hi"}], + response_format=response_format, + ) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_OUTPUT_TYPE) == expected + + +def test_user_subclass_of_a_patched_model_keeps_its_provider( + instrument_with_content, span_exporter, metric_reader +) -> None: + # A subclass inherits the patched generate, so it is instrumented; resolving + # the provider by exact class name would report "unknown" on both the span + # and the metrics. + class TenantOpenAIModel(OpenAIModel): + pass + + model = TenantOpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="http://localhost:11434/v1", + ) + model.client = _stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + # A non-default port is part of the endpoint, unlike the HTTPS default. + assert attr(span, server_attributes.SERVER_ADDRESS) == "localhost" + assert attr(span, server_attributes.SERVER_PORT) == 11434 + duration = metrics_by_name(metric_reader)[ + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION + ] + assert data_point_attributes(duration)[0][GenAI.GEN_AI_PROVIDER_NAME] == ( + "openai" + ) + + +def test_provider_error_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = _openai_model() + model.client = _stub_openai_client( + "", error=ConnectionError("connection reset") + ) + + with pytest.raises(ConnectionError, match="connection reset"): + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def _drain_stream(model: OpenAIModel, **kwargs: Any) -> list[Any]: + return list( + model.generate_stream( + messages=[{"role": "user", "content": "Hi"}], **kwargs + ) + ) + + +def test_generate_stream_is_lazy_and_records_the_drained_response( + instrument_with_content, span_exporter +) -> None: + model = _openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] + ) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + # A streamed response isn't finished until the caller drains it. + assert spans_by_operation(span_exporter.get_finished_spans(), "chat") == [] + + deltas = list(stream) + assert "".join(delta.content or "" for delta in deltas) == "Bonjour" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "openai" + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == "gpt-4o" + assert attr(span, GenAI.GEN_AI_REQUEST_STREAM) is True + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "Bonjour"}] + # Deltas carry no finish reason, and "stop" would hide a truncation. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def test_generate_stream_stays_a_generator( + instrument_with_content, span_exporter +) -> None: + # Instrumentation observes; it must not change what generate_stream returns. + model = _openai_model() + model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + assert isinstance(stream, Generator) + assert inspect.isgenerator(stream) + list(stream) + + +def test_generate_stream_accumulates_tool_calls( + instrument_with_content, span_exporter +) -> None: + model = _openai_model() + model.client = stub_streaming_openai_client( + [ + tool_call_chunk(0, call_id="call_1", name="get_weather"), + tool_call_chunk(0, arguments='{"location":'), + tool_call_chunk(0, arguments='"Paris"}'), + ] + ) + + _drain_stream(model, tools_to_call_from=[GetWeatherTool()]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + (part,) = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES)[0]["parts"] + assert part == { + "type": "tool_call", + "id": "call_1", + "name": "get_weather", + "arguments": '{"location":"Paris"}', + } + # Tool calls are the only evidence of why a streamed generation stopped. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("tool_calls",) + + +def test_generate_stream_error_mid_iteration_is_recorded_and_reraised( + instrument_with_content, span_exporter +) -> None: + model = _openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon")], error=ConnectionError("stream died") + ) + + with pytest.raises(ConnectionError, match="stream died"): + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ConnectionError" + # What was streamed before the failure is still recorded. + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [{"type": "text", "content": "Bon"}] + + +def test_caller_error_inside_generate_stream_context( + instrument_with_content, span_exporter +) -> None: + # The caller fails before draining the stream, so nothing was streamed and + # the error is the caller's, not the provider's. + model = _openai_model() + model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + + with pytest.raises(RuntimeError, match="caller exploded"): + with model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ): + raise RuntimeError("caller exploded") + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "RuntimeError" + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_generate_stream_close_before_drain_finalizes_once( + instrument_with_content, span_exporter +) -> None: + model = _openai_model() + model.client = stub_streaming_openai_client([text_chunk("Bonjour")]) + + stream = model.generate_stream( + messages=[{"role": "user", "content": "Hi"}] + ) + stream.close() + stream.close() # idempotent + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert span.status.status_code == StatusCode.UNSET + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_generate_stream_records_chunk_metrics( + instrument_with_content, span_exporter, metric_reader +) -> None: + # Unlike an agent run, a generation call does report per-chunk timing: its + # chunks are pieces of a model response. + model = _openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bon"), text_chunk("jour"), usage_chunk(3, 2)] + ) + + _drain_stream(model) + + metrics = metrics_by_name(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metrics + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE in metrics + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK in metrics + ) + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_PER_OUTPUT_CHUNK in metrics + ) + + +def test_generate_stream_no_content( + instrument_no_content, span_exporter +) -> None: + model = _openai_model() + model.client = stub_streaming_openai_client( + [text_chunk("Bonjour"), usage_chunk(3, 2)] + ) + + _drain_stream(model) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == 2 + + +def test_event_only_content_capture( + instrument_event_only, span_exporter, log_exporter +) -> None: + model = _openai_model() + model.client = _stub_openai_client("Bonjour") + model.generate(messages=[{"role": "user", "content": "Hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + # Metadata still goes on the span. + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == 3 + + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.event_name == "gen_ai.client.inference.operation.details" + # Event attributes carry the messages as structured values, not JSON text. + attributes = record.attributes or {} + inputs = attributes[GenAI.GEN_AI_INPUT_MESSAGES] + outputs = attributes[GenAI.GEN_AI_OUTPUT_MESSAGES] + assert inputs[0]["parts"][0]["content"] == "Hi" + assert outputs[0]["parts"][0]["content"] == "Bonjour" + assert outputs[0]["finish_reason"] == "stop" + + +def test_to_tool_definitions_uses_json_schema() -> None: + (definition,) = to_tool_definitions([GetWeatherTool()]) + assert definition.name == "get_weather" + assert definition.description == "Get the weather for a given city" + # smolagents' raw ``inputs`` map is not a JSON Schema on its own. + assert definition.parameters == { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["location"], + } + + +def test_to_input_messages_maps_smolagents_only_roles() -> None: + # smolagents converts these roles itself inside generate(), after the + # wrapper has already read the messages. + messages = to_input_messages( + [ + {"role": MessageRole.TOOL_CALL, "content": "call"}, + {"role": MessageRole.TOOL_RESPONSE, "content": "response"}, + {"role": MessageRole.SYSTEM, "content": "sys"}, + ] + ) + assert [message.role for message in messages] == [ + "assistant", + "user", + "system", + ] + + +def test_to_input_messages_dict_and_chatmessage() -> None: + messages = to_input_messages( + [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + ChatMessage(role=MessageRole.ASSISTANT, content="Hi there"), + ] + ) + assert messages[0].role == "user" + assert messages[0].parts == [Text(content="Hello")] + assert messages[1].role == "assistant" + assert messages[1].parts == [Text(content="Hi there")] + + +def test_to_input_messages_image_and_base64() -> None: + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image", "image": "aVZCT1J3MEtHZ28="}, + ], + } + ] + ) + parts = messages[0].parts + assert parts[0] == Uri(mime_type=None, modality="image", uri=IMAGE_URL) + assert isinstance(parts[1], Blob) + assert parts[1].modality == "image" + assert parts[1].mime_type == "image/png" + assert isinstance(parts[1].content, bytes) + + +def test_to_input_messages_data_url_keeps_media_type() -> None: + messages = to_input_messages( + [ + { + "role": "user", + "content": [ + { + "type": "image", + "image": "data:image/jpeg;base64,aVZCT1J3MEtHZ28=", + } + ], + } + ] + ) + (part,) = messages[0].parts + assert isinstance(part, Blob) + assert part.mime_type == "image/jpeg" + + +@pytest.mark.parametrize( + "image", + [ + "not base64!!!!", + # A path is not base64, but a non-validating decode silently turns this + # one into 15 bytes of garbage after dropping the invalid characters. + "/tmp/photos/my-cat.png", + ], +) +def test_to_input_messages_drops_malformed_base64_image(image: str) -> None: + messages = to_input_messages( + [{"role": "user", "content": [{"type": "image", "image": image}]}] + ) + assert messages[0].parts == [] + + +def test_to_output_message_text_reasoning_and_tool_calls() -> None: + class _Msg: + role = MessageRole.ASSISTANT + content = "The answer" + tool_calls = [ + ChatMessageToolCall( + id="call_1", + type="function", + function=ChatMessageToolCallFunction( + name="get_weather", arguments='{"location": "Paris"}' + ), + ) + ] + + class raw: # noqa: N801 + class _Choice: + class message: # noqa: N801 + reasoning_content = "thinking about it" + + choices = [_Choice()] + + output = to_output_message(_Msg()) + assert output.role == "assistant" + assert Text(content="The answer") in output.parts + assert Reasoning(content="thinking about it") in output.parts + tool_calls = [p for p in output.parts if isinstance(p, ToolCallRequest)] + assert tool_calls[0].name == "get_weather" + assert tool_calls[0].id == "call_1" + assert output.finish_reason == "tool_calls" + + +def test_to_output_message_from_a_dict_raw_response() -> None: + # AmazonBedrockModel, TransformersModel, VLLMModel and MLXModel put a plain + # dict on ChatMessage.raw rather than an SDK object. + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="done", + raw={"stopReason": "max_tokens"}, + ) + output = to_output_message(message) + assert output.parts == [Text(content="done")] + # "max_tokens" is Bedrock's stopReason spelling for a length cutoff. + assert output.finish_reason == "length" + assert response_id(message) is None + assert response_model_name(message) is None + + +def test_to_output_message_unwraps_the_role_enum() -> None: + output = to_output_message( + ChatMessage(role=MessageRole.ASSISTANT, content="done") + ) + assert output.role == "assistant" + + +def test_to_output_message_maps_image_content() -> None: + # A response whose content is a list carries the same element shapes as a + # request, so an image in it maps the same way. + output = to_output_message( + ChatMessage( + role=MessageRole.ASSISTANT, + content=[ + {"type": "text", "text": "Here it is"}, + {"type": "image_url", "image_url": {"url": IMAGE_URL}}, + {"type": "image", "image": "aVZCT1J3MEtHZ28="}, + ], + ) + ) + assert output.parts[0] == Text(content="Here it is") + assert output.parts[1] == Uri( + mime_type=None, modality="image", uri=IMAGE_URL + ) + blob = output.parts[2] + assert isinstance(blob, Blob) + assert blob.mime_type == "image/png" + assert blob.modality == "image" + + +LOCAL_RUNTIME_RAW = { + "out": "done", + "completion_kwargs": {"max_new_tokens": 4096}, +} + + +@pytest.mark.parametrize( + "raw, tool_calls, expected", + [ + # API-backed models: the provider's own value, whatever it is. + ( + SimpleNamespace(choices=[SimpleNamespace(finish_reason="stop")]), + [], + "stop", + ), + ( + SimpleNamespace(choices=[SimpleNamespace(finish_reason="length")]), + [], + "length", + ), + # Bedrock's stopReason values map onto the semconv vocabulary. + ({"stopReason": "max_tokens"}, [], "length"), + ({"stopReason": "tool_use"}, [], "tool_calls"), + # An unmapped value passes through rather than being guessed at. + ({"stopReason": "guardrail_intervened"}, [], "guardrail_intervened"), + # The local runtimes report no reason, so none is recorded and + # util-genai omits the empty value. + (LOCAL_RUNTIME_RAW, [], ""), + (None, [], ""), + # Tool calls in the response give the reason without guessing. + ( + LOCAL_RUNTIME_RAW, + [ + ChatMessageToolCall( + id="call_1", + type="function", + function=ChatMessageToolCallFunction( + name="get_weather", arguments="{}" + ), + ) + ], + "tool_calls", + ), + ], +) +def test_finish_reason_follows_the_provider_response( + raw: Any, tool_calls: list[ChatMessageToolCall], expected: str +) -> None: + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="done", + tool_calls=tool_calls or None, + raw=raw, + ) + assert to_output_message(message).finish_reason == expected + + +def test_model_reporting_no_finish_reason_omits_the_attribute( + instrument_with_content, span_exporter +) -> None: + # InferenceClientModel is a patched class whose provider response can come + # back without a finish reason; the span must then carry none. + from smolagents import InferenceClientModel # noqa: PLC0415 + + model = InferenceClientModel( + model_id="Qwen/Qwen2.5-Coder-32B-Instruct", token="hf_test" + ) + model.client = SimpleNamespace( + chat_completion=lambda **_: SimpleNamespace( + id="hf-1", + model="Qwen/Qwen2.5-Coder-32B-Instruct", + choices=[ + SimpleNamespace( + message=SimpleNamespace( + role="assistant", content="ok", tool_calls=None + ) + ) + ], + usage=SimpleNamespace(prompt_tokens=5, completion_tokens=2), + ) + ) + + model.generate(messages=[{"role": "user", "content": "hi"}]) + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == "huggingface" + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["finish_reason"] == "" + + +# The local runtimes flatten a message's content as text, so the content has to +# be a list of parts rather than a bare string. +LOCAL_RUNTIME_MESSAGES: list[dict[str, Any]] = [ + { + "role": "user", + "content": [{"type": "text", "text": "Where is the Louvre?"}], + } +] + + +def _mlx_model(monkeypatch: pytest.MonkeyPatch) -> Any: + """An ``MLXModel`` whose ``mlx_lm`` pieces are stubbed. + + ``MLXModel.generate`` imports nothing itself; it drives ``stream_generate`` + over ``self.model`` and ``self.tokenizer``, which ``__init__`` loads from + ``mlx_lm``. Bypassing ``__init__`` is therefore enough to run the real + ``generate``, and the runtime doesn't have to be installed. ``monkeypatch`` + is unused here; it keeps the factory signature uniform with the vllm one, + which does have modules to stub. + """ + from smolagents.models import MLXModel # noqa: PLC0415 + + model = object.__new__(MLXModel) + model.model_id = "mlx-community/Qwen2.5-0.5B-Instruct-4bit" + model.kwargs = {} + model.flatten_messages_as_text = True + model.apply_chat_template_kwargs = {} + model.model = object() + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, tools=None, **_: [1, 2, 3] + ) + model.stream_generate = lambda *_, **__: iter( + [SimpleNamespace(text="In "), SimpleNamespace(text="Paris")] + ) + return model + + +def _vllm_model(monkeypatch: pytest.MonkeyPatch) -> Any: + """A ``VLLMModel`` with ``vllm`` itself stubbed. + + ``VLLMModel.generate`` imports ``SamplingParams`` and + ``StructuredOutputsParams`` from ``vllm`` when it runs, and neither test env + installs vllm, so both modules are faked for the duration of the test. + Everything the wrapper reads still comes from the real ``generate``. + """ + from smolagents.models import VLLMModel # noqa: PLC0415 + + def fake_params(**kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(**kwargs) + + vllm = ModuleType("vllm") + sampling_params = ModuleType("vllm.sampling_params") + setattr(vllm, "SamplingParams", fake_params) + setattr(sampling_params, "StructuredOutputsParams", fake_params) + setattr(vllm, "sampling_params", sampling_params) + monkeypatch.setitem(sys.modules, "vllm", vllm) + monkeypatch.setitem(sys.modules, "vllm.sampling_params", sampling_params) + + completion = SimpleNamespace( + prompt_token_ids=[1, 2, 3, 4], + outputs=[SimpleNamespace(text="In Paris", token_ids=[5, 6])], + ) + model = object.__new__(VLLMModel) + model.model_id = "Qwen/Qwen2.5-0.5B-Instruct" + model.kwargs = {} + model.flatten_messages_as_text = True + model._is_vlm = False + model.apply_chat_template_kwargs = {} + model.tokenizer = SimpleNamespace( + apply_chat_template=lambda messages, **_: "prompt" + ) + model.model = SimpleNamespace(generate=lambda *_, **__: [completion]) + return model + + +@pytest.mark.parametrize( + "model_factory, provider, request_model, input_tokens, output_tokens", + [ + ( + _mlx_model, + "mlx", + "mlx-community/Qwen2.5-0.5B-Instruct-4bit", + 3, + 2, + ), + (_vllm_model, "vllm", "Qwen/Qwen2.5-0.5B-Instruct", 4, 2), + ], + ids=["mlx", "vllm"], +) +def test_local_runtime_response_is_recorded( + instrument_with_content, + span_exporter, + monkeypatch: pytest.MonkeyPatch, + model_factory: Any, + provider: str, + request_model: str, + input_tokens: int, + output_tokens: int, +) -> None: + output = model_factory(monkeypatch).generate( + messages=LOCAL_RUNTIME_MESSAGES + ) + assert output.content == "In Paris" + + (span,) = spans_by_operation(span_exporter.get_finished_spans(), "chat") + assert attr(span, GenAI.GEN_AI_PROVIDER_NAME) == provider + assert attr(span, GenAI.GEN_AI_REQUEST_MODEL) == request_model + assert attr(span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) == input_tokens + assert attr(span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) == output_tokens + # A local runtime returns no provider response envelope and listens on no + # socket, so it reports no finish reason, id or response model, and there is + # no endpoint to derive server.address from. + assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_ID) is None + assert attr(span, GenAI.GEN_AI_RESPONSE_MODEL) is None + assert attr(span, server_attributes.SERVER_ADDRESS) is None + assert attr(span, server_attributes.SERVER_PORT) is None + assert span.status.status_code == StatusCode.UNSET + + inputs = parse_messages(span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0]["parts"] == [ + {"type": "text", "content": "Where is the Louvre?"} + ] + outputs = parse_messages(span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["role"] == "assistant" + assert outputs[0]["parts"] == [{"type": "text", "content": "In Paris"}] + assert outputs[0]["finish_reason"] == "" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_tools.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_tools.py new file mode 100644 index 000000000..0dde7a1fb --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_tools.py @@ -0,0 +1,559 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Tool (execute_tool) instrumentation tests.""" + +from __future__ import annotations + +import json +import sys +import tempfile +from types import ModuleType +from typing import Any + +import pytest +from smolagents import CodeAgent, Tool, ToolCallingAgent, tool +from smolagents.models import ( + ChatMessage, + ChatMessageToolCall, + ChatMessageToolCallFunction, +) +from smolagents.monitoring import TokenUsage +from smolagents.tools import PipelineTool + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv.attributes import error_attributes +from opentelemetry.trace import StatusCode + +from .test_utils import ( + BrokenTool, + FakeCodeModel, + GetWeatherTool, + ImageTool, + attr, + spans_by_operation, +) + + +class ForecastTool(Tool): + """Two inputs, one of them optional, to exercise argument binding.""" + + name = "get_forecast" + description = "Get the forecast for a given city" + inputs = { + "location": {"type": "string", "description": "The city"}, + "unit": { + "type": "string", + "description": "The temperature unit", + "nullable": True, + }, + } + output_type = "string" + + def forward(self, location: str, unit: str = "C") -> str: + return f"sunny in {location} ({unit})" + + +def _tool_span(span_exporter: Any) -> Any: + (span,) = spans_by_operation( + span_exporter.get_finished_spans(), "execute_tool" + ) + return span + + +def test_tool_returns_string(instrument_with_content, span_exporter) -> None: + assert GetWeatherTool()("Paris") == "sunny" + + span = _tool_span(span_exporter) + assert span.name == "execute_tool get_weather" + assert attr(span, GenAI.GEN_AI_TOOL_NAME) == "get_weather" + assert attr(span, GenAI.GEN_AI_TOOL_TYPE) == "function" + assert ( + attr(span, GenAI.GEN_AI_TOOL_DESCRIPTION) + == "Get the weather for a given city" + ) + assert attr(span, GenAI.GEN_AI_TOOL_CALL_RESULT) == "sunny" + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_CALL_ARGUMENTS)) == { + "location": "Paris" + } + + +class ReversedInputsTool(Tool): + """A tool whose declared ``inputs`` are ordered differently from ``forward``. + + smolagents accepts this: ``inputs`` describes the schema the model sees, + while positional calls are bound by ``forward``'s signature. + """ + + name = "compare" + description = "Compare two cities" + inputs = { + "second": {"type": "string", "description": "The second city"}, + "first": {"type": "string", "description": "The first city"}, + } + output_type = "string" + + def forward(self, first: str, second: str) -> str: + return f"{first} vs {second}" + + +@pytest.mark.parametrize( + "tool_factory, args, kwargs, expected", + [ + # Positional arguments are named after forward()'s parameters... + (ForecastTool, ("Paris",), {}, {"location": "Paris"}), + # ...including when only some of them are positional. + ( + ForecastTool, + ("Paris",), + {"unit": "F"}, + {"location": "Paris", "unit": "F"}, + ), + (ForecastTool, ("Paris", "F"), {}, {"location": "Paris", "unit": "F"}), + ( + ForecastTool, + (), + {"location": "Paris", "unit": "F"}, + {"location": "Paris", "unit": "F"}, + ), + # A lone dict of declared inputs is forwarded as kwargs by Tool.__call__. + (ForecastTool, ({"location": "Paris"},), {}, {"location": "Paris"}), + # The declared inputs order is not the call order: naming arguments + # after it would record every value under the wrong name. + ( + ReversedInputsTool, + ("Paris", "Berlin"), + {}, + {"first": "Paris", "second": "Berlin"}, + ), + ], +) +def test_tool_argument_binding( + instrument_with_content, + span_exporter, + tool_factory: type[Tool], + args: tuple[Any, ...], + kwargs: dict[str, Any], + expected: dict[str, Any], +) -> None: + result = tool_factory()(*args, **kwargs) + + span = _tool_span(span_exporter) + arguments = json.loads(attr(span, GenAI.GEN_AI_TOOL_CALL_ARGUMENTS)) + assert arguments == expected + # The tool's own output proves which parameter each value really reached. + for value in expected.values(): + assert str(value) in result + + +def test_tool_argument_binding_falls_back_to_declared_inputs( + instrument_with_content, span_exporter +) -> None: + # The gradio and LangChain tool wrappers set + # skip_forward_signature_validation and declare a generic forward, which + # names nothing; the declared inputs are all there is to go on. + class GenericForwardTool(Tool): + skip_forward_signature_validation = True + name = "generic" + description = "Wraps a foreign tool" + inputs = {"query": {"type": "string", "description": "The query"}} + output_type = "string" + + def forward(self, *args: Any, **kwargs: Any) -> str: + return f"ran {args}{kwargs}" + + GenericForwardTool()("weather") + + span = _tool_span(span_exporter) + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_CALL_ARGUMENTS)) == { + "query": "weather" + } + + +def test_tool_argument_binding_drops_extra_positional( + instrument_with_content, span_exporter +) -> None: + # smolagents rejects the extra argument itself; the arguments recorded + # before the call must not name it after an input it doesn't belong to. + with pytest.raises(TypeError): + ForecastTool()("Paris", "F", "extra") + + span = _tool_span(span_exporter) + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_CALL_ARGUMENTS)) == { + "location": "Paris", + "unit": "F", + } + + +def test_tool_returns_dict(instrument_with_content, span_exporter) -> None: + class WeatherDictTool(Tool): + name = "get_weather" + description = "Get detailed weather" + inputs = {"location": {"type": "string", "description": "city"}} + output_type = "object" + + def forward(self, location: str) -> dict[str, Any]: + return {"condition": "sunny", "temperature": 25} + + assert WeatherDictTool()("Paris") == { + "condition": "sunny", + "temperature": 25, + } + + span = _tool_span(span_exporter) + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_CALL_RESULT)) == { + "condition": "sunny", + "temperature": 25, + } + + +def test_tool_returns_tuple(instrument_with_content, span_exporter) -> None: + @tool + def get_population(location: str) -> tuple[str, str]: + """Get population and location type. + + Args: + location: the location + """ + return f"Population in {location} is 10 million", "City" + + assert get_population("Paris") == ( + "Population in Paris is 10 million", + "City", + ) + + span = _tool_span(span_exporter) + assert attr(span, GenAI.GEN_AI_TOOL_NAME) == "get_population" + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_CALL_RESULT)) == [ + "Population in Paris is 10 million", + "City", + ] + + +def test_tool_returning_a_side_effecting_string_is_not_stringified( + instrument_with_content, span_exporter +) -> None: + # AgentText and AgentAudio subclass str and inherit AgentType.__str__, which + # calls to_string(); for audio that writes a .wav into a temp directory. + # Reading the underlying str keeps telemetry side-effect free. + class SideEffectingText(str): + def __str__(self) -> str: + raise AssertionError("telemetry called __str__") + + @tool + def transcribe(path: str) -> str: + """Transcribe a recording. + + Args: + path: the recording + """ + return SideEffectingText("transcript") + + transcribe("clip.wav") + + span = _tool_span(span_exporter) + assert attr(span, GenAI.GEN_AI_TOOL_CALL_RESULT) == "transcript" + + +def test_tool_returning_an_image_records_its_type( + instrument_with_content, span_exporter, monkeypatch +) -> None: + # util-genai falls back to str() for values it can't serialize, which for a + # PIL image is a heap address and for smolagents' AgentImage wrapper is a + # PNG written to a temp directory. + def _no_temp_dirs(*args: Any, **kwargs: Any) -> str: + raise AssertionError("telemetry wrote an image to disk") + + monkeypatch.setattr(tempfile, "mkdtemp", _no_temp_dirs) + + ImageTool()() + + span = _tool_span(span_exporter) + assert attr(span, GenAI.GEN_AI_TOOL_CALL_RESULT) == "Image" + + +def test_tool_content_capture_disabled( + instrument_no_content, span_exporter +) -> None: + assert GetWeatherTool()("Paris") == "sunny" + + span = _tool_span(span_exporter) + # Metadata stays; argument/result content is omitted. + assert attr(span, GenAI.GEN_AI_TOOL_NAME) == "get_weather" + assert attr(span, GenAI.GEN_AI_TOOL_CALL_ARGUMENTS) is None + assert attr(span, GenAI.GEN_AI_TOOL_CALL_RESULT) is None + + +def test_tool_error_reraises_and_records( + instrument_with_content, span_exporter +) -> None: + with pytest.raises(ValueError, match="tool exploded"): + BrokenTool()("Paris") + + span = _tool_span(span_exporter) + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ValueError" + + +class _ToolCallingModel: + """Replays a scripted list of tool calls, one message per step.""" + + model_id = "scripted-model" + kwargs: dict[str, Any] = {} + + def __init__(self, steps: list[list[tuple[str, str, Any]]]) -> None: + self._steps = steps + self.calls = 0 + + def generate(self, messages: list[Any], **kwargs: Any) -> ChatMessage: + step = self._steps[min(self.calls, len(self._steps) - 1)] + self.calls += 1 + return ChatMessage( + role="assistant", + content="", + tool_calls=[ + ChatMessageToolCall( + id=call_id, + type="function", + function=ChatMessageToolCallFunction( + name=name, arguments=arguments + ), + ) + for call_id, name, arguments in step + ], + token_usage=TokenUsage(input_tokens=1, output_tokens=1), + ) + + def __call__(self, *args: Any, **kwargs: Any) -> ChatMessage: + return self.generate(*args, **kwargs) + + def parse_tool_calls(self, message: ChatMessage) -> ChatMessage: + return message + + +def _call_ids_by_tool(span_exporter: Any) -> dict[str, Any]: + return { + attr(span, GenAI.GEN_AI_TOOL_NAME): attr( + span, GenAI.GEN_AI_TOOL_CALL_ID + ) + for span in spans_by_operation( + span_exporter.get_finished_spans(), "execute_tool" + ) + } + + +def test_tool_call_id_is_recorded( + instrument_with_content, span_exporter +) -> None: + # Tool.__call__ gets only the argument values, so the id comes from the + # ToolCall objects the step published. + agent = ToolCallingAgent( + tools=[GetWeatherTool()], + model=_ToolCallingModel( + [ + [("call_weather", "get_weather", {"location": "Paris"})], + [("call_final", "final_answer", "sunny in Paris")], + ] + ), + max_steps=3, + ) + agent.run("What is the weather in Paris?") + + assert _call_ids_by_tool(span_exporter) == { + "get_weather": "call_weather", + "final_answer": "call_final", + } + + +def test_parallel_tool_calls_each_record_their_own_id( + instrument_with_content, span_exporter +) -> None: + # Two calls in one message run in worker threads. Each tool claims the + # pending call that matches its name. + agent = ToolCallingAgent( + tools=[GetWeatherTool(), ForecastTool()], + model=_ToolCallingModel( + [ + [ + ("call_weather", "get_weather", {"location": "Paris"}), + ("call_forecast", "get_forecast", {"location": "Rome"}), + ], + [("call_final", "final_answer", "done")], + ] + ), + max_steps=3, + ) + agent.run("Compare Paris and Rome.") + + call_ids = _call_ids_by_tool(span_exporter) + assert call_ids["get_weather"] == "call_weather" + assert call_ids["get_forecast"] == "call_forecast" + + +def test_repeated_calls_to_one_tool_in_a_step_record_no_id( + instrument_with_content, span_exporter +) -> None: + # execute_tool_call rewrites state-variable arguments, so two calls to one + # tool in a step cannot be told apart. + agent = ToolCallingAgent( + tools=[GetWeatherTool()], + model=_ToolCallingModel( + [ + [ + ("call_paris", "get_weather", {"location": "Paris"}), + ("call_rome", "get_weather", {"location": "Rome"}), + ], + [("call_final", "final_answer", "done")], + ] + ), + max_steps=3, + ) + agent.run("Compare Paris and Rome.") + + weather_ids = { + attr(span, GenAI.GEN_AI_TOOL_CALL_ID) + for span in spans_by_operation( + span_exporter.get_finished_spans(), "execute_tool" + ) + if attr(span, GenAI.GEN_AI_TOOL_NAME) == "get_weather" + } + assert weather_ids == {None} + + +def test_code_agent_tool_span_records_no_call_id( + instrument_with_content, span_exporter +) -> None: + # A CodeAgent's model writes code, so there is no tool call id to record. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question") + + assert _call_ids_by_tool(span_exporter) == {"final_answer": None} + + +class _WeatherCodeModel(FakeCodeModel): + """Drives a CodeAgent to call ``get_weather`` from generated code.""" + + def generate(self, messages: list[Any], **kwargs: Any) -> ChatMessage: + return ChatMessage( + role="assistant", + content=( + "Thought: check the weather.\nCode:\n```py\n" + 'final_answer(get_weather("Rome"))\n```' + ), + token_usage=TokenUsage(input_tokens=1, output_tokens=1), + ) + + +def test_a_managed_agents_tool_does_not_claim_the_managers_call_id( + instrument_with_content, span_exporter +) -> None: + # The manager's step calls a managed CodeAgent and its own get_weather at + # once, and the managed agent calls a get_weather of its own from code. A + # CodeAgent publishes no tool calls, so unless the nested run clears the + # manager's, its tool would claim the manager's id. + managed = CodeAgent( + tools=[GetWeatherTool()], + model=_WeatherCodeModel(), + max_steps=2, + name="search_agent", + description="Runs searches.", + ) + manager = ToolCallingAgent( + tools=[GetWeatherTool()], + model=_ToolCallingModel( + [ + [ + ("call_delegate", "search_agent", {"task": "Look it up"}), + ("call_weather", "get_weather", {"location": "Paris"}), + ], + [("call_final", "final_answer", "done")], + ] + ), + managed_agents=[managed], + max_steps=2, + ) + manager.run("Delegate and check Paris.") + + weather_ids = sorted( + str(attr(span, GenAI.GEN_AI_TOOL_CALL_ID)) + for span in spans_by_operation( + span_exporter.get_finished_spans(), "execute_tool" + ) + if attr(span, GenAI.GEN_AI_TOOL_NAME) == "get_weather" + ) + # The manager's call keeps its id; the managed agent's does not take it. + assert weather_ids == ["None", "call_weather"] + + +class _EchoPipelineTool(PipelineTool): + """A ``PipelineTool`` that echoes its prompt. + + ``PipelineTool.__init__`` needs torch and accelerate; the test only needs + the inherited ``__call__``. + """ + + name = "echo_pipeline" + description = "Echoes its input through a pipeline" + inputs = {"prompt": {"type": "string", "description": "The prompt"}} + output_type = "string" + + def __init__(self) -> None: # pylint: disable=super-init-not-called + self.is_initialized = True + self.device = "cpu" + + def encode(self, raw_inputs: Any) -> dict[str, Any]: + return {"prompt": raw_inputs} + + def forward(self, inputs: dict[str, Any]) -> str: + return f"echo: {inputs['prompt']}" + + def decode(self, outputs: Any) -> Any: + return outputs + + +@pytest.fixture +def fake_torch_and_accelerate(monkeypatch): + """Satisfy the imports ``PipelineTool.__call__`` does at call time.""" + torch = ModuleType("torch") + # A real class, because __call__ splits its inputs with isinstance(). + setattr(torch, "Tensor", type("Tensor", (), {})) + accelerate = ModuleType("accelerate") + accelerate_utils = ModuleType("accelerate.utils") + setattr(accelerate_utils, "send_to_device", lambda obj, _device: obj) + setattr(accelerate, "utils", accelerate_utils) + monkeypatch.setitem(sys.modules, "torch", torch) + monkeypatch.setitem(sys.modules, "accelerate", accelerate) + monkeypatch.setitem(sys.modules, "accelerate.utils", accelerate_utils) + + +def test_pipeline_tool_emits_one_span( + instrument_with_content, span_exporter, fake_torch_and_accelerate +) -> None: + # PipelineTool overrides Tool.__call__ without delegating to it. Patching + # the defining class shadows Tool's patch, so a call emits one span. + assert _EchoPipelineTool()("hello") == "echo: hello" + + span = _tool_span(span_exporter) + assert attr(span, GenAI.GEN_AI_TOOL_NAME) == "echo_pipeline" + assert attr(span, GenAI.GEN_AI_TOOL_TYPE) == "function" + assert attr(span, GenAI.GEN_AI_TOOL_CALL_RESULT) == "echo: hello" + assert json.loads(attr(span, GenAI.GEN_AI_TOOL_CALL_ARGUMENTS)) == { + "prompt": "hello" + } + + +def test_pipeline_tool_error_reraises_and_records( + instrument_with_content, span_exporter, fake_torch_and_accelerate +) -> None: + class _BrokenPipelineTool(_EchoPipelineTool): + def forward(self, inputs: dict[str, Any]) -> str: + raise ValueError("pipeline exploded") + + with pytest.raises(ValueError, match="pipeline exploded"): + _BrokenPipelineTool()("hello") + + span = _tool_span(span_exporter) + assert span.status.status_code == StatusCode.ERROR + assert attr(span, error_attributes.ERROR_TYPE) == "ValueError" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py new file mode 100644 index 000000000..f9503366f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py @@ -0,0 +1,279 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers and fake models for smolagents instrumentation tests.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import Any + +from smolagents import Tool +from smolagents.models import ChatMessage, ChatMessageStreamDelta +from smolagents.monitoring import TokenUsage + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) + +CODE_FINAL_ANSWER = """ +Thought: Return the final answer. +Code: +```py +final_answer("Test result from CodeAgent") +``` +""" + +CODE_NO_OP = """ +Thought: Keep going. +Code: +```py +x = 1 +``` +""" + + +class FakeCodeModel: + """A stand-in model that drives a CodeAgent to a final answer. + + Its ``generate`` is not one of the wrapped model classes, so it emits no + ``chat`` span. The agent and tool spans come from the wrapped ``run`` and + ``__call__``. + """ + + def __init__(self, model_id: str = "fake-model") -> None: + self.model_id = model_id + self.kwargs: dict[str, Any] = {} + + def generate( + self, + messages: list[Any], + stop_sequences: list[str] | None = None, + response_format: Any = None, + tools_to_call_from: list[Any] | None = None, + **kwargs: Any, + ) -> ChatMessage: + return ChatMessage( + role="assistant", + content=CODE_FINAL_ANSWER, + token_usage=TokenUsage(input_tokens=11, output_tokens=7), + ) + + def __call__(self, *args: Any, **kwargs: Any) -> ChatMessage: + return self.generate(*args, **kwargs) + + +class FakeStreamingCodeModel(FakeCodeModel): + """A ``FakeCodeModel`` that also streams, for ``stream_outputs=True`` runs. + + smolagents yields every ``ChatMessageStreamDelta`` to the caller and then + yields the ``ActionStep`` carrying the aggregate of exactly those deltas, so + a run streams 11/7 tokens in total, not 22/14. + """ + + def generate_stream( + self, + messages: list[Any], + stop_sequences: list[str] | None = None, + response_format: Any = None, + tools_to_call_from: list[Any] | None = None, + **kwargs: Any, + ) -> Any: + yield ChatMessageStreamDelta( + content=CODE_FINAL_ANSWER, + token_usage=TokenUsage(input_tokens=11, output_tokens=7), + ) + + +class NeverFinishingCodeModel(FakeCodeModel): + """A ``FakeCodeModel`` that never calls ``final_answer``. + + A run driven by it uses up ``max_steps``, which smolagents reports by + appending an ``ActionStep`` carrying ``AgentMaxStepsError`` and returning a + synthesized answer rather than raising. + """ + + def generate(self, messages: list[Any], **kwargs: Any) -> ChatMessage: + return ChatMessage( + role="assistant", + content=CODE_NO_OP, + token_usage=TokenUsage(input_tokens=3, output_tokens=5), + ) + + +class ImageTool(Tool): + """Returns a PIL image, which a CodeAgent then hands to ``final_answer``.""" + + name = "make_image" + description = "Make a tiny image" + inputs: dict[str, Any] = {} + output_type = "image" + + def forward(self) -> Any: + from PIL import Image # noqa: PLC0415 + + return Image.new("RGB", (4, 4), color="red") + + +class GetWeatherTool(Tool): + name = "get_weather" + description = "Get the weather for a given city" + inputs = { + "location": { + "type": "string", + "description": "The city to get the weather for", + } + } + output_type = "string" + + def forward(self, location: str) -> str: + return "sunny" + + +class BrokenTool(Tool): + name = "broken_tool" + description = "A tool that always fails" + inputs = {"location": {"type": "string", "description": "ignored"}} + output_type = "string" + + def forward(self, location: str) -> str: + raise ValueError("tool exploded") + + +def stub_streaming_openai_client( + chunks: list[Any], error: Exception | None = None +) -> Any: + """An ``openai.OpenAI`` stand-in whose ``create`` returns a chunk stream. + + ``OpenAIModel.generate_stream`` reads ``event.usage`` and + ``event.choices[0].delta``, so the chunks are real ``ChatCompletionChunk`` + objects. ``error`` is raised after the chunks are yielded, which is how a + provider failure part-way through a stream reaches the caller. + """ + + def create(**_: Any) -> Any: + def stream() -> Any: + yield from chunks + if error is not None: + raise error + + return stream() + + return SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + +def text_chunk(content: str) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 + Choice, + ChoiceDelta, + ) + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[Choice(index=0, delta=ChoiceDelta(content=content))], + ) + + +def tool_call_chunk( + index: int, + call_id: str | None = None, + name: str | None = None, + arguments: str | None = None, +) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.chat.chat_completion_chunk import ( # noqa: PLC0415 + Choice, + ChoiceDelta, + ChoiceDeltaToolCall, + ChoiceDeltaToolCallFunction, + ) + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[ + Choice( + index=0, + delta=ChoiceDelta( + tool_calls=[ + ChoiceDeltaToolCall( + index=index, + id=call_id, + type="function", + function=ChoiceDeltaToolCallFunction( + name=name, arguments=arguments + ), + ) + ] + ), + ) + ], + ) + + +def usage_chunk(prompt_tokens: int, completion_tokens: int) -> Any: + from openai.types.chat import ChatCompletionChunk # noqa: PLC0415 + from openai.types.completion_usage import CompletionUsage # noqa: PLC0415 + + return ChatCompletionChunk( + id="chatcmpl-stub", + model="gpt-4o-2024-08-06", + object="chat.completion.chunk", + created=0, + choices=[], + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +def spans_by_operation( + spans: list[ReadableSpan], operation: str +) -> list[ReadableSpan]: + return [ + span + for span in spans + if (span.attributes or {}).get(GenAI.GEN_AI_OPERATION_NAME) + == operation + ] + + +def attr(span: ReadableSpan, name: str) -> Any: + return (span.attributes or {}).get(name) + + +def parse_messages(span: ReadableSpan, name: str) -> list[dict[str, Any]]: + raw = attr(span, name) + return json.loads(raw) if isinstance(raw, str) else [] + + +def part_types(messages: list[dict[str, Any]]) -> list[str]: + return [part["type"] for message in messages for part in message["parts"]] + + +def metrics_by_name(metric_reader: Any) -> dict[str, Any]: + data = metric_reader.get_metrics_data() + if data is None: + return {} + return { + metric.name: metric + for resource_metric in data.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + } + + +def data_point_attributes(metric: Any) -> list[dict[str, Any]]: + return [dict(point.attributes) for point in metric.data.data_points] diff --git a/tox.ini b/tox.ini index 55d61bfaf..7216c0952 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,7 @@ envlist = ; instrumentation-genai-smolagents py3{10,11,12,13,14}-test-instrumentation-genai-smolagents-latest py310-test-instrumentation-genai-smolagents-oldest + py314-test-instrumentation-genai-smolagents-conformance lint-instrumentation-genai-smolagents ; instrumentation-genai-anthropic @@ -153,6 +154,8 @@ deps = smolagents-latest: {[testenv]test_deps} smolagents-latest: {[testenv]pytest_deps} smolagents-latest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt + smolagents-conformance: {[testenv]pytest_deps} + smolagents-conformance: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt anthropic-oldest: {[testenv]pytest_deps} anthropic-oldest: -e {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic[instruments] @@ -245,6 +248,7 @@ commands = lint-instrumentation-genai-agno: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-agno" test-instrumentation-genai-smolagents-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests --vcr-record=none {posargs} + test-instrumentation-genai-smolagents-conformance: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py --vcr-record=none {posargs} lint-instrumentation-genai-smolagents: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-genai-smolagents" test-instrumentation-genai-anthropic-{oldest,latest}: pytest --ignore={toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests/test_conformance.py {toxinidir}/instrumentation/opentelemetry-instrumentation-genai-anthropic/tests --vcr-record=none {posargs}