diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added new file mode 100644 index 000000000..2c4b09dc1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added @@ -0,0 +1 @@ +(Openinference Migration: Langchain) - Capture multimodal image content (OpenAI ``image_url`` and Anthropic ``image`` blocks) as ``Blob``/``Uri`` message parts. diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index ef726f1ac..d0a0dbf7d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -4,6 +4,7 @@ from __future__ import annotations +import base64 import json from collections.abc import Iterable from typing import Any, cast @@ -19,6 +20,8 @@ gen_ai_attributes as GenAIAttributes, ) from opentelemetry.util.genai.types import ( + Blob, + ContentCapturingMode, FunctionToolDefinition, InputMessage, MessagePart, @@ -28,7 +31,9 @@ ToolCallRequest, ToolCallResponse, ToolDefinition, + Uri, ) +from opentelemetry.util.genai.utils import get_content_capturing_mode # Mapping from LangChain ``ls_provider`` metadata values to the well-known # ``gen_ai.provider.name`` values defined by the GenAI semantic conventions. @@ -75,6 +80,89 @@ def _normalize_role(message: BaseMessage) -> str: return _ROLE_MAP.get(message.type, message.type) +def _decode_base64(data: str) -> bytes | None: + # Skip the decode entirely when message content is not being captured; + # the resulting bytes would never be emitted under ``NO_CONTENT``. + if get_content_capturing_mode() is ContentCapturingMode.NO_CONTENT: + return None + try: + return base64.b64decode("".join(data.split()), validate=True) + except Exception: # pylint: disable=broad-exception-caught + return None + + +def _media_part(item: dict[str, Any]) -> MessagePart | None: + """Convert a LangChain multimodal image content block into a media part. + + Handles the two shapes LangChain chat models accept: + + - OpenAI style ``{"type": "image_url", "image_url": {"url": ...}}`` (or a + bare ``"image_url": "..."`` string). A ``data:;base64,`` + URL becomes a :class:`Blob`; any other URL becomes a :class:`Uri`. + - Anthropic style ``{"type": "image", "source": {...}}`` where ``source`` + is either ``{"type": "base64", "media_type": ..., "data": ...}`` (→ + :class:`Blob`) or ``{"type": "url", "url": ...}`` (→ :class:`Uri`). + """ + block_type = item.get("type") + if block_type == "image_url": + image_url = item.get("image_url") + url: str | None = None + if isinstance(image_url, str): + url = image_url + elif isinstance(image_url, dict): + image_url_dict = cast(dict[str, Any], image_url) + raw_url = image_url_dict.get("url") + url = raw_url if isinstance(raw_url, str) else None + if not url: + return None + return _image_from_url(url) + if block_type == "image": + source = item.get("source") + if not isinstance(source, dict): + return None + source_dict = cast(dict[str, Any], source) + source_type = source_dict.get("type") + if source_type == "base64": + data = source_dict.get("data") + if not isinstance(data, str): + return None + decoded = _decode_base64(data) + if decoded is None: + return None + media_type = source_dict.get("media_type") + return Blob( + mime_type=media_type if isinstance(media_type, str) else None, + modality="image", + content=decoded, + ) + if source_type == "url": + source_url = source_dict.get("url") + if isinstance(source_url, str) and source_url: + return _image_from_url(source_url) + return None + + +def _image_from_url(url: str) -> MessagePart | None: + """Return a :class:`Blob` for a ``data:`` URL, else a :class:`Uri`.""" + + if url.startswith("data:"): + header, _, payload = url[len("data:") :].partition(",") + mime_type = header.split(";", 1)[0] or None + if ";base64" in header: + decoded = _decode_base64(payload) + if decoded is None: + return None + content = decoded + else: + content = payload.encode("utf-8") + return Blob( + mime_type=mime_type, + modality="image", + content=content, + ) + return Uri(mime_type=None, modality="image", uri=url) + + def _content_to_parts( content: str | list[str | dict[str, Any]], ) -> list[MessagePart]: @@ -109,6 +197,10 @@ def _content_to_parts( ) if isinstance(reasoning_value, str) and reasoning_value: parts.append(Reasoning(content=reasoning_value)) + elif block_type in ("image_url", "image"): + media = _media_part(item) + if media is not None: + parts.append(media) return parts diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml new file mode 100644 index 000000000..935890c3b --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml @@ -0,0 +1,48 @@ +# TODO: this is generated by AI, re-record +# against the live Anthropic API once an ANTHROPIC_API_KEY is available. +interactions: +- request: + body: |- + {"model": "claude-sonnet-4-5", "max_tokens": 1024, "messages": [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3XQQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAgICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg=="}}]}], "temperature": 0.1} + headers: + Content-Type: + - application/json + User-Agent: + - !!binary | + QW50aHJvcGljL1B5dGhvbiAxLjAuMA== + x-api-key: + - test_key + anthropic-version: + - '2023-06-01' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_01MultimodalImagePlaceholder", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + { + "type": "text", + "text": "This is a tiny 1x1 pixel PNG image." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 16, + "output_tokens": 12 + } + } + headers: + Content-Type: + - application/json + Date: + - Thu, 04 Sep 2025 20:00:58 GMT + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml new file mode 100644 index 000000000..92a562269 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml @@ -0,0 +1,230 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": [ + { + "type": "text", + "text": "What is in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3XQQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAgICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg==" + } + } + ], + "role": "user" + } + ], + "model": "gpt-4o", + "max_completion_tokens": 100, + "stream": false, + "temperature": 0.1 + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '406' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.48.0 + x-stainless-arch: + - other:amd64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Windows + x-stainless-package-version: + - 2.48.0 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "choices": [ + { + "content_filter_results": { + "hate": { + "filtered": false, + "severity": "safe" + }, + "protected_material_code": { + "detected": false, + "filtered": false + }, + "protected_material_text": { + "detected": false, + "filtered": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "This image consists of a yellow square centered on a blue background.", + "refusal": null, + "role": "assistant" + } + } + ], + "created": 1785281713, + "id": "chatcmpl-E6lcHo1HleCxFE1A5235OXq03S1Jc", + "model": "gpt-4o-2024-11-20", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": { + "filtered": false, + "severity": "safe" + }, + "jailbreak": { + "detected": false, + "filtered": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + } + } + ], + "service_tier": "default", + "system_fingerprint": "fp_91d870f097", + "usage": { + "completion_tokens": 14, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "latency_checkpoint": { + "engine_tbt_ms": 10, + "engine_ttft_ms": 48, + "engine_ttlt_ms": 186, + "pre_inference_ms": 127, + "service_tbt_ms": 11, + "service_ttft_ms": 477, + "service_ttlt_ms": 617, + "total_duration_ms": 497, + "user_visible_ttft_ms": 350 + }, + "prompt_tokens": 223, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 237 + } + } + headers: + Set-Cookie: test_set_cookie + apim-request-id: + - 5761d0a9-ee6a-488a-9878-ddea53ab04ea + azureai-fe-is-streaming: + - 'False' + azureai-fe-requested-service-tier: + - PayGo + azureai-fe-requested-zone: + - hot + azureml-model-session: + - d20260721052918-9061a802 + content-length: + - '1523' + content-type: + - application/json + date: + - Tue, 28 Jul 2026 23:35:13 GMT + openai-organization: test_openai_org_id + openai-project: test_openai_project_id + skip-error-remapping: + - 'true' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-ms-client-request-id: + - Not-Set + x-ms-is-spilled-over: + - 'false' + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US 2 + x-ms-served-model: + - gpt-4o-2024-11-20 + x-ratelimit-abusepenalty-active: + - 'False' + x-ratelimit-key: + - gpt-4o + x-ratelimit-limit-requests: + - '600' + x-ratelimit-limit-tokens: + - '100000' + x-ratelimit-remaining-requests: + - '599' + x-ratelimit-remaining-tokens: + - '99012' + x-ratelimit-renewalperiod-requests: + - '10' + x-ratelimit-renewalperiod-tokens: + - '60' + x-ratelimit-reset-requests: + - '0' + x-ratelimit-reset-tokens: + - '0' + x-request-id: + - 5761d0a9-ee6a-488a-9878-ddea53ab04ea + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py index 95597a46e..d5c6d250a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py @@ -99,6 +99,16 @@ def fixture_gemini(): yield llm +@pytest.fixture(scope="function", name="chat_openai_vision") +def fixture_chat_openai_image(): + llm = ChatOpenAI( + model="gpt-4o", + temperature=0.1, + max_tokens=100, + ) + yield llm + + @pytest.fixture(scope="function") def start_instrumentation( tracer_provider, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 2f212b69c..e5418c5da 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -8,6 +8,7 @@ the callback-handler logic and the invocation-manager bookkeeping. """ +import base64 import uuid from unittest import mock @@ -20,7 +21,10 @@ OpenTelemetryLangChainCallbackHandler, ) from opentelemetry.instrumentation.genai.langchain.utils import ( + _decode_base64, + _image_from_url, _legacy_function_call_request, + _media_part, extract_token_details, make_input_message, make_last_output_message, @@ -36,10 +40,12 @@ WorkflowInvocation, ) from opentelemetry.util.genai.types import ( + Blob, InputMessage, OutputMessage, Text, ToolCallRequest, + Uri, ) # --------------------------------------------------------------------------- @@ -1764,3 +1770,339 @@ def test_empty_header_value_ignored(self): handler.on_llm_end(response=response, run_id=run_id) assert llm_inv.response_model_name is None + + +# --------------------------------------------------------------------------- +# utils - multimodal image parsing (_media_part / _image_from_url) +# --------------------------------------------------------------------------- + + +@pytest.fixture(name="enable_span_content") +def fixture_enable_span_content(monkeypatch): + """Enable message-content capture so inline media is base64-decoded. + + ``_decode_base64`` short-circuits to ``None`` under the default + ``NO_CONTENT`` mode, so tests that assert the decoded bytes must opt in. + """ + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + + +def test_image_from_url_data_uri_returns_blob(enable_span_content): + part = _image_from_url("data:image/jpeg;base64,QUJD") + assert isinstance(part, Blob) + assert part.mime_type == "image/jpeg" + assert part.modality == "image" + assert part.content == b"ABC" + + +def test_image_from_url_http_returns_uri(): + part = _image_from_url("https://example.com/cat.png") + assert isinstance(part, Uri) + assert part.uri == "https://example.com/cat.png" + assert part.modality == "image" + + +def test_image_from_url_data_uri_without_base64_keeps_text_bytes(): + part = _image_from_url("data:text/plain,hello") + assert isinstance(part, Blob) + assert part.mime_type == "text/plain" + assert part.content == b"hello" + + +def test_image_from_url_data_uri_no_mime_type(enable_span_content): + part = _image_from_url("data:;base64,QUJD") + assert isinstance(part, Blob) + assert part.mime_type is None + assert part.content == b"ABC" + + +def test_image_from_url_data_uri_malformed_base64_returns_none(): + part = _image_from_url("data:image/png;base64,not!valid!base64!") + assert part is None + + +def test_media_part_openai_image_url_dict(): + item = { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + } + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/a.png" + + +def test_media_part_openai_image_url_string(): + item = {"type": "image_url", "image_url": "https://example.com/b.png"} + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/b.png" + + +def test_media_part_anthropic_base64_source_returns_blob(enable_span_content): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "R0lGODlh", + }, + } + part = _media_part(item) + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == b"GIF89a" + + +def test_media_part_anthropic_base64_source_without_media_type( + enable_span_content, +): + item = { + "type": "image", + "source": {"type": "base64", "data": "QUJD"}, + } + part = _media_part(item) + assert isinstance(part, Blob) + assert part.mime_type is None + assert part.content == b"ABC" + + +def test_media_part_anthropic_url_source_returns_uri(): + item = { + "type": "image", + "source": {"type": "url", "url": "https://example.com/c.png"}, + } + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/c.png" + + +def test_media_part_unrecognized_returns_none(): + assert _media_part({"type": "text", "text": "hi"}) is None + assert _media_part({"type": "image_url", "image_url": {}}) is None + assert ( + _media_part({"type": "image_url", "image_url": {"url": 123}}) is None + ) + assert _media_part({"type": "image", "source": "nope"}) is None + assert ( + _media_part({"type": "image", "source": {"type": "base64", "data": 5}}) + is None + ) + assert ( + _media_part({"type": "image", "source": {"type": "url", "url": ""}}) + is None + ) + + +def test_media_part_malformed_base64_returns_none(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "not!valid!base64!", + }, + } + assert _media_part(item) is None + + +def test_decode_base64_valid_returns_bytes(enable_span_content): + assert _decode_base64("QUJD") == b"ABC" + + +def test_decode_base64_valid_with_padding(enable_span_content): + assert _decode_base64("R0lGODlh") == b"GIF89a" + + +def test_decode_base64_strips_whitespace_and_newlines(enable_span_content): + # Wrapped/whitespaced-but-valid payloads still decode. + assert _decode_base64("QU\nJD") == b"ABC" + assert _decode_base64(" QUJD ") == b"ABC" + assert _decode_base64("QU JD") == b"ABC" + + +def test_decode_base64_malformed_returns_none(): + # Non-base64 characters are rejected (validate=True), not silently + # dropped, so malformed input deterministically returns None. + assert _decode_base64("not!valid!base64!") is None + assert _decode_base64("@@@@") is None + assert _decode_base64("****") is None + + +def test_decode_base64_wrong_padding_returns_none(): + # Correct base64 alphabet but invalid length/padding. + assert _decode_base64("QUJ") is None + assert _decode_base64("QQ") is None + + +def test_decode_base64_empty_returns_empty_bytes(enable_span_content): + assert _decode_base64("") == b"" + + +_REAL_PNG_BYTES = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00" + b"\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc" + b"\xf8\xcf\xc0\xf0\x1f\x00\x05\x05\x02\x00\xa1\r\xf7\xdf\x00\x00\x00" + b"\x00IEND\xaeB`\x82" +) +_REAL_PNG_B64 = base64.b64encode(_REAL_PNG_BYTES).decode("ascii") + + +def test_decode_base64_real_image_round_trips(enable_span_content): + assert _decode_base64(_REAL_PNG_B64) == _REAL_PNG_BYTES + + +def test_image_from_url_real_png_data_uri_returns_blob(enable_span_content): + part = _image_from_url(f"data:image/png;base64,{_REAL_PNG_B64}") + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == _REAL_PNG_BYTES + + +def test_media_part_openai_real_png_data_uri_returns_blob(enable_span_content): + item = { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{_REAL_PNG_B64}"}, + } + part = _media_part(item) + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == _REAL_PNG_BYTES + + +def test_media_part_anthropic_real_png_source_returns_blob( + enable_span_content, +): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + } + part = _media_part(item) + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == _REAL_PNG_BYTES + + +def test_image_from_url_real_png_truncated_base64_returns_none(): + corrupted = _REAL_PNG_B64[:-4] + "!!!!" + part = _image_from_url(f"data:image/png;base64,{corrupted}") + assert part is None + + +def test_media_part_anthropic_real_png_corrupted_base64_returns_none(): + corrupted = _REAL_PNG_B64[:10] + "@@@@" + _REAL_PNG_B64[14:] + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": corrupted, + }, + } + assert _media_part(item) is None + + +def test_media_part_real_png_url_source_returns_uri(): + item = { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/real-image.png", + }, + } + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/real-image.png" + + +def test_to_input_messages_extracts_image_part(enable_span_content): + image_url = "data:image/jpeg;base64,QUJD" + content = [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": image_url}}, + ] + messages = to_input_messages([HumanMessage(content=content)]) + assert len(messages) == 1 + parts = messages[0].parts + + assert any(isinstance(p, Blob) for p in parts) + blob = next(p for p in parts if isinstance(p, Blob)) + assert blob.mime_type == "image/jpeg" + assert blob.content == b"ABC" + + +# --------------------------------------------------------------------------- +# utils - media decode is gated off under NO_CONTENT +# --------------------------------------------------------------------------- + + +@pytest.fixture(name="disable_content") +def fixture_disable_content(monkeypatch): + """Force the default ``NO_CONTENT`` capture mode deterministically.""" + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "NO_CONTENT" + ) + + +def test_decode_base64_no_content_returns_none(disable_content): + # Valid base64, but content capture is disabled: no decode is performed. + assert _decode_base64("QUJD") is None + assert _decode_base64(_REAL_PNG_B64) is None + + +def test_image_from_url_base64_no_content_returns_none(disable_content): + # A ``data:`` base64 URL cannot produce a Blob when the decode is gated off. + assert _image_from_url("data:image/jpeg;base64,QUJD") is None + assert _image_from_url(f"data:image/png;base64,{_REAL_PNG_B64}") is None + + +def test_media_part_openai_base64_no_content_returns_none(disable_content): + item = { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{_REAL_PNG_B64}"}, + } + assert _media_part(item) is None + + +def test_media_part_anthropic_base64_no_content_returns_none(disable_content): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + } + assert _media_part(item) is None + + +def test_image_from_url_http_uri_unaffected_by_no_content(disable_content): + # URI references carry no decoded bytes, so they are still emitted; only + # the base64 decode is gated. + part = _image_from_url("https://example.com/cat.png") + assert isinstance(part, Uri) + assert part.uri == "https://example.com/cat.png" + + +def test_to_input_messages_no_content_keeps_text_omits_image(disable_content): + # Existing text parsing is unchanged; only the base64-decoded image Blob + # is dropped under NO_CONTENT. + image_url = "data:image/jpeg;base64,QUJD" + content = [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": image_url}}, + ] + messages = to_input_messages([HumanMessage(content=content)]) + assert len(messages) == 1 + parts = messages[0].parts + + assert not any(isinstance(p, Blob) for p in parts) + assert any( + isinstance(p, Text) and p.content == "What's in this image?" + for p in parts + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index 8529a061b..07fa5f44f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -1,6 +1,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +import base64 from importlib.metadata import version as _pkg_version from typing import Optional @@ -56,6 +57,13 @@ def _langchain_openai_version() -> tuple: # cannot hold on those versions. _supports_reasoning_token_details = _langchain_openai_version() >= (0, 2, 1) +_REAL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3X" + "QQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAg" + "ICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg==" +) +_REAL_PNG_BYTES = base64.b64decode(_REAL_PNG_B64) + # span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming from fixtures defined in conftest.py @pytest.mark.parametrize( @@ -196,6 +204,63 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( assert len(logs) == 0 +def test_chat_openai_multimodal_image_llm_call( + span_exporter, + start_instrumentation, + chat_openai_vision, + monkeypatch, + vcr, +): + """End-to-end: an OpenAI ``image_url`` content block is captured as an + image ``Blob`` part in ``gen_ai.input.messages``.""" + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + + messages = [ + HumanMessage( + content=[ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{_REAL_PNG_B64}" + }, + }, + ] + ), + ] + + payload = chat_openai_vision._get_request_payload(messages, stop=None) + if "n" in payload: + pytest.skip( + "langchain-openai < 1.0 sends a different request body " + "(explicit n/temperature); only the modern cassette is recorded" + ) + with vcr.use_cassette("test_chat_openai_multimodal_image_llm_call.yaml"): + chat_openai_vision.invoke(messages) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( + "gpt-4o" + ) + + input_message = span.attributes.get( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES + ) + assert input_message is not None + assert '"role":"user"' in input_message + assert '"type":"text"' in input_message + assert '"content":"What is in this image?"' in input_message + assert '"type":"blob"' in input_message + assert '"modality":"image"' in input_message + assert '"mime_type":"image/png"' in input_message + assert _REAL_PNG_B64 in input_message + + # span_exporter, start_instrumentation, us_amazon_nova_lite_v1_0 are coming from fixtures defined in conftest.py def test_us_amazon_nova_lite_v1_0_bedrock_llm_call( span_exporter, start_instrumentation, us_amazon_nova_lite_v1_0, vcr @@ -372,6 +437,58 @@ def test_chat_openai_legacy_function_call( assert '"location"' in tool_definitions +@pytest.mark.vcr() +def test_chat_anthropic_multimodal_image_llm_call( + span_exporter, + start_instrumentation, + chat_anthropic_claude_sonnet, + monkeypatch, +): + """End-to-end: an Anthropic ``image`` content block is captured as an + image ``Blob`` part in ``gen_ai.input.messages``.""" + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + + messages = [ + HumanMessage( + content=[ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + }, + ] + ), + ] + + chat_anthropic_claude_sonnet.invoke(messages) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( + "claude-sonnet-4-5" + ) + + input_message = span.attributes.get( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES + ) + assert input_message is not None + assert '"role":"user"' in input_message + assert '"type":"text"' in input_message + assert '"content":"What is in this image?"' in input_message + assert '"type":"blob"' in input_message + assert '"modality":"image"' in input_message + assert '"mime_type":"image/png"' in input_message + assert _REAL_PNG_B64 in input_message + + # span_exporter, start_instrumentation, gemini are coming from fixtures defined in conftest.py def test_gemini(span_exporter, start_instrumentation, gemini, vcr): messages = [