(Openinference Migration: Langchain): Capture multimodal image content (OpenAI image_url and Anthropic image blocks) as Blob/Uri message parts.

instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added

@@ -0,0 +1 @@
+(Openinference Migration: Langchain) - Capture multimodal image content (OpenAI ``image_url`` and Anthropic ``image`` blocks) as ``Blob``/``Uri`` message parts.

instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py

@@ -4,6 +4,7 @@
 
 from __future__ import annotations
 
+import base64
 import json
 from collections.abc import Iterable
 from typing import Any, cast
@@ -19,6 +20,8 @@
     gen_ai_attributes as GenAIAttributes,
 )
 from opentelemetry.util.genai.types import (
+    Blob,
+    ContentCapturingMode,
     FunctionToolDefinition,
     InputMessage,
     MessagePart,
@@ -28,7 +31,9 @@
     ToolCallRequest,
     ToolCallResponse,
     ToolDefinition,
+    Uri,
 )
+from opentelemetry.util.genai.utils import get_content_capturing_mode
 
 # Mapping from LangChain ``ls_provider`` metadata values to the well-known
 # ``gen_ai.provider.name`` values defined by the GenAI semantic conventions.
@@ -75,6 +80,89 @@ def _normalize_role(message: BaseMessage) -> str:
     return _ROLE_MAP.get(message.type, message.type)
 
 
+def _decode_base64(data: str) -> bytes | None:
+    # Skip the decode entirely when message content is not being captured;
+    # the resulting bytes would never be emitted under ``NO_CONTENT``.
+    if get_content_capturing_mode() is ContentCapturingMode.NO_CONTENT:
+        return None
+    try:
+        return base64.b64decode("".join(data.split()), validate=True)
+    except Exception:  # pylint: disable=broad-exception-caught
+        return None
+
+
+def _media_part(item: dict[str, Any]) -> MessagePart | None:
+    """Convert a LangChain multimodal image content block into a media part.
+
+    Handles the two shapes LangChain chat models accept:
+
+    - OpenAI style ``{"type": "image_url", "image_url": {"url": ...}}`` (or a
+      bare ``"image_url": "..."`` string). A ``data:<mime>;base64,<payload>``
+      URL becomes a :class:`Blob`; any other URL becomes a :class:`Uri`.
+    - Anthropic style ``{"type": "image", "source": {...}}`` where ``source``
+      is either ``{"type": "base64", "media_type": ..., "data": ...}`` (→
+      :class:`Blob`) or ``{"type": "url", "url": ...}`` (→ :class:`Uri`).
+    """
+    block_type = item.get("type")
+    if block_type == "image_url":
+        image_url = item.get("image_url")
+        url: str | None = None
+        if isinstance(image_url, str):
+            url = image_url
+        elif isinstance(image_url, dict):
+            image_url_dict = cast(dict[str, Any], image_url)
+            raw_url = image_url_dict.get("url")
+            url = raw_url if isinstance(raw_url, str) else None
+        if not url:
+            return None
+        return _image_from_url(url)
+    if block_type == "image":
+        source = item.get("source")
+        if not isinstance(source, dict):
+            return None
+        source_dict = cast(dict[str, Any], source)
+        source_type = source_dict.get("type")
+        if source_type == "base64":
+            data = source_dict.get("data")
+            if not isinstance(data, str):
+                return None
+            decoded = _decode_base64(data)
+            if decoded is None:
+                return None
+            media_type = source_dict.get("media_type")
+            return Blob(
+                mime_type=media_type if isinstance(media_type, str) else None,
+                modality="image",
+                content=decoded,
+            )
+        if source_type == "url":
+            source_url = source_dict.get("url")
+            if isinstance(source_url, str) and source_url:
+                return _image_from_url(source_url)
+    return None
+
+
+def _image_from_url(url: str) -> MessagePart | None:
+    """Return a :class:`Blob` for a ``data:`` URL, else a :class:`Uri`."""
+
+    if url.startswith("data:"):
+        header, _, payload = url[len("data:") :].partition(",")
+        mime_type = header.split(";", 1)[0] or None
+        if ";base64" in header:
+            decoded = _decode_base64(payload)
+            if decoded is None:
+                return None
+            content = decoded
+        else:
+            content = payload.encode("utf-8")
+        return Blob(
+            mime_type=mime_type,
+            modality="image",
+            content=content,
+        )
+    return Uri(mime_type=None, modality="image", uri=url)
+
+
 def _content_to_parts(
     content: str | list[str | dict[str, Any]],
 ) -> list[MessagePart]:
@@ -109,6 +197,10 @@ def _content_to_parts(
             )
             if isinstance(reasoning_value, str) and reasoning_value:
                 parts.append(Reasoning(content=reasoning_value))
+        elif block_type in ("image_url", "image"):
+            media = _media_part(item)
+            if media is not None:
+                parts.append(media)
     return parts
 
 

instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml

@@ -0,0 +1,48 @@
+# TODO: this is generated by AI, re-record
+# against the live Anthropic API once an ANTHROPIC_API_KEY is available.
+interactions:
+- request:
+    body: |-
+      {"model": "claude-sonnet-4-5", "max_tokens": 1024, "messages": [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3XQQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAgICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg=="}}]}], "temperature": 0.1}
+    headers:
+      Content-Type:
+      - application/json
+      User-Agent:
+      - !!binary |
+        QW50aHJvcGljL1B5dGhvbiAxLjAuMA==
+      x-api-key:
+      - test_key
+      anthropic-version:
+      - '2023-06-01'
+    method: POST
+    uri: https://api.anthropic.com/v1/messages
+  response:
+    body:
+      string: |-
+        {
+          "id": "msg_01MultimodalImagePlaceholder",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-sonnet-4-5",
+          "content": [
+            {
+              "type": "text",
+              "text": "This is a tiny 1x1 pixel PNG image."
+            }
+          ],
+          "stop_reason": "end_turn",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 16,
+            "output_tokens": 12
+          }
+        }
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 04 Sep 2025 20:00:58 GMT
+    status:
+      code: 200
+      message: OK
+version: 1