openai: convert list-of-parts message content to semconv message parts

instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/358.fixed

@@ -0,0 +1 @@
+fix chat message content being dropped from `gen_ai.input.messages`/`gen_ai.output.messages` when it is a list of content parts

instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import base64
 import json
 from typing import Any, Iterable, List, Mapping
 from urllib.parse import urlparse
@@ -22,13 +23,17 @@
     InferenceInvocation,
 )
 from opentelemetry.util.genai.types import (
+    Blob,
+    File,
     FunctionToolDefinition,
     InputMessage,
+    MessagePart,
     OutputMessage,
     Text,
     ToolCallRequest,
     ToolCallResponse,
     ToolDefinition,
+    Uri,
 )
 
 _OpenAIOmit = getattr(openai, "Omit", None)
@@ -165,11 +170,87 @@ def get_value(v: Any):
     return None
 
 
-def _is_text_part(content: Any) -> bool:
-    return isinstance(content, str) or (
-        isinstance(content, Iterable)
-        and all(isinstance(part, str) for part in content)
-    )
+def _decode_base64(data: str) -> bytes | None:
+    try:
+        return base64.b64decode(data)
+    except Exception:  # pylint: disable=broad-exception-caught
+        return None
+
+
+def _convert_content_part(part: Any) -> MessagePart | None:
+    """Convert one OpenAI content part (TypedDict/dict or attribute object)
+    to a semconv MessagePart. Returns None for unrecognized parts."""
+    if isinstance(part, str):
+        return Text(content=part)
+
+    part_type = get_property_value(part, "type")
+
+    if part_type == "text":
+        text = get_property_value(part, "text")
+        return Text(content=str(text) if text is not None else "")
+
+    if part_type == "image_url":
+        image_url = get_property_value(part, "image_url")
+        url = (
+            get_property_value(image_url, "url")
+            if image_url is not None
+            else None
+        )
+        if not isinstance(url, str):
+            return None
+        # Both remote (http/https) and inline (data:) URLs are URIs; record
+        # them as sent rather than decoding inline payloads.
+        return Uri(mime_type=None, modality="image", uri=url)
+
+    if part_type == "input_audio":
+        input_audio = get_property_value(part, "input_audio")
+        if input_audio is None:
+            return None
+        data = get_property_value(input_audio, "data")
+        decoded = _decode_base64(data) if isinstance(data, str) else None
+        if decoded is None:
+            return None
+        audio_format = get_property_value(input_audio, "format")
+        return Blob(
+            mime_type=f"audio/{audio_format}"
+            if isinstance(audio_format, str)
+            else None,
+            modality="audio",
+            content=decoded,
+        )
+
+    if part_type == "file":
+        file_obj = get_property_value(part, "file")
+        file_id = (
+            get_property_value(file_obj, "file_id")
+            if file_obj is not None
+            else None
+        )
+        if not isinstance(file_id, str):
+            return None
+        return File(mime_type=None, modality="document", file_id=file_id)
+
+    if part_type == "refusal":
+        # The refusal string is the message's user-visible text content.
+        refusal = get_property_value(part, "refusal")
+        return Text(content=str(refusal) if refusal is not None else "")
+
+    return None
+
+
+def _content_to_parts(content: Any) -> list[MessagePart]:
+    """Convert an OpenAI message ``content`` value — a plain string or a
+    list of content parts — to semconv message parts."""
+    if isinstance(content, str):
+        return [Text(content=content)]
+    if isinstance(content, Iterable):
+        parts: list[MessagePart] = []
+        for item in content:
+            part = _convert_content_part(item)
+            if part is not None:
+                parts.append(part)
+        return parts
+    return []
 
 
 def _prepare_input_messages(messages) -> List[InputMessage]:
@@ -185,8 +266,7 @@ def _prepare_input_messages(messages) -> List[InputMessage]:
             tool_calls = get_property_value(message, "tool_calls")
             if tool_calls:
                 chat_message.parts += extract_tool_calls_new(tool_calls)
-            if _is_text_part(content):
-                chat_message.parts.append(Text(content=str(content)))
+            chat_message.parts += _content_to_parts(content)
 
         elif role == "tool":
             tool_call_id = get_property_value(message, "tool_call_id")
@@ -196,8 +276,7 @@ def _prepare_input_messages(messages) -> List[InputMessage]:
 
         else:
             # system, developer, user, fallback
-            if _is_text_part(content):
-                chat_message.parts.append(Text(content=str(content)))
+            chat_message.parts += _content_to_parts(content)
     return chat_messages
 
 
@@ -254,8 +333,7 @@ def _prepare_output_messages(choices) -> List[OutputMessage]:
             if tool_calls:
                 parts += extract_tool_calls_new(tool_calls)
             content = get_property_value(choice.message, "content")
-            if _is_text_part(content):
-                parts.append(Text(content=str(content)))
+            parts += _content_to_parts(content)
 
             message = OutputMessage(
                 finish_reason=choice.finish_reason or "error",

instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py

@@ -0,0 +1,234 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Unit tests for ``_prepare_input_messages`` content-part handling.
+
+Chat Completions ``content`` may be a plain string or a list of typed
+content parts (text, image_url, input_audio, file, refusal). The list
+form used to be dropped entirely (recorded as ``parts=[]``); these tests
+pin the conversion of every part shape.
+"""
+
+from __future__ import annotations
+
+import base64
+from types import SimpleNamespace
+
+from opentelemetry.instrumentation.genai.openai.utils import (
+    _prepare_input_messages,
+)
+from opentelemetry.util.genai.types import (
+    Blob,
+    File,
+    Text,
+    ToolCallRequest,
+    Uri,
+)
+
+
+def test_string_content_is_single_text_part():
+    messages = [{"role": "user", "content": "Say this is a test"}]
+
+    result = _prepare_input_messages(messages)
+
+    assert len(result) == 1
+    assert result[0].role == "user"
+    assert result[0].parts == [Text(content="Say this is a test")]
+
+
+def test_text_content_parts_list():
+    # The OpenAI multi-part content shape; previously recorded as parts=[].
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "user: hello"},
+                {"type": "text", "text": "assistant: hi"},
+            ],
+        }
+    ]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [
+        Text(content="user: hello"),
+        Text(content="assistant: hi"),
+    ]
+
+
+def test_system_message_with_content_parts_list():
+    messages = [
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": "You are helpful."}],
+        }
+    ]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].role == "system"
+    assert result[0].parts == [Text(content="You are helpful.")]
+
+
+def test_text_part_as_attribute_object():
+    part = SimpleNamespace(type="text", text="from an object")
+    messages = [{"role": "user", "content": [part]}]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [Text(content="from an object")]
+
+
+def test_mixed_text_and_image_url_parts():
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://example.com/cat.png"},
+                },
+            ],
+        }
+    ]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [
+        Text(content="What is in this image?"),
+        Uri(
+            mime_type=None,
+            modality="image",
+            uri="https://example.com/cat.png",
+        ),
+    ]
+
+
+def test_image_url_data_uri_recorded_as_uri():
+    data_uri = "data:image/png;base64,aGVsbG8="
+    messages = [
+        {
+            "role": "user",
+            "content": [{"type": "image_url", "image_url": {"url": data_uri}}],
+        }
+    ]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [
+        Uri(mime_type=None, modality="image", uri=data_uri)
+    ]
+
+
+def test_input_audio_part_decoded_to_blob():
+    audio_bytes = b"fake wav bytes"
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "input_audio",
+                    "input_audio": {
+                        "data": base64.b64encode(audio_bytes).decode(),
+                        "format": "wav",
+                    },
+                }
+            ],
+        }
+    ]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [
+        Blob(mime_type="audio/wav", modality="audio", content=audio_bytes)
+    ]
+
+
+def test_file_part_by_file_id():
+    messages = [
+        {
+            "role": "user",
+            "content": [{"type": "file", "file": {"file_id": "file-123"}}],
+        }
+    ]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [
+        File(mime_type=None, modality="document", file_id="file-123")
+    ]
+
+
+def test_assistant_list_content_and_tool_calls():
+    messages = [
+        {
+            "role": "assistant",
+            "content": [{"type": "text", "text": "checking the weather"}],
+            "tool_calls": [
+                {
+                    "id": "call_1",
+                    "function": {
+                        "name": "get_weather",
+                        "arguments": '{"city": "Seattle"}',
+                    },
+                }
+            ],
+        }
+    ]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [
+        ToolCallRequest(
+            id="call_1",
+            name="get_weather",
+            arguments={"city": "Seattle"},
+        ),
+        Text(content="checking the weather"),
+    ]
+
+
+def test_assistant_refusal_part_recorded_as_text():
+    messages = [
+        {
+            "role": "assistant",
+            "content": [{"type": "refusal", "refusal": "I cannot do that."}],
+        }
+    ]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [Text(content="I cannot do that.")]
+
+
+def test_unrecognized_part_is_skipped():
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "someday_a_new_modality", "payload": "x"},
+                {"type": "text", "text": "still captured"},
+            ],
+        }
+    ]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [Text(content="still captured")]
+
+
+def test_list_of_plain_strings():
+    messages = [{"role": "user", "content": ["one", "two"]}]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == [Text(content="one"), Text(content="two")]
+
+
+def test_none_content_yields_no_parts():
+    messages = [{"role": "assistant", "content": None}]
+
+    result = _prepare_input_messages(messages)
+
+    assert result[0].parts == []