From 56a80b2c4b73f9e0dfc84c888ddeea3980976a4f Mon Sep 17 00:00:00 2001 From: HQidea Date: Wed, 5 Aug 2026 10:50:19 +0800 Subject: [PATCH 1/2] openai: convert list-of-parts message content to semconv message parts Chat Completions `content` may be a plain string or a list of typed content parts. `_prepare_input_messages` (and `_prepare_output_messages`) gated content on `_is_text_part`, which only accepts `str` or an iterable of `str`, so the list form was dropped entirely and such messages were recorded in `gen_ai.input.messages` as `{"role": ..., "parts": []}`. Replace the gate with a per-part converter mirroring the anthropic package's `convert_content_to_parts`: - `{"type": "text"}` parts -> `Text` (one per part) - `{"type": "image_url"}` -> `Uri` (modality `image`; data: URLs recorded as sent, not decoded) - `{"type": "input_audio"}` -> `Blob` (modality `audio`, base64-decoded) - `{"type": "file"}` with `file_id` -> `File` - `{"type": "refusal"}` -> `Text` (the message's user-visible text) - unrecognized part types are skipped instead of nuking the message Plain-string content behaves exactly as before. A list of plain strings now yields one `Text` part per string (previously the whole list was stringified into a single part). Fixes #357 Co-Authored-By: Claude Fable 5 --- .../instrumentation/genai/openai/utils.py | 100 +++++++- .../tests/test_prepare_input_messages_unit.py | 234 ++++++++++++++++++ 2 files changed, 323 insertions(+), 11 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py index edb63e322..1d969d198 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/utils.py @@ -3,6 +3,7 @@ from __future__ import annotations +import base64 import json from typing import Any, Iterable, List, Mapping from urllib.parse import urlparse @@ -22,13 +23,17 @@ InferenceInvocation, ) from opentelemetry.util.genai.types import ( + Blob, + File, FunctionToolDefinition, InputMessage, + MessagePart, OutputMessage, Text, ToolCallRequest, ToolCallResponse, ToolDefinition, + Uri, ) _OpenAIOmit = getattr(openai, "Omit", None) @@ -165,11 +170,87 @@ def get_value(v: Any): return None -def _is_text_part(content: Any) -> bool: - return isinstance(content, str) or ( - isinstance(content, Iterable) - and all(isinstance(part, str) for part in content) - ) +def _decode_base64(data: str) -> bytes | None: + try: + return base64.b64decode(data) + except Exception: # pylint: disable=broad-exception-caught + return None + + +def _convert_content_part(part: Any) -> MessagePart | None: + """Convert one OpenAI content part (TypedDict/dict or attribute object) + to a semconv MessagePart. Returns None for unrecognized parts.""" + if isinstance(part, str): + return Text(content=part) + + part_type = get_property_value(part, "type") + + if part_type == "text": + text = get_property_value(part, "text") + return Text(content=str(text) if text is not None else "") + + if part_type == "image_url": + image_url = get_property_value(part, "image_url") + url = ( + get_property_value(image_url, "url") + if image_url is not None + else None + ) + if not isinstance(url, str): + return None + # Both remote (http/https) and inline (data:) URLs are URIs; record + # them as sent rather than decoding inline payloads. + return Uri(mime_type=None, modality="image", uri=url) + + if part_type == "input_audio": + input_audio = get_property_value(part, "input_audio") + if input_audio is None: + return None + data = get_property_value(input_audio, "data") + decoded = _decode_base64(data) if isinstance(data, str) else None + if decoded is None: + return None + audio_format = get_property_value(input_audio, "format") + return Blob( + mime_type=f"audio/{audio_format}" + if isinstance(audio_format, str) + else None, + modality="audio", + content=decoded, + ) + + if part_type == "file": + file_obj = get_property_value(part, "file") + file_id = ( + get_property_value(file_obj, "file_id") + if file_obj is not None + else None + ) + if not isinstance(file_id, str): + return None + return File(mime_type=None, modality="document", file_id=file_id) + + if part_type == "refusal": + # The refusal string is the message's user-visible text content. + refusal = get_property_value(part, "refusal") + return Text(content=str(refusal) if refusal is not None else "") + + return None + + +def _content_to_parts(content: Any) -> list[MessagePart]: + """Convert an OpenAI message ``content`` value — a plain string or a + list of content parts — to semconv message parts.""" + if isinstance(content, str): + return [Text(content=content)] + if isinstance(content, Iterable): + parts: list[MessagePart] = [] + for item in content: + part = _convert_content_part(item) + if part is not None: + parts.append(part) + return parts + return [] def _prepare_input_messages(messages) -> List[InputMessage]: @@ -185,8 +266,7 @@ def _prepare_input_messages(messages) -> List[InputMessage]: tool_calls = get_property_value(message, "tool_calls") if tool_calls: chat_message.parts += extract_tool_calls_new(tool_calls) - if _is_text_part(content): - chat_message.parts.append(Text(content=str(content))) + chat_message.parts += _content_to_parts(content) elif role == "tool": tool_call_id = get_property_value(message, "tool_call_id") @@ -196,8 +276,7 @@ def _prepare_input_messages(messages) -> List[InputMessage]: else: # system, developer, user, fallback - if _is_text_part(content): - chat_message.parts.append(Text(content=str(content))) + chat_message.parts += _content_to_parts(content) return chat_messages @@ -254,8 +333,7 @@ def _prepare_output_messages(choices) -> List[OutputMessage]: if tool_calls: parts += extract_tool_calls_new(tool_calls) content = get_property_value(choice.message, "content") - if _is_text_part(content): - parts.append(Text(content=str(content))) + parts += _content_to_parts(content) message = OutputMessage( finish_reason=choice.finish_reason or "error", diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py new file mode 100644 index 000000000..2214f2eda --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_prepare_input_messages_unit.py @@ -0,0 +1,234 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for ``_prepare_input_messages`` content-part handling. + +Chat Completions ``content`` may be a plain string or a list of typed +content parts (text, image_url, input_audio, file, refusal). The list +form used to be dropped entirely (recorded as ``parts=[]``); these tests +pin the conversion of every part shape. +""" + +from __future__ import annotations + +import base64 +from types import SimpleNamespace + +from opentelemetry.instrumentation.genai.openai.utils import ( + _prepare_input_messages, +) +from opentelemetry.util.genai.types import ( + Blob, + File, + Text, + ToolCallRequest, + Uri, +) + + +def test_string_content_is_single_text_part(): + messages = [{"role": "user", "content": "Say this is a test"}] + + result = _prepare_input_messages(messages) + + assert len(result) == 1 + assert result[0].role == "user" + assert result[0].parts == [Text(content="Say this is a test")] + + +def test_text_content_parts_list(): + # The OpenAI multi-part content shape; previously recorded as parts=[]. + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "user: hello"}, + {"type": "text", "text": "assistant: hi"}, + ], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [ + Text(content="user: hello"), + Text(content="assistant: hi"), + ] + + +def test_system_message_with_content_parts_list(): + messages = [ + { + "role": "system", + "content": [{"type": "text", "text": "You are helpful."}], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].role == "system" + assert result[0].parts == [Text(content="You are helpful.")] + + +def test_text_part_as_attribute_object(): + part = SimpleNamespace(type="text", text="from an object") + messages = [{"role": "user", "content": [part]}] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [Text(content="from an object")] + + +def test_mixed_text_and_image_url_parts(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/cat.png"}, + }, + ], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [ + Text(content="What is in this image?"), + Uri( + mime_type=None, + modality="image", + uri="https://example.com/cat.png", + ), + ] + + +def test_image_url_data_uri_recorded_as_uri(): + data_uri = "data:image/png;base64,aGVsbG8=" + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": data_uri}}], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [ + Uri(mime_type=None, modality="image", uri=data_uri) + ] + + +def test_input_audio_part_decoded_to_blob(): + audio_bytes = b"fake wav bytes" + messages = [ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": base64.b64encode(audio_bytes).decode(), + "format": "wav", + }, + } + ], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [ + Blob(mime_type="audio/wav", modality="audio", content=audio_bytes) + ] + + +def test_file_part_by_file_id(): + messages = [ + { + "role": "user", + "content": [{"type": "file", "file": {"file_id": "file-123"}}], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [ + File(mime_type=None, modality="document", file_id="file-123") + ] + + +def test_assistant_list_content_and_tool_calls(): + messages = [ + { + "role": "assistant", + "content": [{"type": "text", "text": "checking the weather"}], + "tool_calls": [ + { + "id": "call_1", + "function": { + "name": "get_weather", + "arguments": '{"city": "Seattle"}', + }, + } + ], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [ + ToolCallRequest( + id="call_1", + name="get_weather", + arguments={"city": "Seattle"}, + ), + Text(content="checking the weather"), + ] + + +def test_assistant_refusal_part_recorded_as_text(): + messages = [ + { + "role": "assistant", + "content": [{"type": "refusal", "refusal": "I cannot do that."}], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [Text(content="I cannot do that.")] + + +def test_unrecognized_part_is_skipped(): + messages = [ + { + "role": "user", + "content": [ + {"type": "someday_a_new_modality", "payload": "x"}, + {"type": "text", "text": "still captured"}, + ], + } + ] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [Text(content="still captured")] + + +def test_list_of_plain_strings(): + messages = [{"role": "user", "content": ["one", "two"]}] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [Text(content="one"), Text(content="two")] + + +def test_none_content_yields_no_parts(): + messages = [{"role": "assistant", "content": None}] + + result = _prepare_input_messages(messages) + + assert result[0].parts == [] From b8e3bb9c453b4c4ff7878056e74ce7d66f7f33c5 Mon Sep 17 00:00:00 2001 From: HQidea Date: Wed, 5 Aug 2026 10:51:32 +0800 Subject: [PATCH 2/2] Add changelog fragment for #358 Co-Authored-By: Claude Fable 5 --- .../.changelog/358.fixed | 1 + 1 file changed, 1 insertion(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/358.fixed diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/358.fixed b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/358.fixed new file mode 100644 index 000000000..7e65ba709 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/358.fixed @@ -0,0 +1 @@ +fix chat message content being dropped from `gen_ai.input.messages`/`gen_ai.output.messages` when it is a list of content parts